Megatest

Check-in [32dacffcc7]
Login
Overview
Comment:added caching of specified databases (defaulted to megatest.db)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | cached-copy-srehman
Files: files | file ages | folders
SHA1: 32dacffcc7586a248326f10df59e90a07d388c0d
User & Date: srehman on 2016-10-27 16:24:47
Original Comment: added args to prepare for caching of specified databases (defaulted to megatest.db)
Other Links: branch diff | manifest | tags
Context
2016-10-27
17:55
Merged in v1.62 and few minor fixes check-in: 41915f7997 user: mrwellan tags: cached-copy-srehman
16:24
added caching of specified databases (defaulted to megatest.db) check-in: 32dacffcc7 user: srehman tags: cached-copy-srehman
2016-10-20
11:38
Create new branch named "cached-copy-srehman" check-in: 8f45ff2d6f user: srehman tags: cached-copy-srehman
Changes

Modified db.scm from [67b7a55241] to [dbffcb7557].

320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
(define (db:setup run-id #!key (local #f))
  (let* ((dbdir    (db:dbfile-path #f)) ;; (conc (configf:lookup *configdat* "setup" "linktree") "/.db"))
	 (dbstruct (make-dbr:dbstruct path: dbdir local: local)))
    dbstruct))

;; Open the classic megatest.db file in toppath
;;
(define (db:open-megatest-db)
  (let* ((dbpath       (conc *toppath* "/megatest.db"))
	 (dbexists     (file-exists? dbpath))
	 (db           (db:lock-create-open dbpath
					    (lambda (db)
					      (db:initialize-main-db db)
					      (db:initialize-run-id-db db))))
	 (write-access (file-write-access? dbpath)))
    (if (and dbexists (not write-access))







|
|







320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
(define (db:setup run-id #!key (local #f))
  (let* ((dbdir    (db:dbfile-path #f)) ;; (conc (configf:lookup *configdat* "setup" "linktree") "/.db"))
	 (dbstruct (make-dbr:dbstruct path: dbdir local: local)))
    dbstruct))

;; Open the classic megatest.db file in toppath
;;
(define (db:open-megatest-db #!key (path #f))
  (let* ((dbpath       (or path (conc *toppath* "/megatest.db")))
	 (dbexists     (file-exists? dbpath))
	 (db           (db:lock-create-open dbpath
					    (lambda (db)
					      (db:initialize-main-db db)
					      (db:initialize-run-id-db db))))
	 (write-access (file-write-access? dbpath)))
    (if (and dbexists (not write-access))
779
780
781
782
783
784
785























786
787
788
789
790
791
792
                              last_update INTEGER DEFAULT (strftime('%s','now')))")
  (sqlite3:execute maindb "CREATE TRIGGER  IF NOT EXISTS update_run_stats_trigger AFTER UPDATE ON run_stats
                             FOR EACH ROW
                               BEGIN 
                                 UPDATE run_stats SET last_update=(strftime('%s','now'))
                                   WHERE id=old.id;
                               END;"))
























;; options:
;;
;;  'killservers  - kills all servers
;;  'dejunk       - removes junk records
;;  'adj-testids  - move test-ids into correct ranges
;;  'old2new      - sync megatest.db records to .db/{main,1,2 ...}.db







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
                              last_update INTEGER DEFAULT (strftime('%s','now')))")
  (sqlite3:execute maindb "CREATE TRIGGER  IF NOT EXISTS update_run_stats_trigger AFTER UPDATE ON run_stats
                             FOR EACH ROW
                               BEGIN 
                                 UPDATE run_stats SET last_update=(strftime('%s','now'))
                                   WHERE id=old.id;
                               END;"))

(define (db:cache-for-read-only source target)
	(let* ((toppath  (launch:setup))
		(cache-db (db:open-megatest-db path: target))
		(source-db (db:open-megatest-db path: source))
		(curr-time (current-seconds))
		(res '()))
		(print source-db)
		(begin
			(if (not (file-exists? target)) 
				((db:sync-tables (db:sync-main-list source-db) source-db cache-db)
				(db:sync-tables db:sync-tests-only source-db cache-db)
				(db:clean-up-rundb cache-db))
				((sqlite3:for-each-row
				     (lambda (id release runname state status owner event_time comment fail_count pass_count )
				       (set! res (cons (id release runname state status owner event_time comment fail_count pass_count ) res)))
				     (db:dbdat-get-db source-db)
				     "SELECT id, release, runname, state, status, owner, event_time, comment, fail_count, pass_count FROM runs;"))
				)
			(print res)
			(sqlite3:finalize! (db:dbdat-get-db cache-db))
		   ))
	)

;; options:
;;
;;  'killservers  - kills all servers
;;  'dejunk       - removes junk records
;;  'adj-testids  - move test-ids into correct ranges
;;  'old2new      - sync megatest.db records to .db/{main,1,2 ...}.db
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
	 servers))

    ;; clear out junk records
    ;;
    (if (member 'dejunk options)
	(begin
	  (db:delay-if-busy mtdb)
	  (db:clean-up mtdb)))

    ;; adjust test-ids to fit into proper range
    ;;
    (if (member 'adj-testids options)
	(begin
	  (db:delay-if-busy mtdb)
	  (db:prep-megatest.db-for-migration mtdb)))







|







840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
	 servers))

    ;; clear out junk records
    ;;
    (if (member 'dejunk options)
	(begin
	  (db:delay-if-busy mtdb)
	  (db:clean-up mtdb)))	

    ;; adjust test-ids to fit into proper range
    ;;
    (if (member 'adj-testids options)
	(begin
	  (db:delay-if-busy mtdb)
	  (db:prep-megatest.db-for-migration mtdb)))

Modified megatest.scm from [f073a21a21] to [1120b5ee36].

257
258
259
260
261
262
263


264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
			"-o"
			"-log"
			"-archive"
			"-since"
			"-fields"
			"-recover-test" ;; run-id,test-id - used internally to recover a test stuck in RUNNING state
			"-sort"


			) 
		 (list  "-h" "-help" "--help"
			"-manual"
			"-version"
		        "-force"
		        "-xterm"
		        "-showkeys"
		        "-show-keys"
		        "-test-status"
			"-set-values"
			"-load-test-data"
			"-summarize-items"
		        "-gui"
			"-daemonize"
			"-preclean"
			"-rerun-clean"
			"-rerun-all"
			"-clean-cache"

			;; misc
			"-repl"
			"-lock"
			"-unlock"
			"-list-servers"
                        "-run-wait"      ;; wait on a run to complete (i.e. no RUNNING)
			"-local"         ;; run some commands using local db access







>
>
|
|
















|







257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
			"-o"
			"-log"
			"-archive"
			"-since"
			"-fields"
			"-recover-test" ;; run-id,test-id - used internally to recover a test stuck in RUNNING state
			"-sort"
			"-target-db"
			"-source-db"
			)
 		 (list  "-h" "-help" "--help"
			"-manual"
			"-version"
		        "-force"
		        "-xterm"
		        "-showkeys"
		        "-show-keys"
		        "-test-status"
			"-set-values"
			"-load-test-data"
			"-summarize-items"
		        "-gui"
			"-daemonize"
			"-preclean"
			"-rerun-clean"
			"-rerun-all"
			"-clean-cache"
			"-cache-db"
			;; misc
			"-repl"
			"-lock"
			"-unlock"
			"-list-servers"
                        "-run-wait"      ;; wait on a run to complete (i.e. no RUNNING)
			"-local"         ;; run some commands using local db access
480
481
482
483
484
485
486









487
488
489
490
491
492
493

(on-exit std-exit-procedure)

;;======================================================================
;; Misc general calls
;;======================================================================










;; handle a clean-cache request as early as possible
;;
(if (args:get-arg "-clean-cache")
    (begin
      (set! *didsomething* #t) ;; suppress the help output.
      (if (getenv "MT_TARGET") ;; no point in trying if no target
	  (if (args:get-arg "-runname")







>
>
>
>
>
>
>
>
>







482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504

(on-exit std-exit-procedure)

;;======================================================================
;; Misc general calls
;;======================================================================

(if (args:get-arg "-cache-db")
	(begin
		(set! *didsomething* #t)
		(let* ((temp-dir (or (args:get-arg "-target-db") (create-directory (conc "/tmp/" (getenv "USER") "/" (string-translate (current-directory) "/" "_")))))
			(target-db (conc temp-dir "/cached.db"))
			(source-db (args:get-arg "-source-db")))

		(db:cache-for-read-only source-db target-db))))

;; handle a clean-cache request as early as possible
;;
(if (args:get-arg "-clean-cache")
    (begin
      (set! *didsomething* #t) ;; suppress the help output.
      (if (getenv "MT_TARGET") ;; no point in trying if no target
	  (if (args:get-arg "-runname")