Megatest

Check-in [07907018df]
Login
☰Home Timeline Files Branches Tags Tickets Wiki Login
Overview
Comment:Start of code to sync to megatest.db
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | megatest.db-sync
Files: files | file ages | folders
SHA1: 07907018dffab8aaa0bdb67d39c46000f57d8f63
User & Date: matt on 2014-03-18 02:14:05
Other Links: branch diff | manifest | tags
Context
2014-03-18
17:31
Fixed typo check-in: 1911dc98ed user: mrwellan tags: megatest.db-sync
02:14
Start of code to sync to megatest.db check-in: 07907018df user: matt tags: megatest.db-sync
01:51
Have test ids start at 30k * run-id check-in: d43128116f user: matt tags: v1.60
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Modified db.scm from [92d5c9e289] to [8946983652].

︙︙
133
134
135
136
137
138
139

140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158

159
160
161
162
163
164
165

    (if rdb
	rdb
	(let* ((dbpath       (db:dbfile-path run-id)) ;; (conc toppath "/db/" run-id ".db"))
	       (dbexists     (file-exists? dbpath))
	       (inmem        (if local #f (db:open-inmem-db)))
	       (refdb        (if local #f (db:open-inmem-db)))
	       (db           (sqlite3:open-database dbpath))

	       (write-access (file-write-access? dbpath))
	       (handler      (make-busy-timeout 136000)))
	  (if (and dbexists (not write-access))
	      (set! *db-write-access* #f)) ;; only unset so other db's also can use this control
	  (if write-access
	      (begin
		(if (not dbexists)
		    (begin
		      (db:initialize-run-id-db db)
		      (sqlite3:execute 
		       db
		       "INSERT OR IGNORE INTO tests (id,run_id,testname,event_time,item_path,state,status) VALUES (?,?,'bogustest',strftime('%s','now'),'nowherepath','DELETED','n/a');"
		       (* run-id 30000) ;; allow for up to 30k tests per run
		       run-id)
		      )) ;; add strings db to rundb, not in use yet
		(sqlite3:set-busy-handler! db handler)
		(sqlite3:execute db "PRAGMA synchronous = 1;"))) ;; was 0 but 0 is a gamble
	  (dbr:dbstruct-set-rundb! dbstruct db)
	  (dbr:dbstruct-set-inuse! dbstruct #t)

	  (if local
	      (begin
		(dbr:dbstruct-set-localdb! dbstruct run-id db) ;; (dbr:dbstruct-set-inmem! dbstruct db) ;; direct access ...
		db)
	      (begin
		(dbr:dbstruct-set-inmem! dbstruct inmem)
		(db:sync-tables db:sync-tests-only db inmem)








>



















>








133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167

    (if rdb
	rdb
	(let* ((dbpath       (db:dbfile-path run-id)) ;; (conc toppath "/db/" run-id ".db"))
	       (dbexists     (file-exists? dbpath))
	       (inmem        (if local #f (db:open-inmem-db)))
	       (refdb        (if local #f (db:open-inmem-db)))
	       (db           (sqlite3:open-database dbpath))
	       (olddb        (db:open-megatest-db))
	       (write-access (file-write-access? dbpath))
	       (handler      (make-busy-timeout 136000)))
	  (if (and dbexists (not write-access))
	      (set! *db-write-access* #f)) ;; only unset so other db's also can use this control
	  (if write-access
	      (begin
		(if (not dbexists)
		    (begin
		      (db:initialize-run-id-db db)
		      (sqlite3:execute 
		       db
		       "INSERT OR IGNORE INTO tests (id,run_id,testname,event_time,item_path,state,status) VALUES (?,?,'bogustest',strftime('%s','now'),'nowherepath','DELETED','n/a');"
		       (* run-id 30000) ;; allow for up to 30k tests per run
		       run-id)
		      )) ;; add strings db to rundb, not in use yet
		(sqlite3:set-busy-handler! db handler)
		(sqlite3:execute db "PRAGMA synchronous = 1;"))) ;; was 0 but 0 is a gamble
	  (dbr:dbstruct-set-rundb! dbstruct db)
	  (dbr:dbstruct-set-inuse! dbstruct #t)
	  (dbr:dbstruct-set-olddb! dbstruct olddb)
	  (if local
	      (begin
		(dbr:dbstruct-set-localdb! dbstruct run-id db) ;; (dbr:dbstruct-set-inmem! dbstruct db) ;; direct access ...
		db)
	      (begin
		(dbr:dbstruct-set-inmem! dbstruct inmem)
		(db:sync-tables db:sync-tests-only db inmem)
︙︙
177
178
179
180
181
182
183

184
185
186
187
188
189
190
191
192
193
194

195
196
197
198
199
200
201

	       ;; (link-tree-path  (configf:lookup *configdat* "setup" "linktree"))
	       (dbpath       (db:dbfile-path 0)) ;; (let ((dbdir (conc *toppath* "/db"))) ;; use this opportunity to create our db dir
			                         ;;       (if (not (directory-exists? dbdir))
				                 ;;           (create-direcory dbdir))
			                         ;;           (conc *toppath* "/db/main.db")))
	       (dbexists     (file-exists? dbpath))
	       (db           (sqlite3:open-database dbpath))

	       (write-access (file-write-access? dbpath))
	       (handler      (make-busy-timeout 136000)))
	  (if (and dbexists (not write-access))
	      (set! *db-write-access* #f))
	  (if write-access 
	      (begin
		(sqlite3:set-busy-handler! db handler)
		(sqlite3:execute db "PRAGMA synchronous = 0;")))
	  (if (not dbexists)
	      (db:initialize-main-db db))
	  (dbr:dbstruct-set-main! dbstruct db)

	  db))))

;; Make the dbstruct, setup up auxillary db's and call for main db at least once
;;
(define (db:setup run-id #!key (local #f))
  (let* ((dbdir    (conc (configf:lookup *configdat* "setup" "linktree") "/.db"))
	 (dbstruct (make-dbr:dbstruct path: dbdir local: local)))








>











>








179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205

	       ;; (link-tree-path  (configf:lookup *configdat* "setup" "linktree"))
	       (dbpath       (db:dbfile-path 0)) ;; (let ((dbdir (conc *toppath* "/db"))) ;; use this opportunity to create our db dir
			                         ;;       (if (not (directory-exists? dbdir))
				                 ;;           (create-direcory dbdir))
			                         ;;           (conc *toppath* "/db/main.db")))
	       (dbexists     (file-exists? dbpath))
	       (db           (sqlite3:open-database dbpath))
	       (olddb        (db:open-megatest-db))
	       (write-access (file-write-access? dbpath))
	       (handler      (make-busy-timeout 136000)))
	  (if (and dbexists (not write-access))
	      (set! *db-write-access* #f))
	  (if write-access 
	      (begin
		(sqlite3:set-busy-handler! db handler)
		(sqlite3:execute db "PRAGMA synchronous = 0;")))
	  (if (not dbexists)
	      (db:initialize-main-db db))
	  (dbr:dbstruct-set-main! dbstruct db)
	  (dbr:dbstruct-set-olddb! dbstruct olddb)
	  db))))

;; Make the dbstruct, setup up auxillary db's and call for main db at least once
;;
(define (db:setup run-id #!key (local #f))
  (let* ((dbdir    (conc (configf:lookup *configdat* "setup" "linktree") "/.db"))
	 (dbstruct (make-dbr:dbstruct path: dbdir local: local)))
︙︙
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235

236
237
238
239
240
241
242
243
244
245
246
247
248
249
250

251
252
253
254
255
256
257
258
259
260
261
262

	  (sqlite3:execute db "PRAGMA synchronous = 0;")))
    (if (not dbexists)
	(begin
	  (db:initialize-main-db db)
	  (db:initialize-run-id-db db)))
    db))

;; sync all touched runs to disk
;;
(define (db:sync-touched dbstruct #!key (force-sync #f))
  (let ((tot-synced 0))
    (for-each
     (lambda (runvec)
       (let ((mtime (vector-ref runvec (dbr:dbstruct-field-name->num 'mtime)))
	     (stime (vector-ref runvec (dbr:dbstruct-field-name->num 'stime)))
	     (rundb (vector-ref runvec (dbr:dbstruct-field-name->num 'rundb)))
	     (inmem (vector-ref runvec (dbr:dbstruct-field-name->num 'inmem)))
	     (refdb (vector-ref runvec (dbr:dbstruct-field-name->num 'refdb))))

	 (if (or (> mtime stime) force-sync)
	     (let ((num-synced (db:sync-tables db:sync-tests-only inmem refdb rundb)))
	       (set! tot-synced (+ tot-synced num-synced))
	       (vector-set! runvec (dbr:dbstruct-field-name->num 'stime) (current-milliseconds))))))
     (hash-table-values (vector-ref dbstruct 1)))
    tot-synced))

;; sync run to disk if touched
;;
(define (db:sync-touched dbstruct #!key (force-sync #f))
  (let ((mtime (dbr:dbstruct-get-mtime dbstruct))
	(stime (dbr:dbstruct-get-stime dbstruct))
	(rundb (dbr:dbstruct-get-rundb dbstruct))
	(inmem (dbr:dbstruct-get-inmem dbstruct))
	(refdb (dbr:dbstruct-get-refdb dbstruct)))

    (if (or (not (number? mtime))
	    (not (number? stime))
	    (> mtime stime)
	    force-sync)
	(let ((num-synced (db:sync-tables db:sync-tests-only inmem refdb rundb)))
	  (dbr:dbstruct-set-stime! dbstruct (current-milliseconds))
	  num-synced)
	0)))

;; close all opened run-id dbs
(define (db:close-all dbstruct)
  ;; finalize main.db








|
|
|
|
|
|
|
|
|
|
|
>
|
|
|
|
|
|








|
>




|








222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268

	  (sqlite3:execute db "PRAGMA synchronous = 0;")))
    (if (not dbexists)
	(begin
	  (db:initialize-main-db db)
	  (db:initialize-run-id-db db)))
    db))

;; ;; sync all touched runs to disk
;; ;;
;; (define (db:sync-touched dbstruct #!key (force-sync #f))
;;   (let ((tot-synced 0))
;;     (for-each
;;      (lambda (runvec)
;;        (let ((mtime (vector-ref runvec (dbr:dbstruct-field-name->num 'mtime)))
;; 	     (stime (vector-ref runvec (dbr:dbstruct-field-name->num 'stime)))
;; 	     (rundb (vector-ref runvec (dbr:dbstruct-field-name->num 'rundb)))
;; 	     (inmem (vector-ref runvec (dbr:dbstruct-field-name->num 'inmem)))
;; 	     (refdb (vector-ref runvec (dbr:dbstruct-field-name->num 'refdb)))
;; 	     (slave (vector-ref runvec (dbr:dbstruct-field-name->num 'slavedb))))
;; 	 (if (or (> mtime stime) force-sync)
;; 	     (let ((num-synced (db:sync-tables db:sync-tests-only inmem refdb rundb)))
;; 	       (set! tot-synced (+ tot-synced num-synced))
;; 	       (vector-set! runvec (dbr:dbstruct-field-name->num 'stime) (current-milliseconds))))))
;;      (hash-table-values (vector-ref dbstruct 1)))
;;     tot-synced))

;; sync run to disk if touched
;;
(define (db:sync-touched dbstruct #!key (force-sync #f))
  (let ((mtime (dbr:dbstruct-get-mtime dbstruct))
	(stime (dbr:dbstruct-get-stime dbstruct))
	(rundb (dbr:dbstruct-get-rundb dbstruct))
	(inmem (dbr:dbstruct-get-inmem dbstruct))
	(refdb (dbr:dbstruct-get-refdb dbstruct))
	(olddb (dbr:dbstruct-get-olddb dbstruct)))
    (if (or (not (number? mtime))
	    (not (number? stime))
	    (> mtime stime)
	    force-sync)
	(let ((num-synced (db:sync-tables db:sync-tests-only inmem refdb rundb slavedb olddb)))
	  (dbr:dbstruct-set-stime! dbstruct (current-milliseconds))
	  num-synced)
	0)))

;; close all opened run-id dbs
(define (db:close-all dbstruct)
  ;; finalize main.db
︙︙

Modified db_records.scm from [ee094931e6] to [139c08b6c7].

︙︙
23
24
25
26
27
28
29

30
31
32
33
34
35
36
37
38
39
40
41
42

43
44
45
46
47
48
49
50
51
52
53
54

(define-inline (dbr:dbstruct-get-inmem   vec)    (vector-ref  vec 5))
(define-inline (dbr:dbstruct-get-mtime   vec)    (vector-ref  vec 6))
(define-inline (dbr:dbstruct-get-rtime   vec)    (vector-ref  vec 7))
(define-inline (dbr:dbstruct-get-stime   vec)    (vector-ref  vec 8))
(define-inline (dbr:dbstruct-get-inuse   vec)    (vector-ref  vec 9))
(define-inline (dbr:dbstruct-get-refdb   vec)    (vector-ref  vec 10))
(define-inline (dbr:dbstruct-get-locdbs  vec)    (vector-ref  vec 11))


(define-inline (dbr:dbstruct-set-main!   vec val)(vector-set! vec 0 val))
(define-inline (dbr:dbstruct-set-strdb!  vec val)(vector-set! vec 1 val))
(define-inline (dbr:dbstruct-set-path!   vec val)(vector-set! vec 2 val))
(define-inline (dbr:dbstruct-set-local!  vec val)(vector-set! vec 3 val))
(define-inline (dbr:dbstruct-set-rundb!  vec val)(vector-set! vec 4 val))
(define-inline (dbr:dbstruct-set-inmem!  vec val)(vector-set! vec 5 val))
(define-inline (dbr:dbstruct-set-mtime!  vec val)(vector-set! vec 6 val))
(define-inline (dbr:dbstruct-set-rtime!  vec val)(vector-set! vec 7 val))
(define-inline (dbr:dbstruct-set-stime!  vec val)(vector-set! vec 8 val))
(define-inline (dbr:dbstruct-set-inuse!  vec val)(vector-set! vec 9 val))
(define-inline (dbr:dbstruct-set-refdb!  vec val)(vector-set! vec 10 val))
(define-inline (dbr:dbstruct-set-locdbs! vec val)(vector-set! vec 11 val))


;; constructor for dbstruct
;;
(define (make-dbr:dbstruct #!key (path #f)(local #f))
  (let ((v (make-vector 12 #f)))
    (dbr:dbstruct-set-path! v path)
    (dbr:dbstruct-set-local! v local)
    (dbr:dbstruct-set-locdbs! v (make-hash-table))
    v))

(define (dbr:dbstruct-get-localdb v run-id)
  (hash-table-ref/default (dbr:dbstruct-get-locdbs v) run-id #f))








>













>




|








23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

(define-inline (dbr:dbstruct-get-inmem   vec)    (vector-ref  vec 5))
(define-inline (dbr:dbstruct-get-mtime   vec)    (vector-ref  vec 6))
(define-inline (dbr:dbstruct-get-rtime   vec)    (vector-ref  vec 7))
(define-inline (dbr:dbstruct-get-stime   vec)    (vector-ref  vec 8))
(define-inline (dbr:dbstruct-get-inuse   vec)    (vector-ref  vec 9))
(define-inline (dbr:dbstruct-get-refdb   vec)    (vector-ref  vec 10))
(define-inline (dbr:dbstruct-get-locdbs  vec)    (vector-ref  vec 11))
(define-inline (dbr:dbstruct-get-olddb   vec)    (vector-ref  vec 12))

(define-inline (dbr:dbstruct-set-main!   vec val)(vector-set! vec 0 val))
(define-inline (dbr:dbstruct-set-strdb!  vec val)(vector-set! vec 1 val))
(define-inline (dbr:dbstruct-set-path!   vec val)(vector-set! vec 2 val))
(define-inline (dbr:dbstruct-set-local!  vec val)(vector-set! vec 3 val))
(define-inline (dbr:dbstruct-set-rundb!  vec val)(vector-set! vec 4 val))
(define-inline (dbr:dbstruct-set-inmem!  vec val)(vector-set! vec 5 val))
(define-inline (dbr:dbstruct-set-mtime!  vec val)(vector-set! vec 6 val))
(define-inline (dbr:dbstruct-set-rtime!  vec val)(vector-set! vec 7 val))
(define-inline (dbr:dbstruct-set-stime!  vec val)(vector-set! vec 8 val))
(define-inline (dbr:dbstruct-set-inuse!  vec val)(vector-set! vec 9 val))
(define-inline (dbr:dbstruct-set-refdb!  vec val)(vector-set! vec 10 val))
(define-inline (dbr:dbstruct-set-locdbs! vec val)(vector-set! vec 11 val))
(define-inline (dbr:dbstruct-set-olddb!  vec val)(vector-set! vec 12 val))

;; constructor for dbstruct
;;
(define (make-dbr:dbstruct #!key (path #f)(local #f))
  (let ((v (make-vector 13 #f)))
    (dbr:dbstruct-set-path! v path)
    (dbr:dbstruct-set-local! v local)
    (dbr:dbstruct-set-locdbs! v (make-hash-table))
    v))

(define (dbr:dbstruct-get-localdb v run-id)
  (hash-table-ref/default (dbr:dbstruct-get-locdbs v) run-id #f))
︙︙

This page was generated in about 0.118s by Fossil 2.24 [452f8e1ffc] 2024-03-26 15:36:00