Megatest

Check-in [fb8df28768]
Login
Overview
Comment:Added brute-force-salvage-db back. It seems to work, at least sometimes.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.70
Files: files | file ages | folders
SHA1: fb8df287684cd147f6bcfa359e04688b4be37c22
User & Date: matt on 2022-09-18 19:31:27
Other Links: branch diff | manifest | tags
Context
2022-09-30
18:13
Updated megatest version to 1.7007 check-in: 11f871fa5c user: mmgraham tags: v1.70, v1.7007
2022-09-18
19:31
Added brute-force-salvage-db back. It seems to work, at least sometimes. check-in: fb8df28768 user: matt tags: v1.70
18:12
Add back and improve waiting for journal plus few other performance helpers check-in: 76fac9ef30 user: matt tags: v1.70
Changes

Modified api.scm from [3289b67f67] to [8aaec3e750].

233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
                   ((inc-var)                      (apply db:inc-var dbstruct params))
		   ((dec-var)                      (apply db:dec-var dbstruct params))
                   ((del-var)                      (apply db:del-var dbstruct params))
		   ((add-var)                      (apply db:add-var dbstruct params))

                   ;; STEPS
                   ((teststep-set-status!)         (apply db:teststep-set-status! dbstruct params))
                   ((delete-steps-for-test!)        (apply db:delete-steps-for-test! dbstruct params))
                   
                   ;; TEST DATA
                   ((test-data-rollup)             (apply db:test-data-rollup dbstruct params))
                   ((csv->test-data)               (apply db:csv->test-data dbstruct params))

                   ;; MISC
                   ((sync-inmem->db)               (let ((run-id (car params)))







|







233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
                   ((inc-var)                      (apply db:inc-var dbstruct params))
		   ((dec-var)                      (apply db:dec-var dbstruct params))
                   ((del-var)                      (apply db:del-var dbstruct params))
		   ((add-var)                      (apply db:add-var dbstruct params))

                   ;; STEPS
                   ((teststep-set-status!)         (apply db:teststep-set-status! dbstruct params))
                   ((delete-steps-for-test!)       (apply db:delete-steps-for-test! dbstruct params))
                   
                   ;; TEST DATA
                   ((test-data-rollup)             (apply db:test-data-rollup dbstruct params))
                   ((csv->test-data)               (apply db:csv->test-data dbstruct params))

                   ;; MISC
                   ((sync-inmem->db)               (let ((run-id (car params)))

Modified dbfile.scm from [3ea5a94d80] to [3c40c08f5f].

486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
    (if (and (file-write-access? fname)
	     (file-exists? busy-file))
	(begin
	  (if (common:low-noise-print 120 busy-file)
	      (dbfile:print-err "INFO: dbfile:cautious-open-database: journal file "
				busy-file" exists, trying again in few seconds."))
	  (thread-sleep! 1)
	;; (if (eq? tries-left 2)
	;;     (begin
	;; 	(dbfile:print-err "INFO: forcing journal rollup "busy-file)
	;; 	(dbfile:brute-force-salvage-db fname)))
	  (dbfile:cautious-open-database fname init-proc (- tries-left 1)))

	(let* ((result (condition-case
		         (if dir-access
			   (dbfile:with-simple-file-lock
			    (conc fname ".lock")
			    (lambda ()







|
|
|
|







486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
    (if (and (file-write-access? fname)
	     (file-exists? busy-file))
	(begin
	  (if (common:low-noise-print 120 busy-file)
	      (dbfile:print-err "INFO: dbfile:cautious-open-database: journal file "
				busy-file" exists, trying again in few seconds."))
	  (thread-sleep! 1)
	  (if (eq? tries-left 2)
	      (begin
	  	(dbfile:print-err "INFO: forcing journal rollup "busy-file)
	  	(dbfile:brute-force-salvage-db fname)))
	  (dbfile:cautious-open-database fname init-proc (- tries-left 1)))

	(let* ((result (condition-case
		         (if dir-access
			   (dbfile:with-simple-file-lock
			    (conc fname ".lock")
			    (lambda ()
1216
1217
1218
1219
1220
1221
1222
1223































1224
1225
1226
1227
1228
1229
1230
      (exn (busy)
	   (db:generic-error-printout exn "ERROR: database " fname
				      " is locked. Try copying to another location, remove original and copy back."))
      (exn (permission)(db:generic-error-printout exn "ERROR: database " fname " has some permissions problem."))
      (exn ()
	   (db:generic-error-printout exn "ERROR: Unknown error with database " fname " message: "
				      ((condition-property-accessor 'exn 'message) exn))))))
      
































;;======================================================================
;; file utils
;;======================================================================

;;======================================================================
;; lazy-safe get file mod time. on any error (file not existing etc.) return 0







|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
      (exn (busy)
	   (db:generic-error-printout exn "ERROR: database " fname
				      " is locked. Try copying to another location, remove original and copy back."))
      (exn (permission)(db:generic-error-printout exn "ERROR: database " fname " has some permissions problem."))
      (exn ()
	   (db:generic-error-printout exn "ERROR: Unknown error with database " fname " message: "
				      ((condition-property-accessor 'exn 'message) exn))))))

;;======================================================================
;; another attempt at a transactionized queue
;;======================================================================

;; ;; ;; (define *transaction-queues* (make-hash-table))
;; ;; ;; 
;; ;; ;; (define (db:get-queue run-id)
;; ;; ;;   (let* ((res (hash-table-ref/default *transaction-queues* run-id #f)))
;; ;; ;;     (if res
;; ;; ;; 	res
;; ;; ;; 	(let* ((newq (make-queue)))
;; ;; ;; 	  (hash-table-set! *transaction-queues* run-id newq)
;; ;; ;; 	  newq))))
;; ;; ;; 
;; ;; ;; (define (db:add-to-transaction-queue dbstruct proc params)
;; ;; ;;   (let* ((mbox (make-mailbox))
;; ;; ;; 	 (q    (db:get-queue run-id)))
;; ;; ;;     (queue-add! *transaction-queue* (list dbstruct proc mbox))
;; ;; ;;     (mailbox-receive mbox)))
;; ;; ;; 
;; ;; ;; (define (db:process-transaction-queue *dbstruct-dbs*)
;; ;; ;;   (for-each
;; ;; ;;    (lambda (run-id)
;; ;; ;;      (let* ((q (hash-table-ref *transaction-queue* run-id)))
;; ;; ;;        ;; with-transaction
;; ;; ;;        ;;     dbstruct
;; ;; ;;        ;; pop items from queue and execute them, return results via mailbox
;; ;; ;;        q
;; ;; ;;        ;; pop 
;; ;; ;;        ))
;; ;; ;;    (hash-table-keys *transaction-queues*)))

;;======================================================================
;; file utils
;;======================================================================

;;======================================================================
;; lazy-safe get file mod time. on any error (file not existing etc.) return 0

Modified http-transport.scm from [8048b5ff50] to [0337db9fa9].

481
482
483
484
485
486
487




488

489
490
491
492
493
494
495

	    ;; (thread-start! *watchdog*)
          ) 
	  (if (and no-sync-db
		   (common:low-noise-print 5 "sync-all")) ;; cheesy way to reduce frequency of running sync :)
              (begin
                (debug:print 0 *default-log-port* "keep-running calling db:all-db-sync at " (time->string (seconds->local-time) "%H:%M:%S"))




                (db:all-db-sync *dbstruct-dbs*)

                ;; (db:do-sync no-sync-db)
	        ;; (db:run-lock-and-sync *no-sync-db*)
              )
          )
      )
      
      ;; when things go wrong we don't want to be doing the various queries too often







>
>
>
>
|
>







481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500

	    ;; (thread-start! *watchdog*)
          ) 
	  (if (and no-sync-db
		   (common:low-noise-print 5 "sync-all")) ;; cheesy way to reduce frequency of running sync :)
              (begin
                (debug:print 0 *default-log-port* "keep-running calling db:all-db-sync at " (time->string (seconds->local-time) "%H:%M:%S"))

		;; This is tougher than it seems - have to deal with multiple dbs
		;; (db:process-transaction-queue *dbstruct-dbs*)

		(db:all-db-sync *dbstruct-dbs*)
		
                ;; (db:do-sync no-sync-db)
	        ;; (db:run-lock-and-sync *no-sync-db*)
              )
          )
      )
      
      ;; when things go wrong we don't want to be doing the various queries too often