Megatest

Check-in [58512a448e]
Login
Overview
Comment:Changed dbfile:cautious-open-database to use most of the code from db:lock-create-open
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.70
Files: files | file ages | folders
SHA1: 58512a448e329bded4bc7ec3eab29940d5cd4b0e
User & Date: mmgraham on 2022-05-30 23:05:09
Other Links: branch diff | manifest | tags
Context
2022-06-07
18:41
merged changes from 024073632a and 9a5898a74e. Fixes for db locking, syncing and dashboard startup check-in: 20a2d7904d user: mmgraham tags: v1.70
2022-05-30
23:05
Changed dbfile:cautious-open-database to use most of the code from db:lock-create-open check-in: 58512a448e user: mmgraham tags: v1.70
2022-05-27
19:21
Commented out some not-used fuctions, removed the server start every 120 seconds and added dbfile handle count check-in: b1db729de1 user: matt tags: v1.70
Changes

Modified dbfile.scm from [330fd055bc] to [fca9c494c5].

184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
;; Make the dbstruct, setup up auxillary db's and call for main db at least once
;;
;; called in http-transport and replicated in rmt.scm for *local* access. 
;;
(define (dbfile:setup do-sync areapath tmppath)
  (cond
   (*dbstruct-dbs*
    (dbfile:print-err "WARNING: dbfile:setup called when *dbstruct-dbs* is already initialized")
    *dbstruct-dbs*) ;; TODO: when multiple areas are supported, this optimization will be a hazard
   (else
    (let* ((dbstruct (make-dbr:dbstruct)))
      (set! *dbstruct-dbs* dbstruct)
      (dbr:dbstruct-areapath-set! dbstruct areapath)
      (dbr:dbstruct-tmppath-set!  dbstruct tmppath)
      dbstruct))))







|







184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
;; Make the dbstruct, setup up auxillary db's and call for main db at least once
;;
;; called in http-transport and replicated in rmt.scm for *local* access. 
;;
(define (dbfile:setup do-sync areapath tmppath)
  (cond
   (*dbstruct-dbs*
    ;; (dbfile:print-err "WARNING: dbfile:setup called when *dbstruct-dbs* is already initialized")
    *dbstruct-dbs*) ;; TODO: when multiple areas are supported, this optimization will be a hazard
   (else
    (let* ((dbstruct (make-dbr:dbstruct)))
      (set! *dbstruct-dbs* dbstruct)
      (dbr:dbstruct-areapath-set! dbstruct areapath)
      (dbr:dbstruct-tmppath-set!  dbstruct tmppath)
      dbstruct))))
457
458
459
460
461
462
463
464
465
466

467
468
469
470
471






472
473



474


475

476
477
478
479
480
481
482
483
484
485
486
487
488




489
490


491




492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512

;; if we are not a server create a db handle. this is not finalized
;; so watch for problems. I'm still not clear if it is needed to manually
;; finalize sqlite3 dbs with the sqlite3 egg.
;;

(define (dbfile:cautious-open-database fname init-proc #!optional (tries-left 50))
  (let* ((busy-file  (conc fname"-journal"))
	 (delay-time (* (- 51 tries-left) 1.1))
	 (retry      (lambda ()

		       (thread-sleep! delay-time)
		       (if (> tries-left 0)
			   (dbfile:cautious-open-database fname init-proc (- tries-left 1))))))
    (assert (>= tries-left 0) (conc "FATAL: too many attempts in dbfile:cautious-open-database of "fname", giving up."))
    (if (and (file-write-access? fname)






	     (file-exists? busy-file))
	(begin



	  (dbfile:print-err "INFO: dbfile:cautious-open-database: journal file " busy-file " exists, trying again in few seconds.")


	  (thread-sleep! 1)

	  (if (eq? tries-left 2)
	      (begin
		(dbfile:print-err "INFO: forcing journal rollup "busy-file)
		(dbfile:brute-force-salvage-db fname)))
	  (dbfile:cautious-open-database fname init-proc (- tries-left 1)))
	(let* ((db-exists (file-exists? fname))
	       (result (condition-case
			   (let* ((db (sqlite3:open-database fname)))
			     (if (and init-proc (not db-exists))
				 (init-proc db))
			     db)
			(exn (io-error)
			     (dbfile:print-err exn "ERROR: i/o error with " fname ". Check permissions, disk space etc. and try again.")




			     (retry))
			(exn (corrupt)


			     (dbfile:print-err exn "ERROR: database " fname " is corrupt. Repair it to proceed.")




			     (retry))
			(exn (busy)
			     (dbfile:print-err exn "ERROR: database " fname
					       " is locked. Try copying to another location, remove original and copy back.")
			     (retry))
			(exn (permission)(dbfile:print-err exn "ERROR: database " fname " has some permissions problem.")
			     (retry))
			(exn ()
			     (dbfile:print-err exn "ERROR: Unknown error with database " fname " message: "
					       ((condition-property-accessor 'exn 'message) exn))
			     (retry)))))
          #;(if (file-write-access? fname)
	    (dbfile:simple-file-release-lock lock-file))
	  result))))

(define (dbfile:brute-force-salvage-db fname)
  (let* ((backupfname (conc fname"-"(current-process-id)".bak"))
	 (cmd (conc "cp "fname" "backupfname";mv "fname" "(conc fname ".delme;")
		    "cp "backupfname" "fname)))
    (dbfile:print-err "WARNING: attempting recovery of file "fname" by running commands:\n"
		      "  "cmd)







|
<
|
>
|
|
<
<
|
>
>
>
>
>
>
|
<
>
>
>
|
>
>
|
>
|
|
<
|
<
|
<
|
|
|
|
<
|
>
>
>
>
|
|
>
>
|
>
>
>
>
|
|
|
|
<
|
<
<
<
|
|
|
<
<







457
458
459
460
461
462
463
464

465
466
467
468


469
470
471
472
473
474
475
476

477
478
479
480
481
482
483
484
485
486

487

488

489
490
491
492

493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510

511



512
513
514


515
516
517
518
519
520
521

;; if we are not a server create a db handle. this is not finalized
;; so watch for problems. I'm still not clear if it is needed to manually
;; finalize sqlite3 dbs with the sqlite3 egg.
;;

(define (dbfile:cautious-open-database fname init-proc #!optional (tries-left 50))
  (let* ((parent-dir   (or (pathname-directory fname)(current-directory))) ;; no parent? go local

         (raw-fname    (pathname-file fname))
	 (dir-writable (file-write-access? parent-dir))
	 (file-exists  (file-exists? fname))
	 (file-write   (if file-exists


			   (file-write-access? fname)
			   dir-writable )))
    ;; (mutex-lock! *db-open-mutex*) ;; tried this mutex, not clear it helped.
    (if file-write ;; dir-writable
	(condition-case
         (let* ((lockfname   (conc fname ".lock"))
                (readyfname  (conc parent-dir "/.ready-" raw-fname))
                (readyexists (file-exists? readyfname)))

           (if (not readyexists)
             (dbfile:simple-file-lock-and-wait lockfname)
           )
           (let ((db      (sqlite3:open-database fname)))
             (sqlite3:set-busy-handler! db (sqlite3:make-busy-timeout 136000))
             (sqlite3:execute db "PRAGMA synchronous = 0;")
             (if (not file-exists)
                 (init-proc db))
             (if (not readyexists)
                 (begin

                   (dbfile:simple-file-release-lock lockfname)

                   (with-output-to-file

                       readyfname
                     (lambda ()
                       (print "Ready at " (current-seconds))))))
             db))

         (exn (io-error)  (dbfile:print-err "ERROR: i/o error with " fname ". Check permissions, disk space etc. and try again."))
         (exn (corrupt)   (dbfile:print-err "ERROR: database " fname " is corrupt. Repair it to proceed."))
         (exn (busy)      (dbfile:print-err "ERROR: database " fname " is locked. Try copying to another location, remove original and copy back."))
         (exn (permission)(dbfile:print-err "ERROR: database " fname " has some permissions problem."))
         (exn () (dbfile:print-err "ERROR: Unknown error with database " fname " message: " ((condition-property-accessor 'exn 'message) exn))))


	(condition-case
         (begin
           (dbfile:print-err "WARNING: opening db in non-writable dir " fname)
           (let ((db (sqlite3:open-database fname)))
             (sqlite3:set-busy-handler! db (sqlite3:make-busy-timeout 136000))
             (sqlite3:execute db "PRAGMA synchronous = 0;")
	     ;; (mutex-unlock! *db-open-mutex*)
             db))
         (exn (io-error)  (dbfile:print-err "ERROR: i/o error with " fname ". Check permissions, disk space etc. and try again."))
         (exn (corrupt)   (dbfile:print-err "ERROR: database " fname " is corrupt. Repair it to proceed."))
         (exn (busy)      (dbfile:print-err "ERROR: database " fname " is locked. Try copying to another location, remove original and copy back."))

         (exn (permission)(dbfile:print-err "ERROR: database " fname " has some permissions problem."))



         (exn () (dbfile:print-err "ERROR: Unknown error with database " fname " message: " ((condition-property-accessor 'exn 'message) exn))))
	)))




(define (dbfile:brute-force-salvage-db fname)
  (let* ((backupfname (conc fname"-"(current-process-id)".bak"))
	 (cmd (conc "cp "fname" "backupfname";mv "fname" "(conc fname ".delme;")
		    "cp "backupfname" "fname)))
    (dbfile:print-err "WARNING: attempting recovery of file "fname" by running commands:\n"
		      "  "cmd)