Megatest

Diff
Login

Differences From Artifact [6257400a66]:

To Artifact [ec5e9b082c]:


392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
	       "ERROR: database " fname " has some permissions problem."))
         (exn ()
	      (dbfile:print-and-exit
	       "ERROR: Unknown error with database " fname " message: " ((condition-property-accessor 'exn 'message) exn))))
	)))


;; This routine creates the db if not already present. It is only called if the db is not already opened
;;
#;(define (db:init-dbstruct dbstruct run-id init-proc #!key (do-sync #t))
  (let* ((subdb          (dbfile:get-subdb dbstruct run-id))
	 (tmpdb-stack    (dbr:subdb-dbstack subdb)) 
	 (max-stale-tmp  (dbr:dbstruct-max-stale-secs dbstruct));; (configf:lookup-number *configdat* "server" "filling-db-max-stale-seconds" default: 10))
	   (dbpath       (dbr:dbstruct-tmppath dbstruct)) ;;  (db:dbfile-path))      ;; path to tmp db area
	   (dbname       (dbfile:run-id->dbname run-id))
	   (dbexists     (file-exists? dbpath))
	   (areapath     (dbr:dbstruct-areapath dbstruct))
	   (mtdbfname    (conc areapath "/"dbname))
	   (mtdbexists   (file-exists? mtdbfname))
	   (mtdbmodtime  (if mtdbexists (dbfile:lazy-sqlite-db-modification-time mtdbfname)  #f))
	   (mtdb         (db:open-sqlite-db mtdbfname init-proc))
	   ;; the reference db for syncing
	   (refdbfname   (conc dbpath "/"dbname"_ref"))
	   (refndb       (db:open-megatest-db refdbfname))
	   ;; (mtdbpath     (dbr:dbdat-dbfile mtdb))
	   ;; the tmpdb
	   (tmpdbfname   (conc dbpath"/"dbname)) ;; /tmp/<stuff>/.db/[main|1,2...].db 
	   (tmpdb        (db:open-megatest-db tmpdbfname)) ;; lock-create-open dbpath db:initialize-main-db))
	   (dbfexists    (common:file-exists? tmpdbfname))  ;; (conc dbpath "/megatest.db")))
	   (tmpdbmodtime (if dbfexists  (common:lazy-sqlite-db-modification-time tmpdbfname) #f))
	   
	   (write-access (file-write-access? mtdbfname))
	   
	   ;; (mtdbmodtime (if mtdbexists
	   ;; (common:lazy-sqlite-db-modification-time mtdbpath)
	   ;; #f)) ; moving this before db:open-megatest-db is
	   ;; called. if wal mode is on -WAL and -shm file get
	   ;; created with causing the tmpdbmodtime timestamp
	   ;; always greater than mtdbmodtime (tmpdbmodtime (if
	   ;; dbfexists (common:lazy-sqlite-db-modification-time
	   ;; tmpdbfname) #f)) if wal mode is on -WAL and -shm
	   ;; file get created when db:open-megatest-db is
	   ;; called. modtimedelta will always be < 10 so db in
	   ;; tmp not get synced (tmpdbmodtime (if dbfexists
	   ;; (db:get-last-update-time (car tmpdb)) #f)) (fmt
	   ;; (file-modification-time tmpdbfname))
	   
	   (modtimedelta (and mtdbmodtime tmpdbmodtime (- mtdbmodtime tmpdbmodtime))))

    (when write-access
      (sqlite3:execute (dbr:dbdat-dbh mtdb) "drop trigger if exists update_tests_trigger")
      (sqlite3:execute (dbr:dbdat-dbh mtdb) "drop trigger if exists update_runs_trigger"))
    
    ;; (print "mtdbmodtime " mtdbmodtime " tmpdbmodtime " tmpdbmodtime " mtdbpath " mtdbpath " " (conc *toppath* "/megatest.db"))
    ;; (debug:print-info 13 *default-log-port* "db:open-db>> mtdbpath="mtdbpath" mtdbexists="mtdbexists" and write-access="write-access)
    (if (and dbexists (not write-access))
	(begin
	  (set! *db-write-access* #f)
	  (dbr:subdb-read-only-set! subdb #t)))
    (dbr:subdb-mtdb-set!   subdb mtdb)
    (dbr:subdb-tmpdb-set!  subdb tmpdb)
    (dbr:subdb-dbstack-set! subdb (make-stack)) ;; BB: why a stack?  Why would the number of db's be indeterminate?  Is this a legacy of 1.db 2.db .. ?
    (stack-push! (dbr:subdb-dbstack subdb) tmpdb) ;; olddb is already a (cons db path)
    (dbr:subdb-refndb-set! subdb refndb)
    (if (and  (or (not dbfexists)
		  (and modtimedelta
		       (> modtimedelta max-stale-tmp))) ;; if db in tmp is over ten seconds older than the file in MTRA then do a sync back
	      do-sync)
	(begin
	  (dbfile:print-err "filling db " (dbr:dbdat-dbfile tmpdb) " with data \n    from " (dbr:dbdat-dbfile mtdb) " mod time delta: " modtimedelta)
	  (db:sync-tables (db:sync-all-tables-list subdb) #f mtdb refndb tmpdb)
	  ;; touch tmp db to avoid wal mode wierdness  
	  (set! (file-modification-time tmpdbfname) (current-seconds))  
	  (dbfile:print-err "INFO: db:sync-all-tables-list done.")
	  )
	(dbfile:print-err " db, " (dbr:dbdat-dbfile tmpdb) " already exists or fresh enough, not propogating data from\n     " (dbr:dbdat-dbfile mtdb) " mod time delta: " modtimedelta) )
    ;; (db:multi-db-sync subdb 'old2new))  ;; migrate data from megatest.db automatically
    tmpdb))
		
;;======================================================================
;; no-sync.db - small bits of data to be shared between servers
;;======================================================================

;; if we are not a server create a db handle. this is not finalized
;; so watch for problems. I'm still not clear if it is needed to manually







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







392
393
394
395
396
397
398







































































399
400
401
402
403
404
405
	       "ERROR: database " fname " has some permissions problem."))
         (exn ()
	      (dbfile:print-and-exit
	       "ERROR: Unknown error with database " fname " message: " ((condition-property-accessor 'exn 'message) exn))))
	)))









































































		
;;======================================================================
;; no-sync.db - small bits of data to be shared between servers
;;======================================================================

;; if we are not a server create a db handle. this is not finalized
;; so watch for problems. I'm still not clear if it is needed to manually
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
                         slave-dbs)))
                   (for-each
                    (lambda (bad-dbdat)
                      (dbfile:print-err "db:sync-tables called with todb not a read-only database " bad-dbdat))
                    readonly-slave-dbs)
                   readonly-slave-dbs))) -6)
    (else
     (dbfile:print-err "db:sync-tables: args are good")

     (let ((stmts       (make-hash-table)) ;; table-field => stmt
	   (all-stmts   '())              ;; ( ( stmt1 value1 ) ( stml2 value2 ))
	   (numrecs     (make-hash-table))
	   (start-time  (current-milliseconds))
	   (tot-count   0))
       (for-each ;; table







|







805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
                         slave-dbs)))
                   (for-each
                    (lambda (bad-dbdat)
                      (dbfile:print-err "db:sync-tables called with todb not a read-only database " bad-dbdat))
                    readonly-slave-dbs)
                   readonly-slave-dbs))) -6)
    (else
    ;; (dbfile:print-err "db:sync-tables: args are good")

     (let ((stmts       (make-hash-table)) ;; table-field => stmt
	   (all-stmts   '())              ;; ( ( stmt1 value1 ) ( stml2 value2 ))
	   (numrecs     (make-hash-table))
	   (start-time  (current-milliseconds))
	   (tot-count   0))
       (for-each ;; table
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
					      #f))
                      (is-trigger-dropped (if (member "last_update" field-names)
                                              (db:is-trigger-dropped db tablename)
					      #f)) 
		      (stmth  (sqlite3:prepare db full-ins))
                      (changed-rows 0))
		 ;; (db:delay-if-busy targdb) ;; NO WAITING
                 (if (member "last_update" field-names)
                     (dbfile:print-err "is-trigger-dropped: " is-trigger-dropped)) 

		 (for-each
		  (lambda (fromdat-lst)
		    (sqlite3:with-transaction
		     db
		     (lambda ()
		       (for-each ;; 







|
|







921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
					      #f))
                      (is-trigger-dropped (if (member "last_update" field-names)
                                              (db:is-trigger-dropped db tablename)
					      #f)) 
		      (stmth  (sqlite3:prepare db full-ins))
                      (changed-rows 0))
		 ;; (db:delay-if-busy targdb) ;; NO WAITING
                 ;; (if (member "last_update" field-names)
                 ;;    (dbfile:print-err "is-trigger-dropped: " is-trigger-dropped)) 

		 (for-each
		  (lambda (fromdat-lst)
		    (sqlite3:with-transaction
		     db
		     (lambda ()
		       (for-each ;;