Megatest

Check-in [4fedbd6b6f]
Login
Overview
Comment:Fixed two cases where dbdats were not being pushed back on stack when done
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.70
Files: files | file ages | folders
SHA1: 4fedbd6b6f17605051aad59bfc4198afae783cee
User & Date: matt on 2022-05-20 19:49:05
Other Links: branch diff | manifest | tags
Context
2022-05-22
15:42
Ensure that db opens are only done once per process per db file. Put out messages if this is not the case. check-in: d0bca99717 user: matt tags: v1.70
2022-05-20
19:49
Fixed two cases where dbdats were not being pushed back on stack when done check-in: 4fedbd6b6f user: matt tags: v1.70
2022-05-15
05:01
Rebased v1.7001-multi-db forward to tip of v1.65, old branch is hidden check-in: 820ac9a873 user: matt tags: v1.70
Changes

Modified db.scm from [767b31815a] to [dc9a4cd486].

173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
;; 	      ;; NOTE: pushing on the stack only happens AFTER the handle has been used
;; 	      ;; (stack-push! (dbr:dbstruct-dbstack dbstruct) newdb)
;; 	      newdb)
;;           (stack-pop! (dbr:subdb-dbstack subdb)))
;; 	(db:open-db subdb run-id))) ;; )


(define (db:get-db dbstruct run-id) 
   (let* ((subdb (dbfile:get-subdb dbstruct run-id))
        (dbdat (dbfile:get-dbdat dbstruct run-id)))
        (if (dbr:dbdat? dbdat)
          dbdat
          (dbfile:open-db *dbstruct-dbs* #f db:initialize-main-db)
        )
   )







|







173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
;; 	      ;; NOTE: pushing on the stack only happens AFTER the handle has been used
;; 	      ;; (stack-push! (dbr:dbstruct-dbstack dbstruct) newdb)
;; 	      newdb)
;;           (stack-pop! (dbr:subdb-dbstack subdb)))
;; 	(db:open-db subdb run-id))) ;; )


#;(define (db:get-db dbstruct run-id) 
   (let* ((subdb (dbfile:get-subdb dbstruct run-id))
        (dbdat (dbfile:get-dbdat dbstruct run-id)))
        (if (dbr:dbdat? dbdat)
          dbdat
          (dbfile:open-db *dbstruct-dbs* #f db:initialize-main-db)
        )
   )
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
	  #f
        ))))




(define (db:all-db-sync dbstruct)
  (db:open-db dbstruct #f)
  (let* ((data-synced 0) ;; count of changed records
    (tmp-area       (common:get-db-tmp-area))
    (dbfiles        (glob (conc tmp-area"/.db/*.db")))
    (sync-durations (make-hash-table))
    (no-sync-db        (db:open-no-sync-db))
    )
    (for-each
     (lambda (file)
       (debug:print-info 3 *default-log-port* "file: " file)
       (let* ((fname (conc (pathname-file file) ".db"))
	      (fulln (conc *toppath*"/.db/"fname))
	      (time1 (if (file-exists? file)
			 (file-modification-time file)







|
|



|
<







1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087

1088
1089
1090
1091
1092
1093
1094
	  #f
        ))))




(define (db:all-db-sync dbstruct)
  (let* ((dbdat (db:open-db dbstruct #f))
	 (data-synced       0) ;; count of changed records
    (tmp-area       (common:get-db-tmp-area))
    (dbfiles        (glob (conc tmp-area"/.db/*.db")))
    (sync-durations (make-hash-table))
    (no-sync-db        (db:open-no-sync-db)))

    (for-each
     (lambda (file)
       (debug:print-info 3 *default-log-port* "file: " file)
       (let* ((fname (conc (pathname-file file) ".db"))
	      (fulln (conc *toppath*"/.db/"fname))
	      (time1 (if (file-exists? file)
			 (file-modification-time file)
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124

1125
1126
1127
1128
1129
1130
1131

1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
		      (else
		       #f))))
	 (if do-cp
	     (let* ((start-time (current-milliseconds))
              (fname (pathname-file file))
              (runid (if (string= fname "main") #f (string->number fname)))
              )
	       (debug:print-info 3 *default-log-port* "db:all-db-sync: fname: " fname", delta: " (- time1 time2) " seconds")

	       (db:lock-and-delta-sync no-sync-db dbstruct fname runid)
	       (hash-table-set! sync-durations (conc fname".db") (- (current-milliseconds) start-time)))

	     (debug:print-info 3 *default-log-port* "skipping sync. " file " is up to date")
         )
       )
     )
     dbfiles
    )
  )

   #t
)






;; options:
;;
;;  'killservers  - kills all servers
;;  'dejunk       - removes junk records
;;  'adj-testids  - move test-ids into correct ranges
;;  'old2new      - sync megatest.db to /tmp/.../megatest.db and /tmp/.../megatest_ref.db
;;  'new2old      - sync /tmp/.../megatest.db to megatest.db and /tmp/.../megatest_ref.db (and update data_synced)
;;  'closeall     - close all opened dbs
;;  'schema       - attempt to apply schema changes
;;  run-ids: '(1 2 3 ...) or #f (for all)
;;
(define (db:multi-db-sync dbstruct . options)

  (db:open-db dbstruct #f)

  (let* ((data-synced 0) ;; count of changed records
    (tmp-area       (common:get-db-tmp-area))
    (dbfiles        (glob (conc tmp-area"/.db/*.db")))
    (sync-durations (make-hash-table))
    )

    (for-each
     (lambda (file)
       (debug:print-info 0 *default-log-port* "file: " file)
       (let* ((fname (conc (pathname-file file) ".db"))
	      (fulln (conc *toppath*"/.db/"fname))
	      (time1 (if (file-exists? file)
			 (file-modification-time file)







|
|

|
>

|
<
<

|
<
>
|
<
<
<
<
<
<













<
|
<
|


|
<
<







1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126


1127
1128

1129
1130






1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143

1144

1145
1146
1147
1148


1149
1150
1151
1152
1153
1154
1155
		      (else
		       #f))))
	 (if do-cp
	     (let* ((start-time (current-milliseconds))
              (fname (pathname-file file))
              (runid (if (string= fname "main") #f (string->number fname)))
              )
	       (debug:print-info 3 *default-log-port* "db:all-db-sync: fname: "
				 fname", delta: " (- time1 time2) " seconds")
	       (db:lock-and-delta-sync no-sync-db dbstruct fname runid)
	       (hash-table-set! sync-durations (conc fname".db")
				(- (current-milliseconds) start-time)))
	     (debug:print-info 3 *default-log-port* "skipping sync. " file " is up to date")
         )))


     dbfiles
     )

    (if dbdat (dbfile:add-dbdat dbstruct #f dbdat)))
  #t)







;; options:
;;
;;  'killservers  - kills all servers
;;  'dejunk       - removes junk records
;;  'adj-testids  - move test-ids into correct ranges
;;  'old2new      - sync megatest.db to /tmp/.../megatest.db and /tmp/.../megatest_ref.db
;;  'new2old      - sync /tmp/.../megatest.db to megatest.db and /tmp/.../megatest_ref.db (and update data_synced)
;;  'closeall     - close all opened dbs
;;  'schema       - attempt to apply schema changes
;;  run-ids: '(1 2 3 ...) or #f (for all)
;;
(define (db:multi-db-sync dbstruct . options)

  (let* ((dbdat       (db:open-db dbstruct #f))

	 (data-synced 0) ;; count of changed records
    (tmp-area       (common:get-db-tmp-area))
    (dbfiles        (glob (conc tmp-area"/.db/*.db")))
    (sync-durations (make-hash-table)))


    (for-each
     (lambda (file)
       (debug:print-info 0 *default-log-port* "file: " file)
       (let* ((fname (conc (pathname-file file) ".db"))
	      (fulln (conc *toppath*"/.db/"fname))
	      (time1 (if (file-exists? file)
			 (file-modification-time file)
1265
1266
1267
1268
1269
1270
1271

1272
1273
1274
1275
1276
1277
1278
	       (db:patch-schema-rundb  (dbr:dbdat-dbh mtdb))
	       (db:patch-schema-rundb  (dbr:dbdat-dbh main-tmpdb))
               )
              )
	    (stack-push! (dbr:subdb-dbstack subdb) main-tmpdb))
	  options)))
     (hash-table-values (dbr:dbstruct-subdbs dbstruct)))

    data-synced)
)

;; Sync all changed db's
;;
(define (db:tmp->megatest.db-sync dbstruct run-id last-update)
  (let* ((subdbs (hash-table-values (dbr:dbstruct-subdbs dbstruct)))







>







1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
	       (db:patch-schema-rundb  (dbr:dbdat-dbh mtdb))
	       (db:patch-schema-rundb  (dbr:dbdat-dbh main-tmpdb))
               )
              )
	    (stack-push! (dbr:subdb-dbstack subdb) main-tmpdb))
	  options)))
     (hash-table-values (dbr:dbstruct-subdbs dbstruct)))
    (if dbdat (dbfile:add-dbdat dbstruct #f dbdat))
    data-synced)
)

;; Sync all changed db's
;;
(define (db:tmp->megatest.db-sync dbstruct run-id last-update)
  (let* ((subdbs (hash-table-values (dbr:dbstruct-subdbs dbstruct)))