Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -429,22 +429,21 @@
     (if (and dbexists (not write-access))
 	(set! *db-write-access* #f))
     ;; (cons db dbpath)))
     (make-dbr:dbdat dbfile: dbpath dbh: db read-only: (not write-access))))
 
-;; sync run to disk if touched
+;; sync run from tmp disk to nfs disk if touched
 ;;
 (define (db:sync-touched dbstruct run-id #!key (force-sync #f))
-  (let* ((subdb   (dbfile:get-subdb dbstruct run-id))
-	 (tmpsubdb   (dbfile:get-subdb dbstruct run-id))
-         (tmpdbfile (dbr:subdb-tmpdbfile tmpsubdb))
+  (debug:print-info 0 *default-log-port* "db:sync-touched Syncing: " (conc (if run-id run-id "main") ".db"))
+
+  (let* (
+	 (subdb   (or (dbfile:get-subdb dbstruct run-id) (dbfile:init-subdb dbstruct run-id db:initialize-main-db)))
+         (tmpdbfile (dbr:subdb-tmpdbfile subdb))
 	 (mtdb    (dbr:subdb-mtdbdat subdb))
          (tmpdb (dbfile:open-sqlite3-db tmpdbfile #f))
-
-	 ;; (refndb  (dbr:subdb-refndb subdb))
 	 (start-t (current-seconds)))
-    (debug:print-info 4 *default-log-port* "Syncing for run-id: " run-id)
     (mutex-lock! *db-multi-sync-mutex*)
     (let ((update_info (cons "last_update" (if force-sync 0 *db-last-sync*) )))
       (mutex-unlock! *db-multi-sync-mutex*)
       (db:sync-tables (db:sync-all-tables-list dbstruct) update_info tmpdb mtdb))
     (mutex-lock! *db-multi-sync-mutex*)
@@ -660,11 +659,11 @@
    exn
    (begin
      (debug:print 0 *default-log-port* "EXCEPTION: database probably overloaded or unreadable in db:sync-tables.")
      (print-call-chain (current-error-port))
      (debug:print 0 *default-log-port* " message: " ((condition-property-accessor 'exn 'message) exn))
-     (debug:print 5 *default-log-port* "exn=" (condition->list exn))
+     (debug:print 0 *default-log-port* "exn=" (condition->list exn))
      (debug:print 0 *default-log-port* " status:  " ((condition-property-accessor 'sqlite3 'status) exn))
      (debug:print 0 *default-log-port* " src db:  " (dbr:dbdat-dbfile fromdb))
      (for-each (lambda (dbdat)
 		 (let ((dbpath (dbr:dbdat-dbfile dbdat)))
 		   (debug:print 0 *default-log-port* " dbpath:  " dbpath)
@@ -676,13 +675,13 @@
      
      0)
 
    ;; this is the work to be done")
    (cond
-    ((not fromdb) (debug:print 3 *default-log-port* "WARNING: db:sync-tables called with fromdb missing")
+    ((not fromdb) (debug:print 0 *default-log-port* "WARNING: db:sync-tables called with fromdb missing")
      -1)
-    ((not todb)   (debug:print 3 *default-log-port* "WARNING: db:sync-tables called with todb missing")
+    ((not todb)   (debug:print 0 *default-log-port* "WARNING: db:sync-tables called with todb missing")
      -2)
     ((not (sqlite3:database? (dbr:dbdat-dbh fromdb)))
      (debug:print-error 0 *default-log-port* "db:sync-tables called with fromdb not a database " fromdb)
    -3)
     ((not (sqlite3:database? (dbr:dbdat-dbh todb)))
@@ -765,12 +764,10 @@
 	     (lambda (field)
 	       (hash-table-set! field->num field count)
 	       (set! count (+ count 1)))
 	     fields)
 
-            (debug:print 3 *default-log-port* "fromdat: " fromdat)
-
 	    ;; read the source table
             ;; store a list of all rows in the table in fromdat, up to batch-len.
             ;; Then add fromdat to the fromdats list, clear fromdat and repeat.
 	    (sqlite3:for-each-row
 	     (lambda (a . b)
@@ -792,11 +789,11 @@
 	    ;; tack on remaining records in fromdat
 	    (if (not (null? fromdat))
 		(set! fromdats (cons fromdat fromdats)))
 
 	    (if (common:low-noise-print 120 "sync-records")
-		(debug:print 4 *default-log-port* "found " totrecords " records to sync"))
+		(debug:print 0 *default-log-port* "found " totrecords " records to sync"))
 
 	    (sqlite3:for-each-row
 	     (lambda (a . b)
 	       (hash-table-set! todat a (apply vector a b)))
 	     (dbr:dbdat-dbh todb)
@@ -852,11 +849,13 @@
                             ))
 			fromdat-lst))))
 		  fromdats)
 
 
-	         (debug:print 3 *default-log-port* "changed rows: " changed-rows)
+                 (if (> changed-rows 0)
+	           (debug:print 0 *default-log-port* "table " tablename " changed rows: " changed-rows)
+                 )
 
 
 		 (sqlite3:finalize! stmth)
                  (if (member "last_update" field-names)
                     (db:create-trigger db tablename))))
@@ -866,11 +865,11 @@
         )
 	tbls)
        (let* ((runtime      (- (current-milliseconds) start-time))
 	      (should-print (or (debug:debug-mode 12)
 				(common:low-noise-print 120 "db sync" (> runtime 500))))) ;; low and high sync times treated as separate.
-	 (if should-print (debug:print 3 *default-log-port* "INFO: db sync, total run time " runtime " ms"))
+	 (if should-print (debug:print 0 *default-log-port* "INFO: db sync, total run time " runtime " ms"))
 	 (for-each 
 	  (lambda (dat)
 	    (let ((tblname (car dat))
 		  (count   (cdr dat)))
 	      (set! tot-count (+ tot-count count))
@@ -1055,25 +1054,86 @@
 
 
 
 ;; Get a lock from the no-sync-db for the from-db, then delta sync the from-db to the to-db, otherwise return #f
 
-(define (db:lock-and-delta-sync no-sync-db from-db-file to-db-file)
+(define (db:lock-and-delta-sync no-sync-db dbstruct from-db-file runid)
   (assert (not *db-sync-in-progress*) "FATAL: db:lock-and-sync called while a sync is in progress.")
-  (let* ((lockdat  (db:no-sync-get-lock no-sync-db from-db))
+  (debug:print-info 3 *default-log-port* "db:lock-and-delta-sync")
+  (let* ((lockdat  (db:no-sync-get-lock no-sync-db from-db-file))
 	 (gotlock  (car lockdat))
 	 (locktime (cdr lockdat)))
+
+    (debug:print-info 3 *default-log-port* "db:lock-and-delta-sync: go lock?")
     (if gotlock
 	(begin
-          (debug:print 0 *default-log-port* "db:lock-and-sync copying db")
-	  ;; (file-copy from-db to-db #t)
-	  (db:no-sync-del! no-sync-db from-db)
+          (debug:print 0 *default-log-port* "db:lock-and-delta-sync copying db")
+          (db:sync-touched dbstruct runid)
+	  (db:no-sync-del! no-sync-db from-db-file)
 	  #t)
         (begin
-          (debug:print 0 *default-log-port* "could not get lock for " from-db " from no-sync-db")
+          (debug:print 0 *default-log-port* "could not get lock for " from-db-file " from no-sync-db")
 	  #f
         ))))
+
+
+
+
+(define (db:all-db-sync dbstruct)
+  (db:open-db dbstruct #f)
+  (let* ((data-synced 0) ;; count of changed records
+    (tmp-area       (common:get-db-tmp-area))
+    (dbfiles        (glob (conc tmp-area"/.db/*.db")))
+    (sync-durations (make-hash-table))
+    (no-sync-db        (db:open-no-sync-db))
+    )
+    (for-each
+     (lambda (file)
+       (debug:print-info 3 *default-log-port* "file: " file)
+       (let* ((fname (conc (pathname-file file) ".db"))
+	      (fulln (conc *toppath*"/.db/"fname))
+	      (time1 (if (file-exists? file)
+			 (file-modification-time file)
+			 (begin
+			   (debug:print-info 0 *default-log-port* "Sync - I do not see file "file)
+			   1)))
+	      (time2 (if (file-exists? fulln)
+			 (file-modification-time fulln)
+			 (begin
+			   (debug:print-info 0 *default-log-port* "Sync - I do not see file "fulln)
+			   0)))
+	      (changed (> time1 time2))
+	      (do-cp (cond
+		      ((not (file-exists? fulln)) ;; shouldn't happen, but this might recover
+		       (debug:print-info 0 *default-log-port* "File "fulln" not found! Copying "fname" to "fulln)
+		       #t)
+		      (changed ;; (and changed
+		       ;; (> (- (current-seconds) time1) 3)) ;; if file is changed and three seconds have passed.
+		       #t)
+		      ((and changed *time-to-exit*) ;; last sync
+		       #t)
+		      (else
+		       #f))))
+	 (if do-cp
+	     (let* ((start-time (current-milliseconds))
+              (fname (pathname-file file))
+              (runid (if (string= fname "main") #f (string->number fname)))
+              )
+	       (debug:print-info 3 *default-log-port* "db:all-db-sync: fname: " fname", delta: " (- time1 time2) " seconds")
+
+	       (db:lock-and-delta-sync no-sync-db dbstruct fname runid)
+	       (hash-table-set! sync-durations (conc fname".db") (- (current-milliseconds) start-time)))
+	     (debug:print-info 3 *default-log-port* "skipping sync. " file " is up to date")
+         )
+       )
+     )
+     dbfiles
+    )
+  )
+   #t
+)
+
 
 
 
 
 
@@ -1087,10 +1147,11 @@
 ;;  'closeall     - close all opened dbs
 ;;  'schema       - attempt to apply schema changes
 ;;  run-ids: '(1 2 3 ...) or #f (for all)
 ;;
 (define (db:multi-db-sync dbstruct . options)
+
   (db:open-db dbstruct #f)
 
   (let* ((data-synced 0) ;; count of changed records
     (tmp-area       (common:get-db-tmp-area))
     (dbfiles        (glob (conc tmp-area"/.db/*.db")))
@@ -5104,29 +5165,35 @@
                 )))
 	    (debug:print-info 13 *default-log-port* "watchdog done."))
 	  (debug:print-info 13 *default-log-port* "no need for watchdog on non-homehost"))))
 
 
-(define (db:do-sync)
+(define (db:do-sync no-sync-db)
   (let* ((syncer (or (configf:lookup *configdat* "server" "sync-method") "delta-sync"))
     (dbstruct (db:setup #t)))
 
     (debug:print 0 *default-log-port* "db:do-sync: sync-method: " syncer)
     (cond
       ((equal? syncer "brute-force-sync")
-       (db:run-lock-and-sync *no-sync-db*))
+       (db:run-lock-and-sync no-sync-db))
       ((equal? syncer "delta-sync")
        (debug:print 0 *default-log-port* "db:do-sync: db:multi-db-sync" )
        (let* (
 	   (tmpdbpth (dbr:dbstruct-tmppath dbstruct))
 	   (lockfile (conc tmpdbpth ".lock"))
 	   (locked   (common:simple-file-lock lockfile)) 
 	   (res      (if locked
-			 (db:multi-db-sync 
-			  dbstruct
-			  'new2old)
-			 #f)))
+                         ;; sync all dbs for this area
+                
+
+
+
+			 (db:all-db-sync dbstruct) 
+                         #f
+                     )
+            )
+           )
            (if res
 	     (begin
 	       (common:simple-file-release-lock lockfile)
 	       (print "db:do-sync: Synced " res " records to megatest.db")
              )

Index: dbfile.scm
==================================================================
--- dbfile.scm
+++ dbfile.scm
@@ -449,11 +449,11 @@
 		  (if (> tries-left 0)
 		      (dbfile:cautious-open-database fname init-proc (- tries-left 1))))))
     (assert (>= tries-left 0) (conc "FATAL: Five attempts in dbfile:cautious-open-database of "fname", giving up."))
     (if (and (file-write-access? fname) (not (dbfile:simple-file-lock lock-file)))
 	(begin
-	  (dbfile:print-err "INFO: lock file "lock-file" exists, trying again in 1 second.")
+	  (dbfile:print-err "INFO: dbfile:cautious-open-database: lock file " lock-file " exists, trying again in 1 second.")
 	  (thread-sleep! 1)
 	  (dbfile:cautious-open-database fname init-proc (- tries-left 1)))
 	(let* ((db-exists (file-exists? fname))
 	       (result (condition-case
 			   (let* ((db (sqlite3:open-database fname)))

Index: http-transport.scm
==================================================================
--- http-transport.scm
+++ http-transport.scm
@@ -393,10 +393,13 @@
   (let* ((api-url      (conc "http://" iface ":" port "/api"))
 	 (api-uri      (uri-reference (conc "http://" iface ":" port "/api")))
 	 (api-req      (make-request method: 'POST uri: api-uri))
 	 (server-dat   (vector iface port api-uri api-url api-req (current-seconds) server-id)))
     server-dat))
+
+
+
 
 ;; run http-transport:keep-running in a parallel thread to monitor that the db is being 
 ;; used and to shutdown after sometime if it is not.
 ;;
 (define (http-transport:keep-running) 
@@ -472,14 +475,15 @@
 	    (set! server-going #t)
 	    (debug:print 0 *default-log-port* "SERVER: running, megatest version: " (common:get-full-version)) ;; NOTE: the server is NOT yet marked as running in the log. We do that in the keep-running routine.
 
 	    ;; (thread-start! *watchdog*)
           ) 
-	  (if *no-sync-db*
+	  (if no-sync-db
               (begin
-                (debug:print 0 *default-log-port* "keep-running calling db:do-sync at " (time->string (seconds->local-time) "%H:%M:%S"))
-                (db:do-sync)
+                (debug:print 0 *default-log-port* "keep-running calling db:all-db-sync at " (time->string (seconds->local-time) "%H:%M:%S"))
+                (db:all-db-sync *dbstruct-dbs*)
+                ;; (db:do-sync no-sync-db)
 	        ;; (db:run-lock-and-sync *no-sync-db*)
               )
           )
       )
       

Index: tests/simplerun/thebeginning.scm
==================================================================
--- tests/simplerun/thebeginning.scm
+++ tests/simplerun/thebeginning.scm
@@ -11,20 +11,22 @@
  ;; dbfile:set-subdb
  ;; db:with-db
  ;; dbfile:get-subdb
  )
 
+(system "touch /tmp/mmgraham/megatest_localdb/simplerun/.nfs.pdx.disks.icf_gwa_001.mmgraham.fossil.megatest1.7.mod.tests.simplerun/.db/10.db")
+
 ;; *************** dbfile.scm tests ****************
 
 
- (debug:print 0 *default-log-port* " tmp area: " (common:get-db-tmp-area))
+;; (debug:print 0 *default-log-port* " tmp area: " (common:get-db-tmp-area))
 
 (define tmpdir (common:get-db-tmp-area))
 (test #f #t (dbr:dbstruct? (dbfile:setup #t *toppath* tmpdir)))
 (test #f #t (dbr:dbstruct? (db:setup #t)))
 (define dbstruct *dbstruct-dbs*)
-(test #f #t (dbr:subdb? (dbfile:init-subdb dbstruct #f db:initialize-main-db))) ;; this opens the nfs main db
+;; (test #f #t (dbr:subdb? (dbfile:init-subdb dbstruct #f db:initialize-main-db))) ;; this opens the nfs main db
 
 ;; (test #f #t (dbr:dbdat? (dbfile:open-db *dbstruct-dbs* #f db:initialize-main-db))) ;; this opens the tmp db.
 ;; (define maindbdat (dbfile:open-db *dbstruct-dbs* #f db:initialize-main-db)) ;; this opens the tmp db.
 ;; (dbfile:add-dbdat dbstruct #f maindbdat)
 
@@ -35,22 +37,23 @@
 ;; (test #f #t (hash-table? (dbr:dbstruct-subdbs dbstruct)))
 ;; (test #f #t (stack? (dbr:subdb-dbstack (dbfile:get-subdb dbstruct #f))))
 ;; (test #f '("SYSTEM" "RELEASE") (db:get-keys *dbstruct-dbs*))
 
 
- (test #f #t (dbr:dbdat? (dbfile:open-db dbstruct 1 db:initialize-main-db)))
- (test #f #t (dbr:dbdat? (dbfile:open-db dbstruct 2 db:initialize-main-db)))
- (define rundbdat (dbfile:open-db dbstruct 1 db:initialize-main-db))
- (define rundbdat2 (dbfile:open-db dbstruct 1 db:initialize-main-db))
- (dbfile:add-dbdat dbstruct 1 rundbdat)
- (dbfile:add-dbdat dbstruct 2 rundbdat2)
+;; (test #f #t (dbr:dbdat? (dbfile:open-db dbstruct 1 db:initialize-main-db)))
+;; (test #f #t (dbr:dbdat? (dbfile:open-db dbstruct 2 db:initialize-main-db)))
+;; (define rundbdat (dbfile:open-db dbstruct 1 db:initialize-main-db))
+;; (define rundbdat2 (dbfile:open-db dbstruct 2 db:initialize-main-db))
+;; (define rundbdat3 (dbfile:open-db dbstruct 3 db:initialize-main-db))
+;; (dbfile:add-dbdat dbstruct 1 rundbdat)
+;; (dbfile:add-dbdat dbstruct 2 rundbdat2)
+;; (dbfile:add-dbdat dbstruct 3 rundbdat3)
 ;; (test #f #t (dbr:subdb? (dbfile:get-subdb dbstruct 1)))
 ;; (test #f #t (dbr:dbdat? (dbfile:get-dbdat dbstruct 1)))
 ;; (test #f #t (dbr:subdb? (dbfile:get-subdb dbstruct 2)))
 ;; (test #f #t (dbr:dbdat? (dbfile:get-dbdat dbstruct 2)))
 
-;; (test #f #t (db:close-all dbstruct))
 
 
 ;; (test #f #t (> (dbfile:lazy-sqlite-db-modification-time ".db/main.db") 0))
 ;; (test #f #t (> (dbfile:lazy-sqlite-db-modification-time ".db/1.db") 0))
 ;; (test #f #t (> (dbfile:lazy-sqlite-db-modification-time ".db/2.db") 0))
@@ -72,38 +75,52 @@
 
 ;; (dbfile:add-dbdat dbstruct #f maindbdat)
 ;; (define maindbdat (dbfile:get-dbdat dbstruct #f))
 ;; (dbfile:add-dbdat dbstruct #f maindbdat)
 
-(define mtdbdat2 (dbr:subdb-mtdbdat (dbfile:get-subdb dbstruct #f)))
-
-(define areapath  (dbr:dbstruct-areapath dbstruct))
-(define mtdbpath  (dbfile:run-id->path areapath #f))
-(define init-proc db:initialize-main-db)
-
-(define mtdbdat   (dbfile:open-sqlite3-db mtdbpath init-proc))
-
-(define maindb-handle (dbr:dbdat-dbh mtdbdat))
-(define maindb-handle2 (dbr:dbdat-dbh mtdbdat2))
-(debug:print 0 *default-log-port* "maindb handle: " maindb-handle)
-(debug:print 0 *default-log-port* "maindb handle2: " maindb-handle2)
-
-(sqlite3:execute maindb-handle "vacuum")
-(sqlite3:execute maindb-handle2 "vacuum")
-
-(define full-sel   (conc "SELECT * from runs"))
-
-(sqlite3:for-each-row
-	     (lambda (a . b)
-               (debug:print 0 *default-log-port* "a: " a " b: " b)
-             )
-	     maindb-handle
-	     full-sel)
-
-(test #f #t (db:sync-touched dbstruct #f))
-(test #f #t (db:sync-touched dbstruct 1))
-(test #f #t (db:sync-touched dbstruct 2))
-
+;; (define mtdbdat2 (dbr:subdb-mtdbdat (dbfile:get-subdb dbstruct #f)))
+
+;; (define areapath  (dbr:dbstruct-areapath dbstruct))
+;; (define mtdbpath  (dbfile:run-id->path areapath #f))
+;; (define init-proc db:initialize-main-db)
+
+;; (define mtdbdat   (dbfile:open-sqlite3-db mtdbpath init-proc))
+
+;; (define maindb-handle (dbr:dbdat-dbh mtdbdat))
+;; (define maindb-handle2 (dbr:dbdat-dbh mtdbdat2))
+
+;; (sqlite3:execute maindb-handle "vacuum")
+;; (sqlite3:execute maindb-handle2 "vacuum")
+
+;; (define full-sel   (conc "SELECT * from runs"))
+
+;; (sqlite3:for-each-row
+;; 	     (lambda (a . b)
+;;                (debug:print 0 *default-log-port* "a: " a " b: " b)
+;;              )
+;; 	     maindb-handle
+;; 	     full-sel)
+
+;; (test #f #t (db:sync-touched dbstruct #f))
+;; (test #f #t (db:sync-touched dbstruct 1))
+;; (test #f #t (db:sync-touched dbstruct 2))
+
+;; (test #f #t (dbr:subdb? (dbfile:get-subdb dbstruct #f)))
+;; (test #f #t (dbr:subdb? (dbfile:get-subdb dbstruct (string->number "1"))))
+;; (test #f #t (dbr:subdb? (dbfile:get-subdb dbstruct 2)))
+
+
+;; (test #f #t (db:sync-touched dbstruct #f))
+;; (test #f #t (db:sync-touched dbstruct 1))
+;; (test #f #t (db:sync-touched dbstruct 2))
+
+
+
+(test #f #t (db:all-db-sync dbstruct))
+
+(exit)
+
+;; (test #f #t (db:close-all dbstruct))
 (test #f #t (db:safely-close-sqlite3-db (dbr:dbdat-dbh rundbdat) (dbr:dbdat-stmt-cache rundbdat)))
 (test #f #t (db:safely-close-sqlite3-db (dbr:dbdat-dbh rundbdat2) (dbr:dbdat-stmt-cache rundbdat2)))
 (test #f #t (db:safely-close-sqlite3-db (dbr:dbdat-dbh mtdbdat) (dbr:dbdat-stmt-cache mtdbdat)))