Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -213,19 +213,19 @@
 			(dbr:dbdat-dbh dbdat)
 			dbstruct))
 	 (fname     (if dbdat
 			(dbr:dbdat-dbfile dbdat)
 			"nofilenameavailable"))
-	 (subdb     (if have-struct
+	 #;(subdb     (if have-struct
 			(dbfile:get-subdb dbstruct run-id)
 			#f))
 	 (use-mutex (> *api-process-request-count* 25))) ;; was 25
     (if (and use-mutex
 	     (common:low-noise-print 120 "over-50-parallel-api-requests"))
 	(debug:print-info 0 *default-log-port* *api-process-request-count* " parallel api requests being processed in process " (current-process-id) ", throttling access"))
     (if (common:low-noise-print 600 (conc "parallel-api-requests" *max-api-process-requests*))
-	(debug:print-info 2 *default-log-port* "Parallel api request count: " *api-process-request-count* " max parallel requests: " *max-api-process-requests*))
+	(debug:print-info 1 *default-log-port* "Parallel api request count: " *api-process-request-count* " max parallel requests: " *max-api-process-requests*))
     (condition-case
 	(begin
 	  (if use-mutex (mutex-lock! *db-with-db-mutex*))
 	  (let ((res (apply proc dbdat db params)))
 	    (if use-mutex (mutex-unlock! *db-with-db-mutex*))
@@ -434,15 +434,17 @@
 
 ;; sync run from tmp disk to nfs disk if touched
 ;;
 (define (db:sync-touched dbstruct run-id #!key (force-sync #f))
   (debug:print-info 0 *default-log-port* "db:sync-touched Syncing: " (conc (if run-id run-id "main") ".db"))
-  (let* ((subdb   (or (dbfile:get-subdb dbstruct run-id) (dbfile:init-subdb dbstruct run-id db:initialize-main-db)))
+  (let* (;; the subdb is needed to access the mtdbdat
+	 (subdb     (or (dbfile:get-subdb dbstruct run-id)
+			(dbfile:init-subdb dbstruct run-id db:initialize-main-db)))
          (tmpdbfile (dbr:subdb-tmpdbfile subdb))
-	 (mtdb    (dbr:subdb-mtdbdat subdb))
-         (tmpdb (dbfile:open-sqlite3-db tmpdbfile #f))
-	 (start-t (current-seconds)))
+	 (mtdb      (dbr:subdb-mtdbdat subdb))
+         (tmpdb     (dbfile:open-db dbstruct run-id db:initialize-main-db)) ;; sqlite3-db tmpdbfile #f))
+	 (start-t   (current-seconds)))
     (mutex-lock! *db-multi-sync-mutex*)
     (let ((update_info (cons "last_update" (if force-sync 0 *db-last-sync*) )))
       (mutex-unlock! *db-multi-sync-mutex*)
       (db:sync-tables (db:sync-all-tables-list dbstruct) update_info tmpdb mtdb))
     (mutex-lock! *db-multi-sync-mutex*)

Index: dbfile.scm
==================================================================
--- dbfile.scm
+++ dbfile.scm
@@ -238,10 +238,13 @@
 				    mtdbdat:   mtdbdat)))
     (dbfile:set-subdb dbstruct run-id newsubdb)
     newsubdb)) ;; return the new subdb - but shouldn't really use it
 
 ;; returns dbdat with dbh and dbfilepath
+;;
+;; NOTE: the handle is on /tmp db file!
+;;
 ;;  1. if needed setup the subdb for the given run-id
 ;;  2. if there is no existing db handle in the stack
 ;;     create a new handle and return it (do NOT add
 ;;     it to the stack).
 ;;
@@ -265,11 +268,11 @@
 ;; this code stabilizes
 (define *dbopens* (make-hash-table))
 (define (dbfile:inc-db-open dbfile)
   (let* ((curr-opens-count (+ (hash-table-ref/default *dbopens* dbfile 0) 1)))
     (if (> curr-opens-count 1) ;; this should NOT be happening
-	(dbfile:print-err "ERROR: db "dbfile" has been opened "curr-opens-count" times!"))
+	(dbfile:print-err "INFO: db "dbfile" has been opened "curr-opens-count" times!"))
     (hash-table-set! *dbopens* dbfile curr-opens-count)
     curr-opens-count))
 
 ;; Open the classic megatest.db file (defaults to open in toppath)
 ;;

Index: dbmod.scm
==================================================================
--- dbmod.scm
+++ dbmod.scm
@@ -31,7 +31,26 @@
 (define (db:run-id->dbname run-id)
   (cond
    ((number? run-id)(conc run-id ".db"))
    ((not run-id)    "main.db")
    (else            run-id)))
- 
+
+
+;;======================================================================
+;; hash of hashs
+;;======================================================================
+
+
+(define (db:hoh-set! dat key1 key2 val)
+  (let* ((subhash (hash-table-ref/default dat key1 #f)))
+    (if subhash
+	(hash-table-set! subhash key2 val)
+	(begin
+	  (hash-table-set! dat key1 (make-hash-table))
+	  (db:hoh-set! dat key1 key2 val)))))
+
+(define (db:hoh-get dat key1 key2)
+  (let* ((subhash (hash-table-ref/default dat key1 #f)))
+    (and subhash
+	 (hash-table-ref/default subhash key2 #f))))
+
 )

Index: runs.scm
==================================================================
--- runs.scm
+++ runs.scm
@@ -1502,10 +1502,12 @@
 ;;
 (define *max-tries-hash* (make-hash-table))
 
 (define (runs:pretty-long-list lst)
    (if (> (length lst) 8)(append (take lst 3)(list "...")) lst))
+
+(define *last-loop-time-ms* 0)
 
 ;;======================================================================
 ;; runs:run-tests-queue is called by runs:run-tests
 ;;======================================================================
 ;;
@@ -1640,10 +1642,22 @@
 			   testmode:    testmode
 			   newtal:      newtal
 			   itemmaps:    itemmaps
 			   ;; prereqs-not-met: prereqs-not-met
 			   )))
+
+	;; too-tight loop detection and delay, this might hide issues
+	;; that occur in long run times. Consider commenting when debugging
+	;; 
+	(if (and (>= num-running max-concurrent-jobs)
+		 (< (- (current-milliseconds) *last-loop-time-ms*) 500))
+	    (begin
+	      (if (runs:lownoise "too-tight-loop" 5)
+		  (debug:print-info 2 *default-log-port* "Excessively fast loop, delaying 1/2 second"))
+	      (thread-sleep! 0.5)))
+	(set! *last-loop-time-ms* (current-milliseconds))
+     
 	(runs:dat-regfull-set! runsdat regfull)
     
 	(if (> num-running 0)
             (set! last-time-some-running (current-seconds)))