Megatest

Check-in [dca3a45c98]
Login
Overview
Comment:Added function for using ATTACH for sync between db's. Not yet in use.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.80
Files: files | file ages | folders
SHA1: dca3a45c980af8acc65daa556f3aa96da7107ac7
User & Date: mrwellan on 2023-02-27 21:25:46
Other Links: branch diff | manifest | tags
Context
2023-02-28
20:27
ATTACH Sync not working. check-in: 692ebd2f40 user: mrwellan tags: v1.80
2023-02-27
21:25
Added function for using ATTACH for sync between db's. Not yet in use. check-in: dca3a45c98 user: mrwellan tags: v1.80
10:54
Added suicide mode to db:with-db for development. It has been very hard to isolate threaded crashes under the tcp-server. check-in: 3b54f22608 user: mrwellan tags: v1.80
Changes

Modified dbmod.scm from [2dfee57e39] to [c0d005cd6b].

304
305
306
307
308
309
310





































311
312
313
314
315
316
317
      (for-each 
       (lambda (dat)
	 (let ((tblname (car dat))
	       (count   (cdr dat)))
	   (set! tot-count (+ tot-count count)))) 
       (sort (hash-table->alist numrecs)(lambda (a b)(> (cdr a)(cdr b))))))
    tot-count))






































;;======================================================================
;; Moved from dbfile
;;======================================================================


)







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
      (for-each 
       (lambda (dat)
	 (let ((tblname (car dat))
	       (count   (cdr dat)))
	   (set! tot-count (+ tot-count count)))) 
       (sort (hash-table->alist numrecs)(lambda (a b)(> (cdr a)(cdr b))))))
    tot-count))

;; direction = fromdest, todest
;; mode = 'full, 'incr
;;
(define (dbmod:attach-sync tables dbh destdbfile direction #!key (mode 'full))
  (let* ((dest-exists  (file-exists? destdbfile)))
    (assert dest-exists "FATAL: sync called with non-existant file, "destdbfile)
    ;; attach the destdbfile
    ;; for each table
    ;;    insert into dest.<table> select * from src.<table> where last_update>last_update
    ;; done
    (sqlite3:execute dbh "ATTACH ? AS auxdb;" destdbfile)
    (for-each
     (lambda (table)
       (let* ((dir    (eq? direction 'todest))
	      (fromdb (if dir "" "auxdb."))
	      (todb   (if dir "auxdb." ""))
	      (stmt1 (conc "INSERT OR IGNORE INTO "todb table
			  " SELECT * FROM "fromdb table";"))
	      (stmt2 (conc "INSERT OR REPLACE INTO "todb table
			   " SELECT * FROM "fromdb table" WHERE "
			   fromdb table".last_update > "
			   todb table".last_update;"))
	      (stmt3 (conc "INSERT OR REPLACE INTO "todb"."table
			   " SELECT * FROM "fromdb table";"))
	      (stmt4 (conc "DELETE FROM "todb table" WHERE "fromdb
			   "tests.last_update > "todb table".last_update;")))
	 ;; (print "stmt1: "stmt1)
	 ;; (print "stmt2: "stmt2)
	 ;; (print "stmt3: "stmt4)
	 ;; (print "stmt1: "stmt1)
	 (sqlite3:execute dbh stmt4)
	 (sqlite3:execute dbh stmt1)
	 ;; (sqlite3:execute dbh stmt1)
	 ;; (sqlite3:execute dbh stmt2)
	 (sqlite3:execute dbh "DETACH auxdb;")))
     tables)))

;;======================================================================
;; Moved from dbfile
;;======================================================================


)

Modified tcp-transportmod.scm from [1b8b1d78a6] to [e26313be2b].

533
534
535
536
537
538
539


540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
;; try running on that host
;;   incidental: rotate logs in logs/ dir.
;;
(define  (tt:server-process-run areapath testsuite mtexe run-id #!key (profile-mode "")) ;; areapath is *toppath* for a given testsuite area
  (assert areapath  "FATAL: tt:server-process-run called without areapath defined.")
  (assert testsuite "FATAL: tt:server-process-run called without testsuite defined.")
  (assert mtexe     "FATAL: tt:server-process-run called without mtexe defined.")


  (let* ((load (get-normalized-cpu-load))
	 (nrun (number-of-processes-running "mtest.*server")))
    (cond
     ((> load 2.0)
      (debug:print 0 *default-log-port* "Normalized load "load" is over the limit of 2.0. Not starting a server.")
      (thread-sleep! 1))
     ((> nrun 100)
      (debug:print 0 *default-log-port* nrun" servers running on this host, not starting another.")
      (thread-sleep! 1))
     (else
      (if (not (file-exists? (conc areapath"/logs")))
	      (create-directory (conc areapath"/logs") #t))
	  (let* ((logfile   (conc areapath "/logs/server.log")) ;; -" curr-pid "-" target-host ".log"))
		 (cmdln     (conc
			     mtexe
			     " -server - ";; (or target-host "-")
			     " -m testsuite:" testsuite
			     ;; " -run-id " (or run-id "main") ;; NO, we do NOT want to have run id as part of this
			     " -db "  (dbmod:run-id->dbfname run-id)
			     " " profile-mode
			     ))) ;; (conc " >> " logfile " 2>&1 &")))))
	    ;; we want the remote server to start in *toppath* so push there
	    ;; (push-directory areapath) ;; use cd in the command line instead
	    (debug:print 0 *default-log-port* "INFO: Trying to start server in tcp mode (" cmdln ") at "(common:human-time)" for "areapath)
	    ;; (debug:print 0 *default-log-port* "INFO: starting server at " (common:human-time))
	    (setenv "NBFAKE_QUIET" "yes") ;; BUG: change to with-environment-variable ...







>
>
|
|
















|







533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
;; try running on that host
;;   incidental: rotate logs in logs/ dir.
;;
(define  (tt:server-process-run areapath testsuite mtexe run-id #!key (profile-mode "")) ;; areapath is *toppath* for a given testsuite area
  (assert areapath  "FATAL: tt:server-process-run called without areapath defined.")
  (assert testsuite "FATAL: tt:server-process-run called without testsuite defined.")
  (assert mtexe     "FATAL: tt:server-process-run called without mtexe defined.")
  ;; mtest -server - -m testsuite:ext-tests -db 6.db
  (let* ((dbfname  (dbmod:run-id->dbfname run-id))
	 (load     (get-normalized-cpu-load))
	 (nrun     (number-of-processes-running (conc "mtest.*server.*"testsuite".*"dbfname))))
    (cond
     ((> load 2.0)
      (debug:print 0 *default-log-port* "Normalized load "load" is over the limit of 2.0. Not starting a server.")
      (thread-sleep! 1))
     ((> nrun 100)
      (debug:print 0 *default-log-port* nrun" servers running on this host, not starting another.")
      (thread-sleep! 1))
     (else
      (if (not (file-exists? (conc areapath"/logs")))
	      (create-directory (conc areapath"/logs") #t))
	  (let* ((logfile   (conc areapath "/logs/server.log")) ;; -" curr-pid "-" target-host ".log"))
		 (cmdln     (conc
			     mtexe
			     " -server - ";; (or target-host "-")
			     " -m testsuite:" testsuite
			     ;; " -run-id " (or run-id "main") ;; NO, we do NOT want to have run id as part of this
			     " -db "  dbfname ;; (dbmod:run-id->dbfname run-id)
			     " " profile-mode
			     ))) ;; (conc " >> " logfile " 2>&1 &")))))
	    ;; we want the remote server to start in *toppath* so push there
	    ;; (push-directory areapath) ;; use cd in the command line instead
	    (debug:print 0 *default-log-port* "INFO: Trying to start server in tcp mode (" cmdln ") at "(common:human-time)" for "areapath)
	    ;; (debug:print 0 *default-log-port* "INFO: starting server at " (common:human-time))
	    (setenv "NBFAKE_QUIET" "yes") ;; BUG: change to with-environment-variable ...