Megatest

Check-in [a29849711f]
Login
Overview
Comment:beginnings of new lock-in for main.db
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.80-revolution
Files: files | file ages | folders
SHA1: a29849711f31a77ec419d756b6bb3f3da72ccc14
User & Date: mrwellan on 2023-11-22 15:45:03
Other Links: branch diff | manifest | tags
Context
2023-11-24
00:14
Rebuild manual. check-in: f9ab3f8da0 user: matt tags: v1.80-revolution
2023-11-22
15:45
beginnings of new lock-in for main.db check-in: a29849711f user: mrwellan tags: v1.80-revolution
04:21
More cleanup of cleanup, removed all use of cleanup-proc check-in: f69329ec9a user: matt tags: v1.80-revolution
Changes

Modified dbfile.scm from [a50374cb4f] to [e1df0bf86e].

526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
                                 reason TEXT DEFAULT 'none',
                                   CONSTRAINT no_sync_processes UNIQUE (host,pid));"
			    ))))))
	 (on-tmp      (equal? (car (string-split dbpath "/")) "tmp"))
	 (db        (if on-tmp
			(dbfile:cautious-open-database dbname init-proc 1 "WAL" force-init: #t) ;; WAL MODE should use syncronous=1
			;; (dbfile:cautious-open-database dbname init-proc 0 #f    force-init: #t)
			(dbfile:cautious-open-database dbname init-proc 0 "MEMORY" force-init: #t)
			;; (sqlite3:open-database dbname)
			)))
    (if on-tmp	      ;; done in cautious-open-database
	(begin
	  ;; (sqlite3:execute db "PRAGMA synchronous = 0;") ;; why was this here when is is handled by cautious-open-database?
	  (sqlite3:set-busy-handler! db (sqlite3:make-busy-timeout 136000))))
    db))

;; mtest processes registry calls

(define (dbfile:insert-or-update-process nsdb dat)
  (let* ((host      (procinf-host dat))
	 (pid       (procinf-pid  dat))







|


|
|
|
|







526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
                                 reason TEXT DEFAULT 'none',
                                   CONSTRAINT no_sync_processes UNIQUE (host,pid));"
			    ))))))
	 (on-tmp      (equal? (car (string-split dbpath "/")) "tmp"))
	 (db        (if on-tmp
			(dbfile:cautious-open-database dbname init-proc 1 "WAL" force-init: #t) ;; WAL MODE should use syncronous=1
			;; (dbfile:cautious-open-database dbname init-proc 0 #f    force-init: #t)
			(dbfile:cautious-open-database dbname init-proc 0 "MEMORY" force-init: #t) ;; Journal mode = memory is fastest?
			;; (sqlite3:open-database dbname)
			)))
    ;; (if on-tmp	      ;; done in cautious-open-database
    ;;    (begin
    ;; (sqlite3:execute db "PRAGMA synchronous = 0;") ;; why was this here when is is handled by cautious-open-database?
    (sqlite3:set-busy-handler! db (sqlite3:make-busy-timeout 136000)) ;; ))
    db))

;; mtest processes registry calls

(define (dbfile:insert-or-update-process nsdb dat)
  (let* ((host      (procinf-host dat))
	 (pid       (procinf-pid  dat))
580
581
582
583
584
585
586


587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609








610
611
612
613
614
615
616
617
618
619
620
621
622
(define (dbfile:register-process nsdb host port pid starttime endtime status purpose dbname mtversion)
  (sqlite3:execute nsdb "INSERT INTO processes (host,port,pid,starttime,endtime,status,purpose,dbname,mtversion) VALUES (?,?,?,?,?,?,?,?,?);"
		   host port pid starttime endtime status purpose dbname mtversion))

(define (dbfile:set-process-status nsdb host pid newstatus)
  (sqlite3:execute nsdb "UPDATE processes SET status=? WHERE host=? AND pid=?;" newstatus host pid))



(define (dbfile:get-process-options nsdb purpose dbname)
  (sqlite3:fold-row
   ;; host port pid starttime status mtversion
   (lambda (res . row)
     (cons row res))
   '()
   nsdb
   "SELECT host,port,pid,starttime,endtime,status,mtversion FROM processes WHERE purpose=? AND dbname LIKE ? AND status='alive';"
   purpose dbname))

(define (dbfile:get-process-info nsdb host pid)
  (let ((res (sqlite3:fold-row
	      ;; host port pid starttime status mtversion
	      (lambda (res . row)
		(cons row res))
	      '()
	      nsdb
	      "SELECT host,port,pid,starttime,endtime,status,purpose,dbname,mtversion FROM processes WHERE host=? AND pid=?;"
	      host pid)))
    (if (null? res)
	#f
	(car res))))









(define (dbfile:set-process-done nsdb host pid reason)
  (sqlite3:execute nsdb "UPDATE processes SET status='ended',endtime=?,reason=? WHERE host=? AND pid=?;" (current-seconds) reason host pid)
  (dbfile:cleanup-old-entries nsdb))

(define (dbfile:cleanup-old-entries nsdb)
  (sqlite3:execute nsdb "DELETE FROM process WHERE status='ended' AND endtime<?;" (- (current-seconds) (* 3600 48))))

;; other no-sync functions

(define (dbfile:with-no-sync-db dbpath proc)
  (mutex-lock! *no-sync-db-mutex*)
  (let* ((already-open *no-sync-db*)
	 (db  (or already-open (dbfile:raw-open-no-sync-db dbpath)))







>
>







|















>
>
>
>
>
>
>
>

|



|







580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
(define (dbfile:register-process nsdb host port pid starttime endtime status purpose dbname mtversion)
  (sqlite3:execute nsdb "INSERT INTO processes (host,port,pid,starttime,endtime,status,purpose,dbname,mtversion) VALUES (?,?,?,?,?,?,?,?,?);"
		   host port pid starttime endtime status purpose dbname mtversion))

(define (dbfile:set-process-status nsdb host pid newstatus)
  (sqlite3:execute nsdb "UPDATE processes SET status=? WHERE host=? AND pid=?;" newstatus host pid))

;; as sorted should be stable. can use to choose "winner"
;;
(define (dbfile:get-process-options nsdb purpose dbname)
  (sqlite3:fold-row
   ;; host port pid starttime status mtversion
   (lambda (res . row)
     (cons row res))
   '()
   nsdb
   "SELECT host,port,pid,starttime,endtime,status,mtversion FROM processes WHERE purpose=? AND dbname LIKE ? AND status IN ('running','alive') ORDER BY starttime ASC,host,port;"
   purpose dbname))

(define (dbfile:get-process-info nsdb host pid)
  (let ((res (sqlite3:fold-row
	      ;; host port pid starttime status mtversion
	      (lambda (res . row)
		(cons row res))
	      '()
	      nsdb
	      "SELECT host,port,pid,starttime,endtime,status,purpose,dbname,mtversion FROM processes WHERE host=? AND pid=?;"
	      host pid)))
    (if (null? res)
	#f
	(car res))))

(define (dbfile:row->procinf row)
  (match row
   ((host port pid starttime endtime status mtversion)
    (make-procinf host: host port: port pid: pid starttime: starttime endtime: endtime status: status mtversion: mtversion))
   (else
    (debug:print 0 *default-log-port* "ERROR: row "row" did not match host,port,pid,starttime,endtime,status,mtversion")
    #f)))

(define (dbfile:set-process-done nsdb host pid reason)
  (sqlite3:execute nsdb "UPDATE processes SET status='done',endtime=?,reason=? WHERE host=? AND pid=?;" (current-seconds) reason host pid)
  (dbfile:cleanup-old-entries nsdb))

(define (dbfile:cleanup-old-entries nsdb)
  (sqlite3:execute nsdb "DELETE FROM process WHERE status='done' AND endtime<?;" (- (current-seconds) (* 3600 48))))

;; other no-sync functions

(define (dbfile:with-no-sync-db dbpath proc)
  (mutex-lock! *no-sync-db-mutex*)
  (let* ((already-open *no-sync-db*)
	 (db  (or already-open (dbfile:raw-open-no-sync-db dbpath)))

Modified tcp-transportmod.scm from [2239dfb672] to [22faa8bbb3].

562
563
564
565
566
567
568
























569
570
571
572
573
574
575
576
			(debug:print 0 *default-log-port* "WARNING: received a stop server from client by remote request.")
			#f)
		       ((null? servers) #f) ;; not ok
		       ((equal? (list-ref (car servers) 6) ;; compare the servinfofile
				(tt-servinf-file ttdat))
			(let* ((res (if db-locked-in
					#t
























					(let* ((lock-result  ;; this is the primary lock - need to double verify that got it
						(dbfile:with-no-sync-db
						 nosyncdbpath
						 (lambda (db)
						   (db:no-sync-lock-and-check db dbfname
									      (tt-servinf-file ttdat)
									      ;; (dbr:dbstruct-dbtmpname dbstruct)
									      ))))







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|







562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
			(debug:print 0 *default-log-port* "WARNING: received a stop server from client by remote request.")
			#f)
		       ((null? servers) #f) ;; not ok
		       ((equal? (list-ref (car servers) 6) ;; compare the servinfofile
				(tt-servinf-file ttdat))
			(let* ((res (if db-locked-in
					#t
					;;
					;; let's replace the below "winning" lock method with:
					;;  1. create a lock file with pid etc.
					;;  2. if there are no other lock files make an entry in the no-sync db
					;;  3. gather the lock entries, apply the "winner" heuristic
					;;  4. if I'm the winner, set tt-state to 'running else set to 'notthewinner
					;;
					;; New idea:
					;;  1. check all processes entries that match the db
					;;  2. sort by fixed heuristic
					;;  3. if I'm number one, set state to 'running and db-locked-in to #t
					(let* ((candidates (dbfile:with-no-sync-db
							     nosyncdbpath
							     (lambda (nsdb)
							       (dbfile:get-process-options nsdb "server" dbfname))))
					       (primecand  (begin
							     (assert (not (null? candidates))
								     "HOW CAN WE NOT BE IN THE PROCESSES DB AS A SERVER?")
							     (dbfile:row->procinf (car candidates)))))
					  ;; compare primecand with myself
					  ;; if not me check that it is reachable
					  ;;   if reachable exit
					  #f)
					
					#;(let* ((lock-result  ;; this is the primary lock - need to double verify that got it
						(dbfile:with-no-sync-db
						 nosyncdbpath
						 (lambda (db)
						   (db:no-sync-lock-and-check db dbfname
									      (tt-servinf-file ttdat)
									      ;; (dbr:dbstruct-dbtmpname dbstruct)
									      ))))