Megatest

Diff
Login

Differences From Artifact [6f20a1d6d0]:

To Artifact [8d243ee646]:


1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958








1959
1960
1961
1962
1963
1964
1965
1966
			       set-verbosity
			       killserver
			       ))

;; not used, intended to indicate to run in calling process
(define db:run-local-queries '()) ;; rollup-tests-pass-fail))

(define (db:process-cached-writes db)
  (let ((queries    (make-hash-table))
	(data       #f))
    (mutex-lock! *incoming-mutex*)
    ;; data is a list of query packets <vector qry-sig query params
    (set! data (reverse *incoming-writes*)) ;;  (sort ... (lambda (a b)(< (vector-ref a 1)(vector-ref b 1)))))
    (set! *server:last-write-flush* (current-milliseconds))
    (set! *incoming-writes* '())
    (mutex-unlock! *incoming-mutex*)
    (if (> (length data) 0)
	;; Process if we have data
	(begin
	  (debug:print-info 7 "Writing cached data " data)
	  
	  ;; Prepare the needed sql statements
	  ;;
	  (for-each (lambda (request-item)
		      (let ((stmt-key (vector-ref request-item 0))
			    (query    (vector-ref request-item 1)))
			(hash-table-set! queries stmt-key (sqlite3:prepare db query))))
		    data)
	  
	  ;; No outer loop needed. Single loop for write items only. Reads trigger flush of queue
	  ;; and then are executed.
	  (sqlite3:with-transaction 
	   db
	   (lambda ()
	     (for-each
	      (lambda (hed)
		(let* ((params   (vector-ref hed 2))
		       (stmt-key (vector-ref hed 0))
		       (stmt     (hash-table-ref/default queries stmt-key #f)))
		  (if stmt
		      (apply sqlite3:execute stmt params)
		      (debug:print 0 "ERROR: Problem Executing " stmt-key " for " params))))
	      data)))
	  
	  ;; let all the waiting calls know all is done
	  (mutex-lock! *completed-mutex*)
	  (for-each (lambda (item)
		      (let ((qry-sig (cdb:packet-get-client-sig item)))
			(debug:print-info 7 "Registering query " qry-sig " as done")
			(hash-table-set! *completed-writes* qry-sig #t)))
		    data)
	  (mutex-unlock! *completed-mutex*)
	  
	  ;; Finalize the statements. Should this be done inside the mutex above?
	  ;; I think sqlite3 mutexes will keep the data safe
	  (for-each (lambda (stmt-key)
		      (sqlite3:finalize! (hash-table-ref queries stmt-key)))
		    (hash-table-keys queries))
	  
	  ;; Do a little record keeping
	  (let ((cache-size (length data)))
	    (if (> cache-size *max-cache-size*)
		(set! *max-cache-size* cache-size)))
	  #t)








	#f)))

(define *db:process-queue-mutex* (make-mutex))

(define *number-of-writes*         0)
(define *writes-total-delay*       0)
(define *total-non-write-delay*    0)
(define *number-non-write-queries* 0)







|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>
>
>
>
>
>
>
>
|







1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
			       set-verbosity
			       killserver
			       ))

;; not used, intended to indicate to run in calling process
(define db:run-local-queries '()) ;; rollup-tests-pass-fail))

;; (define (db:process-cached-writes db)
;;   (let ((queries    (make-hash-table))
;; 	(data       #f))
;;     (mutex-lock! *incoming-mutex*)
;;     ;; data is a list of query packets <vector qry-sig query params
;;     (set! data (reverse *incoming-writes*)) ;;  (sort ... (lambda (a b)(< (vector-ref a 1)(vector-ref b 1)))))
;;     (set! *server:last-write-flush* (current-milliseconds))
;;     (set! *incoming-writes* '())
;;     (mutex-unlock! *incoming-mutex*)
;;     (if (> (length data) 0)
;; 	;; Process if we have data
;; 	(begin
;; 	  (debug:print-info 7 "Writing cached data " data)
;; 	  
;; 	  ;; Prepare the needed sql statements
;; 	  ;;
;; 	  (for-each (lambda (request-item)
;; 		      (let ((stmt-key (vector-ref request-item 0))
;; 			    (query    (vector-ref request-item 1)))
;; 			(hash-table-set! queries stmt-key (sqlite3:prepare db query))))
;; 		    data)
;; 	  
;; 	  ;; No outer loop needed. Single loop for write items only. Reads trigger flush of queue
;; 	  ;; and then are executed.
;; 	  (sqlite3:with-transaction 
;; 	   db
;; 	   (lambda ()
;; 	     (for-each
;; 	      (lambda (hed)
;; 		(let* ((params   (vector-ref hed 2))
;; 		       (stmt-key (vector-ref hed 0))
;; 		       (stmt     (hash-table-ref/default queries stmt-key #f)))
;; 		  (if stmt
;; 		      (apply sqlite3:execute stmt params)
;; 		      (debug:print 0 "ERROR: Problem Executing " stmt-key " for " params))))
;; 	      data)))
;; 	  
;; 	  ;; let all the waiting calls know all is done
;; 	  (mutex-lock! *completed-mutex*)
;; 	  (for-each (lambda (item)
;; 		      (let ((qry-sig (cdb:packet-get-client-sig item)))
;; 			(debug:print-info 7 "Registering query " qry-sig " as done")
;; 			(hash-table-set! *completed-writes* qry-sig #t)))
;; 		    data)
;; 	  (mutex-unlock! *completed-mutex*)
;; 	  
;; 	  ;; Finalize the statements. Should this be done inside the mutex above?
;; 	  ;; I think sqlite3 mutexes will keep the data safe
;; 	  (for-each (lambda (stmt-key)
;; 		      (sqlite3:finalize! (hash-table-ref queries stmt-key)))
;; 		    (hash-table-keys queries))
;; 	  
;; 	  ;; Do a little record keeping
;; 	  (let ((cache-size (length data)))
;; 	    (if (> cache-size *max-cache-size*)
;; 		(set! *max-cache-size* cache-size)))
;; 	  #t)
;; 	#f)))

(define (db:process-write db request-item)
  (let ((stmt-key (vector-ref request-item 0))
	(query    (vector-ref request-item 1))
	(params   (vector-ref request-item 2))
	(queryh   (sqlite3:prepare db query)))
    (apply sqlite3:execute stmt params)
    #f))

(define *db:process-queue-mutex* (make-mutex))

(define *number-of-writes*         0)
(define *writes-total-delay*       0)
(define *total-non-write-delay*    0)
(define *number-non-write-queries* 0)
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045

2046
2047
2048
2049
2050
2051
2052
2053
2054
	(begin
	  (cond
	   ((member stmt-key db:special-queries)
	    (let ((starttime (current-milliseconds)))
	      (debug:print-info 9 "Handling special statement " stmt-key)
	      (case stmt-key
		((immediate)
		 ;; This is a read or mixed read-write query, must clear the cache
		 (case *transport-type*
		   ((http)
		    (mutex-lock! *db:process-queue-mutex*)
		    (db:process-cached-writes db)
		    (mutex-unlock! *db:process-queue-mutex*)))
		 (let* ((proc      (car params))
			(remparams (cdr params))
			;; we are being handed a procedure so call it
			;; (debug:print-info 11 "Running (apply " proc " " remparams ")")
			(result (server:reply return-address qry-sig #t (apply proc remparams))))

		   (set! *total-non-write-delay* (+ *total-non-write-delay* (- (current-milliseconds) starttime))) 
		   (set! *number-non-write-queries* (+ *number-non-write-queries* 1))
		   result))
		((login)
		 (if (< (length params) 3) ;; should get toppath, version and signature
		     (server:reply return-address qry-sig '(#f "login failed due to missing params")) ;; missing params
		     (let ((calling-path (car   params))
			   (calling-vers (cadr  params))
			   (client-key   (caddr params)))







|
<
<
<
<
<



<

>
|
|







2036
2037
2038
2039
2040
2041
2042
2043





2044
2045
2046

2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
	(begin
	  (cond
	   ((member stmt-key db:special-queries)
	    (let ((starttime (current-milliseconds)))
	      (debug:print-info 9 "Handling special statement " stmt-key)
	      (case stmt-key
		((immediate)
		 (debug:print 0 "WARNING: Immediate calls are verboten now!")





		 (let* ((proc      (car params))
			(remparams (cdr params))
			;; we are being handed a procedure so call it

			(result (server:reply return-address qry-sig #t (apply proc remparams))))
		   (debug:print-info 11 "Ran (apply " proc " " remparams ")")
		   ;; (set! *total-non-write-delay* (+ *total-non-write-delay* (- (current-milliseconds) starttime))) 
		   ;; (set! *number-non-write-queries* (+ *number-non-write-queries* 1))
		   result))
		((login)
		 (if (< (length params) 3) ;; should get toppath, version and signature
		     (server:reply return-address qry-sig '(#f "login failed due to missing params")) ;; missing params
		     (let ((calling-path (car   params))
			   (calling-vers (cadr  params))
			   (client-key   (caddr params)))