Megatest

Check-in [948b22f1a9]
Login
Overview
Comment:small fixes
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.64-coalesced-queries
Files: files | file ages | folders
SHA1: 948b22f1a9c3995a8dfd1484538fe9421b5df961
User & Date: matt on 2017-07-03 21:46:34
Other Links: branch diff | manifest | tags
Context
2017-07-04
09:32
typos Closed-Leaf check-in: fd0b9485a3 user: matt tags: v1.64-coalesced-queries
2017-07-03
21:46
small fixes check-in: 948b22f1a9 user: matt tags: v1.64-coalesced-queries
2017-06-29
23:17
queued coalesced queries (trying again) check-in: 0a1b205bcf user: matt tags: v1.64-coalesced-queries
Changes

Modified api.scm from [8da0d403fb] to [00c1b8c41b].

342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
    ;;          (boolean? res))
    ;;      res 
    ;;      (list "ERROR, not string, list, number or boolean" 1 cmd params res)))))
    (db:obj->string res transport: 'http)))

(define api:queue-mutex (make-mutex))

(defstruct api:queues
  (enable      #f)
  (dbstruct    #f)                   ;; must be initialized!
  (mutex       (make-mutex))
  (readq      '())
  (writeq     '())
  (last-read   (current-milliseconds))
  (last-write  (current-milliseconds))
  (read-cvar   (make-condition-variable "reads"))
  (write-cvar  (make-condition-variable "writes"))
  )

;; api queued request handler
;;
;; qry-type: read write transaction
;;
(define (api:queued-request queues qry-type params proc)
  ;; add proc to read, write queue or if transaction do it immediately (for now, not sure but might need to process differently.)
  (if *queues*







<
<
<
<
<
<
<
<
<
<
<
<







342
343
344
345
346
347
348












349
350
351
352
353
354
355
    ;;          (boolean? res))
    ;;      res 
    ;;      (list "ERROR, not string, list, number or boolean" 1 cmd params res)))))
    (db:obj->string res transport: 'http)))

(define api:queue-mutex (make-mutex))













;; api queued request handler
;;
;; qry-type: read write transaction
;;
(define (api:queued-request queues qry-type params proc)
  ;; add proc to read, write queue or if transaction do it immediately (for now, not sure but might need to process differently.)
  (if *queues*

Modified common.scm from [bfed3753ba] to [361eab7cdb].

124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
;; (define *server-id*         #f)
(define *server-info*       #f)  ;; good candidate for easily convert to non-global
(define *time-to-exit*      #f)
(define *server-run*        #t)
(define *run-id*            #f)
(define *server-kind-run*   (make-hash-table))
(define *home-host*         #f)
(define *queues*            (make-api:queues enable: #t)) ;; set up the queues for coalescing queries
;; (define *total-non-write-delay* 0)
(define *heartbeat-mutex*   (make-mutex))
(define *api-process-request-count* 0)
(define *max-api-process-requests* 0)
(define *server-overloaded*  #f)

;; client







|







124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
;; (define *server-id*         #f)
(define *server-info*       #f)  ;; good candidate for easily convert to non-global
(define *time-to-exit*      #f)
(define *server-run*        #t)
(define *run-id*            #f)
(define *server-kind-run*   (make-hash-table))
(define *home-host*         #f)

;; (define *total-non-write-delay* 0)
(define *heartbeat-mutex*   (make-mutex))
(define *api-process-request-count* 0)
(define *max-api-process-requests* 0)
(define *server-overloaded*  #f)

;; client
164
165
166
167
168
169
170















171
172
173
174
175
176
177
  (conndat           #f)
  (transport         *transport-type*)
  (server-timeout    (server:get-timeout)) ;; default from server:get-timeout
  (force-server      #f)
  (ro-mode           #f)  
  (ro-mode-checked   #f)) ;; flag that indicates we have checked for ro-mode
















;; launching and hosts
(defstruct host
  (reachable    #f)
  (last-update  0)
  (last-used    0)
  (last-cpuload 1))








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
  (conndat           #f)
  (transport         *transport-type*)
  (server-timeout    (server:get-timeout)) ;; default from server:get-timeout
  (force-server      #f)
  (ro-mode           #f)  
  (ro-mode-checked   #f)) ;; flag that indicates we have checked for ro-mode

;; api queued requests structure and global (temporary solution)
(defstruct api:queues
  (enable      #f)
  (dbstruct    #f)                   ;; must be initialized!
  (mutex       (make-mutex))
  (readq      '())
  (writeq     '())
  (last-read   (current-milliseconds))
  (last-write  (current-milliseconds))
  (read-cvar   (make-condition-variable "reads"))
  (write-cvar  (make-condition-variable "writes"))
  )

(define *queues*            (make-api:queues enable: #t)) ;; set up the queues for coalescing queries

;; launching and hosts
(defstruct host
  (reachable    #f)
  (last-update  0)
  (last-used    0)
  (last-cpuload 1))

Modified rmt.scm from [36c2c78229] to [adba1500ac].

333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
(define (rmt:open-qry-close-locally cmd run-id params #!key (remretries 5))
  (let* ((qry-is-write   (not (member cmd api:read-only-queries)))
	 (db-file-path   (db:dbfile-path)) ;;  0))
	 (dbstruct-local (db:setup #t))  ;; make-dbr:dbstruct path:  dbdir local: #t)))
	 (read-only      (not (file-write-access? db-file-path)))
	 (start          (current-milliseconds))
	 (resdat         (if (not (and read-only qry-is-write))
			     (let ((v (api:execute-requests dbstruct-local (vector (symbol->string cmd) params *queues*))))
			       (handle-exceptions ;; there has been a long history of receiving strange errors from values returned by the client when things go wrong..
				exn               ;;  This is an attempt to detect that situation and recover gracefully
				(begin
				  (debug:print0 *default-log-port* "ERROR: bad data from server " v " message: "  ((condition-property-accessor 'exn 'message) exn))
				  (vector #t '())) ;; should always get a vector but if something goes wrong return a dummy
				(if (and (vector? v)
					 (> (vector-length v) 1))







|







333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
(define (rmt:open-qry-close-locally cmd run-id params #!key (remretries 5))
  (let* ((qry-is-write   (not (member cmd api:read-only-queries)))
	 (db-file-path   (db:dbfile-path)) ;;  0))
	 (dbstruct-local (db:setup #t))  ;; make-dbr:dbstruct path:  dbdir local: #t)))
	 (read-only      (not (file-write-access? db-file-path)))
	 (start          (current-milliseconds))
	 (resdat         (if (not (and read-only qry-is-write))
			     (let ((v (api:execute-requests dbstruct-local (vector (symbol->string cmd) params) *queues*)))
			       (handle-exceptions ;; there has been a long history of receiving strange errors from values returned by the client when things go wrong..
				exn               ;;  This is an attempt to detect that situation and recover gracefully
				(begin
				  (debug:print0 *default-log-port* "ERROR: bad data from server " v " message: "  ((condition-property-accessor 'exn 'message) exn))
				  (vector #t '())) ;; should always get a vector but if something goes wrong return a dummy
				(if (and (vector? v)
					 (> (vector-length v) 1))