Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -1181,30 +1181,33 @@ (let ((res (proc))) (set! *client-non-blocking-mode* #f) res)) ;; params = 'target cached remparams -(define (cdb:client-call zmq-socket . params) - (debug:print-info 11 "cdb:client-call zmq-socket=" zmq-socket " params=" params) - (let ((zdat (db:obj->string params)) ;; (with-output-to-string (lambda ()(serialize params)))) - (res #f)) - ;; (signal-mask! signal/int) - (set! *received-response* #f) +(define (cdb:client-call zmq-sockets . params) + (debug:print-info 11 "cdb:client-call zmq-sockets=" zmq-sockets " params=" params) + (let* ((push-socket (vector-ref zmq-sockets 0)) + (sub-socket (vector-ref zmq-sockets 1)) + (query-id (conc (server:get-client-signature) "-" (message-digest-string (md5-primitive) (conc params)))) + (zdat (db:obj->string (vector query-id params))) ;; (with-output-to-string (lambda ()(serialize params)))) + (res #f) + (get-res (lambda () + (db:string->obj (if *client-non-blocking-mode* + (receive-message* zmq-socket) + (receive-message zmq-socket)))))) (send-message zmq-socket zdat) - ;; (signal-unmask! signal/int) - (set! res (db:string->obj (if *client-non-blocking-mode* - (receive-message* zmq-socket) - (receive-message zmq-socket)))) - (set! *received-response* #t) - (debug:print-info 11 "zmq-socket " (car params) " res=" res) - res)) + (let loop ((res (get-res))) + (if res res + (begin + (thread-sleep! 0.5) + (get-res)))))) (define (cdb:set-verbosity zmq-socket val) (cdb:client-call zmq-socket 'set-verbosity #f val)) -(define (cdb:login zmq-socket keyval signature) - (cdb:client-call zmq-socket 'login #t keyval megatest-version signature)) +(define (cdb:login zmq-sockets keyval signature) + (cdb:client-call zmq-sockets 'login #t keyval megatest-version signature)) (define (cdb:logout zmq-socket keyval signature) (cdb:client-call zmq-socket 'logout #t keyval signature)) (define (cdb:num-clients zmq-socket) Index: server.scm ================================================================== --- server.scm +++ server.scm @@ -23,10 +23,34 @@ (declare (uses tests)) (declare (uses tasks)) ;; tasks are where stuff is maintained about what is running. (include "common_records.scm") (include "db_records.scm") + +;; Transition to pub --> sub with pull <-- push +;; +;; 1. client sends request to server via push to the pull port +;; 2. server puts request in queue or processes immediately as appropriate +;; 3. server puts responses from completed requests into pub port +;; +;; TODO +;; +;; Done Tested +;; [x] [ ] 1. Add columns pullport pubport to servers table +;; [x] [ ] 2. Add rm of monitor.db if older than 11/12/2012 +;; [x] [ ] 3. Add create of pullport and pubport with finding of available ports +;; [ ] [ ] 4. Add client compose of request +;; [ ] [ ] - name of client: testname/itempath-test_id-hostname +;; [ ] [ ] - name of request: callname, params +;; [ ] [ ] - request key: f(clientname, callname, params) +;; [ ] [ ] 5. Add processing of subscription hits +;; [ ] [ ] - done when get key +;; [ ] [ ] - return results +;; [ ] [ ] 6. Add timeout processing +;; [ ] [ ] - after 60 seconds +;; [ ] [ ] i. check server alive, connect to new if necessary +;; [ ] [ ] ii. resend request (define (server:make-server-url hostport) (if (not hostport) #f (conc "tcp://" (car hostport) ":" (cadr hostport)))) @@ -43,35 +67,49 @@ (mutex-lock! *heartbeat-mutex*) (set! *server-loop-heart-beat* (current-seconds)) (mutex-unlock! *heartbeat-mutex*) (loop)))) +(define-inline (zmqsock:get-pub dat)(vector-ref dat 0)) +(define-inline (zmqsock:get-pull dat)(vector-ref dat 1)) +(define-inline (zmqsock:set-pub! dat s)(vector-set! dat s 0)) +(define-inline (zmqsock:set-pull! dat s)(vector-set! dat s 0)) + (define (server:run hostn) (debug:print 0 "Attempting to start the server ...") (if (not *toppath*) (if (not (setup-for-run)) (begin (debug:print 0 "ERROR: cannot find megatest.config, cannot start server, exiting") (exit)))) - (let* ((zmq-socket #f) - (zmq-socket-dat #f) - (iface (if (string=? "-" hostn) - "*" ;; (get-host-name) - hostn)) - (hostname (get-host-name)) - (ipaddrstr (let ((ipstr (if (string=? "-" hostn) - (string-intersperse (map number->string (u8vector->list (hostname->ip hostname))) ".") - #f))) - (if ipstr ipstr hostname))) - (actual-port #f)) - ;; (set! zmq-socket (server:find-free-port-and-open iface zmq-socket 5555 0)) - (set! zmq-socket-dat (server:find-free-port-and-open ipaddrstr zmq-socket (if (args:get-arg "-port") - (string->number (args:get-arg "-port")) - (+ 5000 (random 1001))) - 0)) - (set! zmq-socket (cadr zmq-socket-dat)) - (set! actual-port (caddr zmq-socket-dat)) + (let* ((zmq-sdat1 #f) + (zmq-sdat2 #f) + (zmq-socket1 #f) + (zmq-socket2 #f) + (p1 #f) + (p2 #f) + (zmq-sockets-dat #f) + (iface (if (string=? "-" hostn) + "*" ;; (get-host-name) + hostn)) + (hostname (get-host-name)) + (ipaddrstr (let ((ipstr (if (string=? "-" hostn) + (string-intersperse (map number->string (u8vector->list (hostname->ip hostname))) ".") + #f))) + (if ipstr ipstr hostname)))) + (set! zmq-sockets-dat (server:setup-ports ipaddrstr (if (args:get-arg "-port") + (string->number (args:get-arg "-port")) + (+ 5000 (random 1001))))) + + (set! zmq-sdat1 (car zmq-socket-dat)) + (set! zmq-socket1 (car zmq-sdat1)) + (set! p1 (caddr zmq-sdat1)) + + (set! zmq-sdat2 (cadr zmq-socket-dat)) + (set! zmq-socket2 (car zmq-sdat2)) + (set! p2 (caddr zmq-sdat2)) + (set! *cache-on* #t) ;; (set! th1 (make-thread (lambda () ;; (server:self-ping ipaddrstr actual-port)))) ;; (thread-start! th1) @@ -79,11 +117,11 @@ ;; what to do when we quit ;; (on-exit (lambda () (if (and *toppath* *server-info*) (begin - (open-run-close tasks:server-deregister-self tasks:open-db ipaddrstr)) + (open-run-close tasks:server-deregister-self tasks:open-db ipaddrstr p1 p2)) (let loop () (let ((queue-len 0)) (thread-sleep! (random 5)) (mutex-lock! *incoming-mutex*) (set! queue-len (length *incoming-data*)) @@ -94,21 +132,13 @@ (loop)))))))) ;; The heavy lifting ;; (let loop () - ;; ;; Ugly yuk. - ;; (mutex-lock! *incoming-mutex*) - ;; (set! *server-loop-heart-beat* (list 'waiting (current-seconds))) - ;; (mutex-unlock! *incoming-mutex*) - (let* ((rawmsg (receive-message* zmq-socket)) + (let* ((rawmsg (receive-message* zmq-socket1)) (params (db:string->obj rawmsg)) ;; (with-input-from-string rawmsg (lambda ()(deserialize)))) (res #f)) - ;;; Ugly yuk. - ;; (mutex-lock! *incoming-mutex*) - ;; (set! *server-loop-heart-beat* (list 'working (current-seconds))) - ;; (mutex-unlock! *incoming-mutex*) (debug:print-info 12 "server=> received params=" params) (set! res (cdb:cached-access params)) (debug:print-info 12 "server=> processed res=" res) (send-message zmq-socket (db:obj->string res)) (if (not *time-to-exit*) @@ -172,12 +202,12 @@ (thread-sleep! 1) (debug:print-info 0 "Max cached queries was " *max-cache-size*) (debug:print-info 0 "Server shutdown complete. Exiting") (exit))))))) -(define (server:find-free-port-and-open iface s port #!key (trynum 50)) - (let ((s (if s s (make-socket 'rep))) +(define (server:find-free-port-and-open iface s port stype #!key (trynum 50)) + (let ((s (if s s (make-socket stype))) (p (if (number? port) port 5555)) (old-handler (current-exception-handler))) (handle-exceptions exn (begin @@ -186,20 +216,28 @@ ;; (old-handler) ;; (print-call-chain) (if (> trynum 0) (server:find-free-port-and-open iface s (+ p 1) trynum: (- trynum 1)) (debug:print-info 0 "Tried ports up to " p - " but all were in use. Please try a different port range by starting the server with parameter \" -port N\" where N is the starting port number to use"))) + " but all were in use. Please try a different port range by starting the server with parameter \" -port N\" where N is the starting port number to use")) + (exit)) ;; To exit or not? That is the question. (let ((zmq-url (conc "tcp://" iface ":" p))) - (print "Trying to start server on " zmq-url) + (debug:print 0 "Trying to start server on " zmq-url) (bind-socket s zmq-url) - (set! *runremote* #f) - (debug:print 0 "Server started on " zmq-url) - (mutex-lock! *heartbeat-mutex*) - (set! *server-info* (open-run-close tasks:server-register tasks:open-db (current-process-id) iface p 0 'live)) - (mutex-unlock! *heartbeat-mutex*) (list iface s port))))) + +(define (server:setup-ports ipadrstr startport) + (let* ((s1 (server:find-free-port-and-open ipadrstr #f startport 'pub)) + (p1 (caddr s1)) + (s2 (server:find-free-port-and-open ipadrstr #f (+ 1 (if p1 p1 (+ startport 1))) 'pull)) + (p2 (caddr s2))) + (set! *runremote* #f) + (debug:print 0 "Server started on " ipaddrstr " ports " p1 " and p2") + (mutex-lock! *heartbeat-mutex*) + (set! *server-info* (open-run-close tasks:server-register tasks:open-db (current-process-id) iface p 0 'live)) + (mutex-unlock! *heartbeat-mutex*) + (list s1 s2))) (define (server:mk-signature) (message-digest-string (md5-primitive) (with-output-to-string (lambda () @@ -211,26 +249,26 @@ (let ((sig (server:mk-signature))) (set! *my-client-signature* sig) *my-client-signature*))) ;; -(define (server:client-connect iface port #!key (context #f)) +(define (server:client-connect iface port #!key (context #f)(type 'req)) (debug:print-info 3 "client-connect " iface ":" port) (let ((connect-ok #f) (zmq-socket (if context - (make-socket 'req context) - (make-socket 'req))) + (make-socket type context) + (make-socket type))) (conurl (server:make-server-url (list iface port)))) (if (socket? zmq-socket) (begin (connect-socket zmq-socket conurl) zmq-socket) #f))) -(define (server:client-login zmq-socket) - (cdb:login zmq-socket *toppath* (server:get-client-signature))) +(define (server:client-login zmq-sockets) + (cdb:login zmq-sockets *toppath* (server:get-client-signature))) (define (server:client-logout zmq-socket) (let ((ok (and (socket? zmq-socket) (cdb:logout zmq-socket *toppath* (server:get-client-signature))))) ;; (close-socket zmq-socket) @@ -243,13 +281,14 @@ (begin (debug:print 0 "ERROR: failed to find megatest.config, exiting") (exit)))) (let ((hostinfo (open-run-close tasks:get-best-server tasks:open-db))) (if hostinfo - (let ((host (car hostinfo)) - (iface (cadr hostinfo)) - (port (caddr hostinfo))) + (let ((host (list-ref hostinfo 0)) + (iface (list-ref hostinfo 1)) + (pullport (list-ref hostinfo 2)) + (pubport (list-ref hostinfo 3))) (debug:print-info 2 "Setting up to connect to " hostinfo) (handle-exceptions exn (begin ;; something went wrong in connecting to the server. In this scenario it is ok @@ -258,15 +297,20 @@ (debug:print 0 " EXCEPTION: " ((condition-property-accessor 'exn 'message) exn)) (debug:print 0 " perhaps jobs killed with -9? Removing server records") (open-run-close tasks:server-deregister tasks:open-db host port: port) (server:client-setup (- numtries 1)) #f) - (let* ((zmq-socket (server:client-connect iface port)) - (login-res (server:client-login zmq-socket)) - (connect-ok (if (null? login-res) #f (car login-res))) + (let* ((push-socket (server:client-connect iface pullport 'push)) + (sub-socket (server:client-connect iface pubport 'sub)) + (zmq-sockets (vector push-socket sub-socket)) + (login-res #f) + ;; (connect-ok (conurl (server:make-server-url (list iface port)))) - (if connect-ok + (socket-option-set! sub-socket 'subscribe (server:get-client-signature)) + (set! login-res (server:client-login zmq-sockets)) + (if (and (not (null? login-res)) + (car login-res)) (begin (debug:print-info 2 "Logged in and connected to " conurl) (set! *runremote* zmq-socket) #t) (begin Index: tasks.scm ================================================================== --- tasks.scm +++ tasks.scm @@ -22,11 +22,15 @@ ;; Tasks db ;;====================================================================== (define (tasks:open-db) (let* ((dbpath (conc *toppath* "/monitor.db")) - (exists (file-exists? dbpath)) + (exists (if (file-exists? dbpath) + ;; BUGGISHNESS: Remove this code in six months. Today is 11/13/2012 + (if (< (file-change-time dbpath) 1352851396.0) + (begin (delete-file dbpath) #f) + #t) #t)) (mdb (sqlite3:open-database dbpath)) ;; (never-give-up-open-db dbpath)) (handler (make-busy-timeout 36000))) (sqlite3:set-busy-handler! mdb handler) (sqlite3:execute mdb (conc "PRAGMA synchronous = 0;")) (if (not exists) @@ -52,11 +56,12 @@ CONSTRAINT monitors_constraint UNIQUE (pid,hostname));") (sqlite3:execute mdb "CREATE TABLE IF NOT EXISTS servers (id INTEGER PRIMARY KEY, pid INTEGER, interface TEXT, hostname TEXT, - port INTEGER, + pullport INTEGER, + pubport INTEGER, start_time TIMESTAMP, priority INTEGER, state TEXT, mt_version TEXT, heartbeat TIMESTAMP, @@ -162,32 +167,42 @@ (sqlite3:for-each-row (lambda (id hostname interface port pid) (set! res (cons (list hostname interface port pid) res)) (debug:print-info 2 "Found existing server " hostname ":" port " registered in db")) mdb - "SELECT id,hostname,interface,port,pid FROM servers WHERE state='live' AND mt_version=? ORDER BY start_time ASC LIMIT 1;" megatest-version) + "SELECT id,hostname,interface,pullport,pubport,pid FROM servers WHERE state='live' AND mt_version=? ORDER BY start_time ASC LIMIT 1;" megatest-version) ;; (print "res=" res) (if (null? res) #f (let loop ((hed (car res)) (tal (cdr res))) ;; (print "hed=" hed ", tal=" tal) - (let* ((host (car hed)) - (iface (cadr hed)) - (port (caddr hed)) - (pid (cadddr hed)) - (alive (open-run-close tasks:server-alive? tasks:open-db #f hostname: host port: port))) + (let* ((host (list-ref hed 0)) + (iface (list-ref hed 1)) + (pullport (list-ref hed 2)) + (pubport (list-ref hed 3)) + (pid (list-ref hed 4)) + (alive (open-run-close tasks:server-alive? tasks:open-db #f hostname: host pullport: pullport))) (if alive (begin - (debug:print-info 2 "Found an existing, alive, server " host ":" port ".") - (list host iface port)) + (debug:print-info 2 "Found an existing, alive, server " host ", " pullport " and " pubport ".") + (list host iface pullport pubport)) (begin - (debug:print-info 1 "Removing " host ":" port " from server registry as it appears to be dead") - (tasks:kill-server #f host port pid) + (debug:print-info 1 "Marking " host ":" pullport " as dead in server registry.") + (if port + (open-run-close tasks:server-deregister tasks:open-db host pullport: pullport) + (open-run-close tasks:server-deregister tasks:open-db host pid: pid)) (if (null? tal) #f (loop (car tal)(cdr tal)))))))))) +(define (tasks:mark-server hostname pullport pid state) + (if port + (open-run-close tasks:server-deregister tasks:open-db hostname port: port) + (open-run-close tasks:server-deregister tasks:open-db hostname pid: pid))) + + +;; NOTE: NOT PORTED TO WORK WITH pullport/pubport (define (tasks:kill-server status hostname port pid) (debug:print-info 1 "Removing defunct server record for " hostname ":" port) (if port (open-run-close tasks:server-deregister tasks:open-db hostname port: port) (open-run-close tasks:server-deregister tasks:open-db hostname pid: pid)) Index: utils/installall.sh ================================================================== --- utils/installall.sh +++ utils/installall.sh @@ -1,6 +1,6 @@ -#!/bin/bash +#! /bin/env bash set -x # Copyright 2007-2010, Matthew Welland. # @@ -11,11 +11,11 @@ # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR # PURPOSE. echo You may need to do the following first: echo sudo apt-get install libreadline-dev -echo sudo apt-get install libwebkitgtk-dev +echo sudo apt-get install libwebkitgtk-dev echo sudo apt-get install libmotif3 -OR- set KTYPE=26g4 echo KTYPE can be 26, 26g4, or 32 echo KTYPE=$KTYPE echo You are using PREFIX=$PREFIX echo You are using proxy="$proxy" @@ -268,11 +268,12 @@ tar xfz ${ZEROMQ}.tar.gz cd ${ZEROMQ} ln -s $PREFIX/include/uuid src # LDFLAGS=-L$PREFIX/lib ./configure --prefix=$PREFIX - ./configure --enable-static --disable-shared --prefix=$PREFIX --with-uuid=$PREFIX LDFLAGS="-L$PREFIX/lib" CPPFLAGS="-fPIC -I$PREFIX/include" LIBS="-lgcc" + ./configure --enable-static --prefix=$PREFIX --with-uuid=$PREFIX LDFLAGS="-L$PREFIX/lib" CPPFLAGS="-fPIC -I$PREFIX/include" LIBS="-lgcc" + # --disable-shared CPPFLAGS="-fPIC # LDFLAGS="-L/usr/lib64 -L$PREFIX/lib" ./configure --enable-static --prefix=$PREFIX make make install CSC_OPTIONS="-I$PREFIX/include -L$CSCLIBS" chicken-install $PROX zmq # CSC_OPTIONS="-I$PREFIX/include -L$CSCLIBS" chicken-install $PROX -deploy -prefix $DEPLOYTARG zmq