Index: Makefile ================================================================== --- Makefile +++ Makefile @@ -11,10 +11,13 @@ http-transport.scm filedb.scm tdb.scm \ client.scm daemon.scm mt.scm \ ezsteps.scm lock-queue.scm sdb.scm \ rmt.scm api.scm \ portlogger.scm archive.scm env.scm diff-report.scm cgisetup/models/pgdb.scm + +# module source files +MSRCFILES = ftail.scm # Eggs to install (straightforward ones) EGGS=matchable readline apropos base64 regex-literals format regex-case test coops trace csv \ dot-locking posix-utils posix-extras directory-utils hostinfo tcp-server rpc csv-xml fmt \ json md5 awful http-client spiffy uri-common intarweb spiffy-request-vars \ @@ -22,10 +25,16 @@ GUISRCF = dashboard-tests.scm dashboard-guimonitor.scm gutils.scm dcommon.scm tree.scm vg.scm OFILES = $(SRCFILES:%.scm=%.o) GOFILES = $(GUISRCF:%.scm=%.o) + +MOFILES = $(addprefix mofiles/,$(MSRCFILES:%.scm=%.o)) + +mofiles/%.o : %.scm + mkdir -p mofiles + csc $(CSCOPTS) -J -c $< -o mofiles/$*.o ADTLSCR=mt_laststep mt_runstep mt_ezstep HELPERS=$(addprefix $(PREFIX)/bin/,$(ADTLSCR)) DEPLOYHELPERS=$(addprefix deploytarg/,$(ADTLSCR)) MTESTHASH=$(shell fossil info|grep checkout:| awk '{print $$2}') @@ -41,15 +50,15 @@ PNGFILES = $(shell cd docs/manual;ls *png) #all : $(PREFIX)/bin/.$(ARCHSTR) mtest dboard mtut ndboard all : $(PREFIX)/bin/.$(ARCHSTR) mtest dboard mtut -mtest: $(OFILES) readline-fix.scm megatest.o - csc $(CSCOPTS) $(OFILES) megatest.o -o mtest +mtest: $(OFILES) readline-fix.scm megatest.o $(MOFILES) + csc $(CSCOPTS) $(OFILES) $(MOFILES) megatest.o -o mtest -dboard : $(OFILES) $(GOFILES) dashboard.scm - csc $(CSCOPTS) $(OFILES) dashboard.scm $(GOFILES) -o dboard +dboard : $(OFILES) $(GOFILES) dashboard.scm $(MOFILES) + csc $(CSCOPTS) $(OFILES) dashboard.scm $(GOFILES) $(MOFILES) -o dboard ndboard : newdashboard.scm $(OFILES) $(GOFILES) csc $(CSCOPTS) $(OFILES) $(GOFILES) newdashboard.scm -o ndboard mtut: $(OFILES) megatest-fossil-hash.scm mtut.scm DELETED file-tail.scm Index: file-tail.scm ================================================================== --- file-tail.scm +++ /dev/null @@ -1,76 +0,0 @@ - -(use (prefix sqlite3 sqlite3:) posix typed-records) - -(define (open-tail-db ) - (let* ((basedir (create-directory (conc "/tmp/" (current-user-name)))) - (dbpath (conc basedir "/megatest_logs.db")) - (dbexists (common:file-exists? dbpath)) - (db (sqlite3:open-database dbpath)) - (handler (sqlite3:make-busy-timeout 136000))) - (sqlite3:set-busy-handler! db handler) - (sqlite3:execute db "PRAGMA synchronous = 0;") - (if (not dbexists) - (begin - (sqlite3:execute db "CREATE TABLE IF NOT EXISTS log_files (id INTEGER PRIMARY KEY,filename TEXT,event_time TIMESTAMP DEFAULT (strftime('%s','now')));") - (sqlite3:execute db "CREATE TABLE IF NOT EXISTS log_data (id INTEGER PRIMARY KEY,fid INTEGER,line TEXT,event_time TIMESTAMP DEFAULT (strftime('%s','now')));") - )) - db)) - -(define (tail-write db fid lines) - (sqlite3:with-transaction - db - (lambda () - (for-each - (lambda (line) - (sqlite3:execute db "INSERT INTO log_data (fid,line) VALUES (?,?);" fid line)) - lines)))) - -(define (tail-get-fid db fname) - (let ((fid (handle-exceptions - exn - #f - (sqlite3:first-result db "SELECT id FROM log_files WHERE filename=?;" fname)))) - (if fid - fid - (begin - (sqlite3:execute db "INSERT INTO log_files (filename) VALUES (?);" fname) - (tail-get-fid db fname))))) - -(define (file-tail fname #!key (db-in #f)) - (let* ((inp (open-input-file fname)) - (db (or db-in (open-tail-db))) - (fid (tail-get-fid db fname))) - (let loop ((inl (read-line inp)) - (lines '()) - (lastwr (current-seconds))) - (if (eof-object? inl) - (let ((timed-out (> (- (current-seconds) lastwr) 60))) - (if timed-out (tail-write db fid (reverse lines))) - (sleep 1) - (if timed-out - (loop (read-line inp) '() (current-seconds)) - (loop (read-line inp) lines lastwr))) - (let* ((savelines (> (length lines) 19))) - ;; (print inl) - (if savelines (tail-write db fid (reverse lines))) - (loop (read-line inp) - (if savelines - '() - (cons inl lines)) - (if savelines - (current-seconds) - lastwr))))))) - -;; offset -20 means get last 20 lines -;; -(define (tail-get-lines db fid offset count) - (if (> offset 0) - (map-row (lambda (id line) - (vector id line)) - db - "SELECT id,line FROM log_data WHERE fid=? OFFSET ? LIMIT ?;" fid offset count) - (reverse ;; get N from the end - (map-row (lambda (id line) - (vector id line)) - db - "SELECT id,line FROM log_data WHERE fid=? ORDER BY id DESC LIMIT ?;" fid (abs offset))))) ADDED ftail.scm Index: ftail.scm ================================================================== --- /dev/null +++ ftail.scm @@ -0,0 +1,99 @@ +;;====================================================================== +;; Copyright 2017, Matthew Welland. +;; +;; This program is made available under the GNU GPL version 2.0 or +;; greater. See the accompanying file COPYING for details. +;; +;; This program is distributed WITHOUT ANY WARRANTY; without even the +;; implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +;; PURPOSE. +;;====================================================================== + +(declare (unit ftail)) + +(module ftail + ( + open-tail-db + tail-write + tail-get-fid + file-tail + ) + +(import scheme chicken data-structures extras) +(use (prefix sqlite3 sqlite3:) posix typed-records) + +(define (open-tail-db ) + (let* ((basedir (create-directory (conc "/tmp/" (current-user-name)))) + (dbpath (conc basedir "/megatest_logs.db")) + (dbexists (file-exists? dbpath)) + (db (sqlite3:open-database dbpath)) + (handler (sqlite3:make-busy-timeout 136000))) + (sqlite3:set-busy-handler! db handler) + (sqlite3:execute db "PRAGMA synchronous = 0;") + (if (not dbexists) + (begin + (sqlite3:execute db "CREATE TABLE IF NOT EXISTS log_files (id INTEGER PRIMARY KEY,filename TEXT,event_time TIMESTAMP DEFAULT (strftime('%s','now')));") + (sqlite3:execute db "CREATE TABLE IF NOT EXISTS log_data (id INTEGER PRIMARY KEY,fid INTEGER,line TEXT,event_time TIMESTAMP DEFAULT (strftime('%s','now')));") + )) + db)) + +(define (tail-write db fid lines) + (sqlite3:with-transaction + db + (lambda () + (for-each + (lambda (line) + (sqlite3:execute db "INSERT INTO log_data (fid,line) VALUES (?,?);" fid line)) + lines)))) + +(define (tail-get-fid db fname) + (let ((fid (handle-exceptions + exn + #f + (sqlite3:first-result db "SELECT id FROM log_files WHERE filename=?;" fname)))) + (if fid + fid + (begin + (sqlite3:execute db "INSERT INTO log_files (filename) VALUES (?);" fname) + (tail-get-fid db fname))))) + +(define (file-tail fname #!key (db-in #f)) + (let* ((inp (open-input-file fname)) + (db (or db-in (open-tail-db))) + (fid (tail-get-fid db fname))) + (let loop ((inl (read-line inp)) + (lines '()) + (lastwr (current-seconds))) + (if (eof-object? inl) + (let ((timed-out (> (- (current-seconds) lastwr) 60))) + (if timed-out (tail-write db fid (reverse lines))) + (sleep 1) + (if timed-out + (loop (read-line inp) '() (current-seconds)) + (loop (read-line inp) lines lastwr))) + (let* ((savelines (> (length lines) 19))) + ;; (print inl) + (if savelines (tail-write db fid (reverse lines))) + (loop (read-line inp) + (if savelines + '() + (cons inl lines)) + (if savelines + (current-seconds) + lastwr))))))) + +;; offset -20 means get last 20 lines +;; +(define (tail-get-lines db fid offset count) + (if (> offset 0) + (sqlite3:map-row (lambda (id line) + (vector id line)) + db + "SELECT id,line FROM log_data WHERE fid=? OFFSET ? LIMIT ?;" fid offset count) + (reverse ;; get N from the end + (sqlite3:map-row (lambda (id line) + (vector id line)) + db + "SELECT id,line FROM log_data WHERE fid=? ORDER BY id DESC LIMIT ?;" fid (abs offset))))) + +) Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -42,10 +42,12 @@ (declare (uses mt)) (declare (uses api)) (declare (uses tasks)) ;; only used for debugging. (declare (uses env)) (declare (uses diff-report)) +(declare (uses ftail)) +(import ftail) (define *db* #f) ;; this is only for the repl, do not use in general!!!! (include "common_records.scm") (include "key_records.scm")