Megatest

Check-in [096366f99c]
Login
Overview
Comment:Added placeholder for pgdb sync
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.64
Files: files | file ages | folders
SHA1: 096366f99cfc0925049fa315510186fca2b4d1c3
User & Date: matt on 2017-02-25 23:18:30
Other Links: branch diff | manifest | tags
Context
2017-02-26
10:54
Misc. work on route to megatest.db -> mt pg db sync check-in: 6a8906cbd3 user: matt tags: v1.64
2017-02-25
23:18
Added placeholder for pgdb sync check-in: 096366f99c user: matt tags: v1.64
21:10
Cleaned up schema for common postgres db check-in: 5574ded5f0 user: matt tags: v1.64
Changes

Modified Makefile from [7d122a8cb8] to [5fd97b3f14].

8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
   ods.scm runconfig.scm server.scm configf.scm \
   db.scm keys.scm margs.scm megatest-version.scm \
   process.scm runs.scm tasks.scm tests.scm genexample.scm \
   http-transport.scm filedb.scm \
   client.scm synchash.scm daemon.scm mt.scm \
   ezsteps.scm lock-queue.scm sdb.scm \
   rmt.scm api.scm tdb.scm rpc-transport.scm \
   portlogger.scm archive.scm env.scm diff-report.scm

# Eggs to install (straightforward ones)
EGGS=matchable readline apropos base64 regex-literals format regex-case test coops trace csv \
dot-locking posix-utils posix-extras directory-utils hostinfo tcp-server rpc csv-xml fmt \
json md5 awful http-client spiffy uri-common intarweb spiffy-request-vars \
spiffy-directory-listing ssax sxml-serializer sxml-modifications iup canvas-draw sqlite3








|







8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
   ods.scm runconfig.scm server.scm configf.scm \
   db.scm keys.scm margs.scm megatest-version.scm \
   process.scm runs.scm tasks.scm tests.scm genexample.scm \
   http-transport.scm filedb.scm \
   client.scm synchash.scm daemon.scm mt.scm \
   ezsteps.scm lock-queue.scm sdb.scm \
   rmt.scm api.scm tdb.scm rpc-transport.scm \
   portlogger.scm archive.scm env.scm diff-report.scm pgdb.scm

# Eggs to install (straightforward ones)
EGGS=matchable readline apropos base64 regex-literals format regex-case test coops trace csv \
dot-locking posix-utils posix-extras directory-utils hostinfo tcp-server rpc csv-xml fmt \
json md5 awful http-client spiffy uri-common intarweb spiffy-request-vars \
spiffy-directory-listing ssax sxml-serializer sxml-modifications iup canvas-draw sqlite3

Modified megatest.scm from [40b4d84b66] to [4741723578].

142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
  -sort fieldname         : in -list-runs sort tests by this field

Misc 
  -start-dir path         : switch to this directory before running megatest
  -contour cname          : add a level of hierarcy to the linktree and run paths
  -rebuild-db             : bring the database schema up to date
  -cleanup-db             : remove any orphan records, vacuum the db
  -import-megatest.db     : migrate a database from v1.55 series to v1.60 series
  -sync-to-megatest.db    : migrate data back to megatest.db
  -use-db-cache           : use cached access to db to reduce load
  -update-meta            : update the tests metadata for all tests
  -setvars VAR1=val1,VAR2=val2 : Add environment variables to a run NB// these are
                                 overwritten by values set in config files.
  -server -|hostname      : start the server (reduces contention on megatest.db), use
                            - to automatically figure out hostname
  -transport http|rpc     : use http or rpc for transport (default is http) 
  -daemonize              : fork into background and disconnect from stdin/out







|
|
|







142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
  -sort fieldname         : in -list-runs sort tests by this field

Misc 
  -start-dir path         : switch to this directory before running megatest
  -contour cname          : add a level of hierarcy to the linktree and run paths
  -rebuild-db             : bring the database schema up to date
  -cleanup-db             : remove any orphan records, vacuum the db
  -import-megatest.db     : push data from megatest.db to cache db files in /tmp/$USER
  -sync-to-megatest.db    : pull data from cache files in /tmp/$USER to megatest.db
  -sync-to-pg             : sync to new postgresql central style database
  -update-meta            : update the tests metadata for all tests
  -setvars VAR1=val1,VAR2=val2 : Add environment variables to a run NB// these are
                                 overwritten by values set in config files.
  -server -|hostname      : start the server (reduces contention on megatest.db), use
                            - to automatically figure out hostname
  -transport http|rpc     : use http or rpc for transport (default is http) 
  -daemonize              : fork into background and disconnect from stdin/out
341
342
343
344
345
346
347

348
349
350
351
352
353
354
355
			"-create-megatest-area"
			"-mark-incompletes"

			"-convert-to-norm"
			"-convert-to-old"
			"-import-megatest.db"
			"-sync-to-megatest.db"


			"-logging"
			"-v" ;; verbose 2, more than normal (normal is 1)
			"-q" ;; quiet 0, errors/warnings only

                        "-diff-rep"
                        )
		 args:arg-hash







>
|







341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
			"-create-megatest-area"
			"-mark-incompletes"

			"-convert-to-norm"
			"-convert-to-old"
			"-import-megatest.db"
			"-sync-to-megatest.db"
			"-sync-to-pg"
			
			"-logging"
			"-v" ;; verbose 2, more than normal (normal is 1)
			"-q" ;; quiet 0, errors/warnings only

                        "-diff-rep"
                        )
		 args:arg-hash

Modified mt-pg.sql from [18b3da6663] to [97be168469].

20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
DROP TABLE IF EXISTS test_steps;
DROP TABLE IF EXISTS test_data;
DROP TABLE IF EXISTS test_rundat;
DROP TABLE IF EXISTS archives;

CREATE TABLE IF NOT EXISTS areas (
       id INTEGER PRIMARY KEY,
       areaname TEXT DEFAULT 'local',
       areapath TEXT DEFAULT '.',
       last_sync INTEGER DEFAULT 0,
       CONSTRAINT areaconstraint UNIQUE (areaname));

INSERT INTO areas (id,areaname,areapath) VALUES (0,'local','.');

CREATE TABLE IF NOT EXISTS ttype (
       id INTEGER PRIMARY KEY,
       target_spec TEXT DEFAULT '');
       
CREATE TABLE IF NOT EXISTS runs (
       id INTEGER PRIMARY KEY,







|
|

|

|







20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
DROP TABLE IF EXISTS test_steps;
DROP TABLE IF EXISTS test_data;
DROP TABLE IF EXISTS test_rundat;
DROP TABLE IF EXISTS archives;

CREATE TABLE IF NOT EXISTS areas (
       id INTEGER PRIMARY KEY,
       area_name TEXT DEFAULT 'local',
       area_path TEXT DEFAULT '.',
       last_sync INTEGER DEFAULT 0,
       CONSTRAINT areaconstraint UNIQUE (area_name));

INSERT INTO areas (id,area_name,area_path) VALUES (0,'local','.');

CREATE TABLE IF NOT EXISTS ttype (
       id INTEGER PRIMARY KEY,
       target_spec TEXT DEFAULT '');
       
CREATE TABLE IF NOT EXISTS runs (
       id INTEGER PRIMARY KEY,
193
194
195
196
197
198
199
200


       state        TEXT DEFAULT 'new',
       status       TEXT DEFAULT 'n/a',
       archive_type TEXT DEFAULT 'bup',
       du           INTEGER,
       archive_path TEXT);
 

TRUNCATE archive_blocks, archive_allocations, extradat, metadat, access_log, tests, test_steps, test_data, test_rundat, archives, keys, runs, run_stats, test_meta, tasks_queue, archive_disks;









|
>
>
193
194
195
196
197
198
199
200
201
202
       state        TEXT DEFAULT 'new',
       status       TEXT DEFAULT 'n/a',
       archive_type TEXT DEFAULT 'bup',
       du           INTEGER,
       archive_path TEXT);
 

TRUNCATE archive_blocks, archive_allocations, extradat, metadat,
access_log, tests, test_steps, test_data, test_rundat, archives, runs,
run_stats, test_meta, tasks_queue, archive_disks;

Modified rmt.scm from [bd29e5791d] to [94c1d184f5].

1
2
3
4
5
6
7
8
9
;;======================================================================
;; Copyright 2006-2013, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.

|







1
2
3
4
5
6
7
8
9
;;======================================================================
;; Copyright 2006-2017, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.

Modified tasks.scm from [6f2c907335] to [b57542e266].

12
13
14
15
16
17
18



19
20
21
22
23
24
25
(use sqlite3 srfi-1 posix regex regex-case srfi-69 dot-locking format)
(import (prefix sqlite3 sqlite3:))

(declare (unit tasks))
(declare (uses db))
(declare (uses rmt))
(declare (uses common))




(include "task_records.scm")

;;======================================================================
;; Tasks db
;;======================================================================








>
>
>







12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
(use sqlite3 srfi-1 posix regex regex-case srfi-69 dot-locking format)
(import (prefix sqlite3 sqlite3:))

(declare (unit tasks))
(declare (uses db))
(declare (uses rmt))
(declare (uses common))
(declare (uses pgdb))

;; (import pgdb) ;; pgdb is a module

(include "task_records.scm")

;;======================================================================
;; Tasks db
;;======================================================================

573
574
575
576
577
578
579











;;     (runs:rollup-run db
;; 		     keys 
;; 		     keyvals
;; 		     (tasks:task-get-name  task)
;; 		     (tasks:task-get-owner  task))
;;     (tasks:set-state mdb (tasks:task-get-id task) "waiting")))



















>
>
>
>
>
>
>
>
>
>
>
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
;;     (runs:rollup-run db
;; 		     keys 
;; 		     keyvals
;; 		     (tasks:task-get-name  task)
;; 		     (tasks:task-get-owner  task))
;;     (tasks:set-state mdb (tasks:task-get-id task) "waiting")))

;;======================================================================
;;  S Y N C   T O   P O S T G R E S Q L
;;======================================================================

;; In the spirit of "dump your junk in the tasks module" I'll put the
;; sync to postgres here for now.

(define (tasks:sync-to-postgres)
  (let* ((dbh (pgdb:open *configdat*))
	 (area-info (pgdb:area-path->area-info dbh *toppath*)))
    (print "area-info: " area-info)))