Megatest

Check-in [044051d315]
Login
☰Home Timeline Files Branches Tags Tickets Wiki Login
Overview
Comment:fixed bug with loading area from Matt
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.63-readonly
Files: files | file ages | folders
SHA1: 044051d3157973386e2ec86740957939c8504edf
User & Date: bjbarcla on 2017-02-21 16:49:31
Other Links: branch diff | manifest | tags
Context
2017-02-21
16:52
fixed bug with loading area from Matt check-in: 6555418e4b user: bjbarcla tags: v1.63-readonly
16:49
fixed bug with loading area from Matt check-in: 044051d315 user: bjbarcla tags: v1.63-readonly
14:28
resolved deadlock in db:setup; fixed read-only detection in db migration proc check-in: ee0d52f1d2 user: bjbarcla tags: v1.63-readonly
Changes
Hide Diffs Side-by-Side Diffs Show Whitespace Changes Patch

Modified api.scm from [563b0aba54] to [4067424284].

︙
16
17
18
19
20
21
22

23
24
25
26
27
28
29

16
17
18
19
20
21
22
23
24
25
26
27
28
29
30








+








(declare (uses db))
(declare (uses tasks))

;; allow these queries through without starting a server
;;
(define api:read-only-queries
  '(get-key-val-pairs
    get-var
    get-keys
    get-key-vals
    test-toplevel-num-items
    get-test-info-by-id
    test-get-rundir-from-test-id
    get-count-tests-running-for-testname
    get-count-tests-running
︙
113
114
115
116
117
118
119
120


121
122
123
124

125
126


127
128

129
130
131
132
133
134



135


136
137
138
139


140
141
142
143
144
145
146

114
115
116
117
118
119
120

121
122
123
124
125
126
127


128
129


130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155








-
+
+




+
-
-
+
+
-
-
+






+
+
+

+
+




+
+








;;    - keep it simple, only return the actual result of the call, i.e. no meta info here
;;
;;    - returns #( flag result )
;;
(define (api:execute-requests dbstruct dat)
  (handle-exceptions
   exn
   (let ((call-chain (get-call-chain)))
   (let ((call-chain (get-call-chain))
         )
     (debug:print 0 *default-log-port* "WARNING: api:execute-requests received an exception from peer")
     (print-call-chain (current-error-port))
     (debug:print 0 *default-log-port* " message: "  ((condition-property-accessor 'exn 'message) exn))       
     (vector #f (vector exn call-chain dat))) ;; return some stuff for debug if an exception happens
   (cond
   (if (not (vector? dat))                    ;; it is an error to not receive a vector
       (vector #f #f "remote must be called with a vector")       
    ((not (vector? dat))                    ;; it is an error to not receive a vector
     (vector #f #f "remote must be called with a vector")       )
       (vector                                   ;; return a vector + the returned data structure
	#t 
    (else  
	(let* ((cmd-in (vector-ref dat 0))
	       (cmd    (if (symbol? cmd-in)
			  cmd-in
			  (string->symbol cmd-in)))
	       (params (vector-ref dat 1))
	       (start-t (current-milliseconds))
            (readonly-mode (dbr:dbstruct-read-only dbstruct))
            (readonly-command (member cmd api:read-only-queries))
            (writecmd-in-readonly-mode (and readonly-mode (not readonly-command)))
	       (res    
             (if writecmd-in-readonly-mode
                 (conc "attempt to run write command "cmd" on a read-only database")
		(case cmd
		  ;;===============================================
		  ;; READ/WRITE QUERIES
		  ;;===============================================

                   ((get-keys-write)                        (db:get-keys dbstruct)) ;; force a dummy "write" query to force server; for debug in -repl

		  ;; SERVERS
		  ((start-server)                    (apply server:kind-run params))
		  ((kill-server)                     (set! *server-run* #f))

		  ;; TESTS
		  ((test-set-state-status-by-id)     (apply db:test-set-state-status-by-id dbstruct params))
︙
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206

201
202
203
204
205
206
207

208
209
210
211
212
213
214








-








		  ;;======================================================================
		  ;; READ ONLY QUERIES
		  ;;======================================================================

		  ;; KEYS
		  ((get-key-val-pairs)               (apply db:get-key-val-pairs dbstruct params))
		  ((get-keys)                        (db:get-keys dbstruct))
		  ((get-keys-write)                        (db:get-keys dbstruct)) ;; force a dummy "write" query to force server
		  ((get-key-vals)                    (apply db:get-key-vals dbstruct params))
		  ((get-target)                      (apply db:get-target dbstruct params))
		  ((get-targets)                     (db:get-targets dbstruct))

		  ;; ARCHIVES
		  ((test-get-archive-block-info)     (apply db:test-get-archive-block-info dbstruct params))
︙
260
261
262
263
264
265
266
267


268
269
270
271
272



273
274
275
276
277
278
279

268
269
270
271
272
273
274

275
276
277
278
279


280
281
282
283
284
285
286
287
288
289








-
+
+



-
-
+
+
+








		  ((sdb-qry)                      (apply sdb:qry params))
		  ((ping)                         (current-process-id))

		  ;; TESTMETA
		  ((testmeta-get-record)       (apply db:testmeta-get-record dbstruct params))

		  ;; TASKS 
		  ((find-task-queue-records)   (apply tasks:find-task-queue-records dbstruct params)))))
                   ((find-task-queue-records)   (apply tasks:find-task-queue-records dbstruct params))))))
       (if (not writecmd-in-readonly-mode)
	  (let ((delta-t (- (current-milliseconds)
			    start-t)))
	    (hash-table-set! *db-api-call-time* cmd
			     (cons delta-t (hash-table-ref/default *db-api-call-time* cmd '()))))
	  res)))))
                              (cons delta-t (hash-table-ref/default *db-api-call-time* cmd '())))
             (vector #t res))
           (vector #f res)))))))

;; http-server  send-response
;;                 api:process-request
;;                    db:*
;;
;; NB// Runs on the server as part of the server loop
;;
︙

Modified common.scm from [892351b73c] to [0ca3492ab6].

︙
279
280
281
282
283
284
285

286
287
288
289
290
291
292

279
280
281
282
283
284
285
286
287
288
289
290
291
292
293








+








                (read-only (not (file-write-access? dbfile)))
                (dbstruct (db:setup)))
	    (debug:print 0 *default-log-port*
			 "WARNING: Version mismatch!\n"
			 "   expected: " (common:version-signature) "\n"
			 "   got:      " (common:get-last-run-version))
            (cond
             ((get-environment-variable "MT_SKIP_DB_MIGRATE") #t)
             ((and (file-exists? mtconf) (file-exists? dbfile) (not read-only)
                   (eq? (current-user-id)(file-owner mtconf))) ;; safe to run -cleanup-db
              (debug:print 0 *default-log-port* "   I see you are the owner of megatest.config, attempting to cleanup and reset to new version")
              (handle-exceptions
               exn
               (begin
                 (debug:print 0 *default-log-port* "Failed to switch versions.")
︙

Modified common_records.scm from [e3400966c5] to [4e8b115b3e].

︙
122
123
124
125
126
127
128
129

130
131
132
133


134
135
136



137


138
139
140
141
142
143
144

122
123
124
125
126
127
128

129
130
131
132

133
134
135
136
137
138
139
140

141
142
143
144
145
146
147
148
149








-
+



-
+
+



+
+
+
-
+
+








	      (apply print params)
	      )))))

;; Brandon's debug printer shortcut (indulge me :)
(define *BB-process-starttime* (current-milliseconds))
(define (BB> . in-args)
  (let* ((stack (get-call-chain))
         (location #f))
         (location "??"))
    (for-each
     (lambda (frame)
       (let* ((this-loc (vector-ref frame 0))
              (this-func (cadr (string-split this-loc " "))))
              (temp     (string-split (->string this-loc) " "))
              (this-func (if (and (list? temp) (> (length temp) 1)) (cadr temp) "???")))
         (if (equal? this-func "BB>")
             (set! location this-loc))))
     stack)
    (let ((dp-args
           (append
            (list 0 *default-log-port*
    (let ((dp-args (append (list 0 *default-log-port* (conc location "@"(/ (- (current-milliseconds) *BB-process-starttime*) 1000)"   ")  ) in-args)))
                  (conc location "@"(/ (- (current-milliseconds) *BB-process-starttime*) 1000)"   ")  )
            in-args)))
      (apply debug:print dp-args))))

(define *BBpp_custom_expanders_list* (make-hash-table))



;; register hash tables with BBpp.
︙

Modified db.scm from [ef6be90d9f] to [a536774740].

︙
559
560
561
562
563
564
565
566
567




568
569


570
571


572
573
574



575
576
577
578

579
580
581
582
583
584
585
586

559
560
561
562
563
564
565


566
567
568
569
570

571
572
573

574
575
576


577
578
579
580
581
582

583

584
585
586
587
588
589
590








-
-
+
+
+
+

-
+
+

-
+
+

-
-
+
+
+



-
+
-








			 (debug:print-error 0 *default-log-port* "Failed to rebuild " dbpath ", exiting now.")
			 (exit)))))
	       (cons todb slave-dbs))
     
     0)
   ;; this is the work to be done
   (cond
    ((not fromdb) (debug:print 3 *default-log-port* "WARNING: db:sync-tables called with fromdb missing") -1)
    ((not todb)   (debug:print 3 *default-log-port* "WARNING: db:sync-tables called with todb missing") -2)
    ((not fromdb) (debug:print 3 *default-log-port* "WARNING: db:sync-tables called with fromdb missing")
     -1)
    ((not todb)   (debug:print 3 *default-log-port* "WARNING: db:sync-tables called with todb missing")
     -2)
    ((not (sqlite3:database? (db:dbdat-get-db fromdb)))
     (debug:print-error 0 *default-log-port* "db:sync-tables called with fromdb not a database " fromdb) -3)
     (debug:print-error 0 *default-log-port* "db:sync-tables called with fromdb not a database " fromdb)
     -3)
    ((not (sqlite3:database? (db:dbdat-get-db todb)))
     (debug:print-error 0 *default-log-port* "db:sync-tables called with todb not a database " todb) -4)
     (debug:print-error 0 *default-log-port* "db:sync-tables called with todb not a database " todb)
     -4)

    ((not (file-write-access? (cadr todb)))
     (debug:print-error 0 *default-log-port* "db:sync-tables called with todb not a read-only database " todb) -5)
    ((not (file-write-access? (db:dbdat-get-path todb)))
     (debug:print-error 0 *default-log-port* "db:sync-tables called with todb not a read-only database " todb)
     -5)
    ((not (null? (let ((readonly-slave-dbs
                        (filter
                         (lambda (dbdat)
                           (not (file-write-access? (cadr todb))))
                           (not (file-write-access? (db:dbdat-get-path todb))))
                         
                         slave-dbs)))
                   (for-each
                    (lambda (bad-dbdat)
                      (debug:print-error
                       0 *default-log-port* "db:sync-tables called with todb not a read-only database " bad-dbdat))
                    readonly-slave-dbs)
                   readonly-slave-dbs))) -6)
︙

Modified runs.scm from [a06e687141] to [442e22f4c3].

︙
200
201
202
203
204
205
206


207
208
209
210
211
212
213
214
215
216





217
218
219
220
221
222
223

200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230








+
+










+
+
+
+
+








;;            
(define (runs:run-tests target runname test-patts user flags #!key (run-count 1)) ;; test-names
  (let* ((keys               (keys:config-get-fields *configdat*))
	 (keyvals            (keys:target->keyval keys target))
	 (run-id             (rmt:register-run keyvals runname "new" "n/a" user))  ;;  test-name)))
	 ;; (deferred          '()) ;; delay running these since they have a waiton clause
	 (runconfigf         (conc  *toppath* "/runconfigs.config"))
         (dbfile             (conc  *toppath* "/megatest.db"))
         (readonly-mode      (not (file-write-access? dbfile)))
	 (test-records       (make-hash-table))
	 ;; need to process runconfigs before generating these lists
	 (all-tests-registry #f)  ;; (tests:get-all)) ;; (tests:get-valid-tests (make-hash-table) test-search-path)) ;; all valid tests to check waiton names
	 (all-test-names     #f)  ;; (hash-table-keys all-tests-registry))
	 (test-names         #f)  ;; Generated by a call to (tests:filter-test-names all-test-names test-patts))
	 (required-tests     #f)  ;; Put fully qualified test/testpath names in this list to be done
	 (task-key           (conc (hash-table->alist flags) " " (get-host-name) " " (current-process-id)))
	 (tdbdat             (tasks:open-db))
	 (config-reruns      (let ((x (configf:lookup *configdat* "setup" "reruns")))
			       (if x (string->number x) #f))))

    ;; check if readonly
    (when readonly-mode
      (debug:print-error 0 *default-log-port* "megatest.db is readonly.  Cannot proceed.")
      (exit 1))

    ;; per user request. If less than 100Meg space on dbdir partition, bail out with error
    ;; this will reduce issues in database corruption
    (common:check-db-dir-and-exit-if-insufficient)
    
    ;; override the number of reruns from the configs
    (if (and config-reruns
︙
1637
1638
1639
1640
1641
1642
1643










1644
1645
1646
1647
1648
1649
1650

1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667








+
+
+
+
+
+
+
+
+
+








	 (header       (vector-ref rundat 0))
	 (runs         (vector-ref rundat 1))
	 (states       (if state  (string-split state  ",") '()))
	 (statuses     (if status (string-split status ",") '()))
	 (state-status (if (string? new-state-status) (string-split new-state-status ",") '(#f #f)))
	 (rp-mutex     (make-mutex))
	 (bup-mutex    (make-mutex)))

    (let* ((write-access-actions '(remove-runs set-state-status archive run-wait))
           (dbfile             (conc  *toppath* "/megatest.db"))
           (readonly-mode      (not (file-write-access? dbfile))))
      (when (and readonly-mode
                 (member action write-access-actions))
        (debug:print-error 0 *default-log-port* "megatest.db is readonly.  Cannot proceed with action ["action"] in which write-access isrequired .")
        (exit 1)))

    
    (debug:print-info 4 *default-log-port* "runs:operate-on => Header: " header " action: " action " new-state-status: " new-state-status)
    (if (> 2 (length state-status))
	(begin
	  (debug:print-error 0 *default-log-port* "the parameter to -set-state-status is a comma delimited string. E.g. COMPLETED,FAIL")
	  (exit)))
    (for-each
     (lambda (run)
︙
1890
1891
1892
1893
1894
1895
1896


1897
1898
1899
1900
1901
1902
1903

1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922








+
+








	(if (launch:setup)
	    (begin
	      (full-runconfigs-read) ;; cache the run config
	      (launch:cache-config)) ;; do not cache here - need to be sure runconfigs is processed
	    (begin 
	      (debug:print 0 *default-log-port* "Failed to setup, exiting")
	      (exit 1)))

        
	(set! keys (keys:config-get-fields *configdat*))
	;; have enough to process -target or -reqtarg here
	(if (args:get-arg "-reqtarg")
	    (let* ((runconfigf (conc  *toppath* "/runconfigs.config")) ;; DO NOT EVALUATE ALL 
		   (runconfig  (read-config runconfigf #f #t environ-patt: #f)))
	      (if (hash-table-ref/default runconfig (args:get-arg "-reqtarg") #f)
		  (keys:target-set-args keys (args:get-arg "-reqtarg") args:arg-hash)
︙

This page was generated in about 0.145s by Fossil 2.24 [452f8e1ffc] 2024-03-26 15:36:00