Megatest

Check-in [884a77869d]
Login
Overview
Comment:switched to strategy of leveraging operate-on
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.65-rerun-logpro
Files: files | file ages | folders
SHA1: 884a77869d8281d66058ef769c4f880fb6f51865
User & Date: bjbarcla on 2018-12-11 18:11:30
Other Links: branch diff | manifest | tags
Context
2018-12-12
13:34
connected operate-on to ezsteps check-in: f2aaee9100 user: bjbarcla tags: v1.65-rerun-logpro
2018-12-11
18:11
switched to strategy of leveraging operate-on check-in: 884a77869d user: bjbarcla tags: v1.65-rerun-logpro
2018-12-06
18:14
added hooks for -rerun-logpro check-in: 8a363f876e user: bjbarcla tags: v1.65-rerun-logpro
Changes

Modified megatest.scm from [1137c4d993] to [d873fa8b25].

114
115
116
117
118
119
120
121

122
123
124
125
126
127
128
114
115
116
117
118
119
120

121
122
123
124
125
126
127
128







-
+







                            Optionally use :state and :status, use -keep-records to remove only
                            the run data.
  -set-state-status X,Y   : set state to X and status to Y, requires controls per -remove-runs
  -rerun FAIL,WARN...     : force re-run for tests with specificed status(s)
  -rerun-clean            : set all tests not COMPLETED+PASS,WARN,WAIVED to NOT_STARTED,n/a
                            and then run the specified testpatt with -preclean
  -rerun-all              : set all tests to NOT_STARTED,n/a and run with -preclean
  -redo-logpro            : do not rerun tests, but reapply logpro rules (ez-step flavor tests only)
  -redo-logpro            : do not rerun tests, but reapply logpro rules (ez-step flavor tests only; runs all tests unless -testpatt specified)
  -lock                   : lock run specified by target and runname
  -unlock                 : unlock run specified by target and runname
  -set-run-status status  : sets status for run to status, requires -target and -runname
  -get-run-status         : gets status for run specified by target and runname
  -run-wait               : wait on run specified by target and runname
  -preclean               : remove the existing test directory before running the test
  -clean-cache            : remove the cached megatest.config and runconfigs.config files
291
292
293
294
295
296
297
298

299
300
301
302
303
304
305
291
292
293
294
295
296
297

298
299
300
301
302
303
304
305







-
+







			"-itempatt"
			"-setlog"
			"-set-toplog"
			"-runstep"
			"-logpro"
			"-m"
			"-rerun"
                        "-redo-logpro"

			"-days"
			"-rename-run"
			"-to"
			;; values and messages
			":category"
			":variable"
			":value"
403
404
405
406
407
408
409

410
411
412
413
414
415
416
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417







+







			"-get-run-status"

			;; queries
			"-test-paths" ;; get path(s) to a test, ordered by youngest first

			"-runall"    ;; run all tests, respects -testpatt, defaults to %
			"-run"       ;; alias for -runall
                        "-redo-logpro"
			"-remove-runs"
                        "-keep-records" ;; use with -remove-runs to remove only the run data
			"-rebuild-db"
			"-cleanup-db"
			"-rollup"
			"-update-meta"
			"-create-megatest-area"
572
573
574
575
576
577
578
579

580
581
582
583
584
585
586
573
574
575
576
577
578
579

580
581
582
583
584
585
586
587







-
+







					    (printf "Sending signal/term to ~A\n" pid)
					    (process-signal pid signal/term))))))
		       (process:children #f))
		      (original-exit exit-code)))))

;; for some switches always print the command to stderr
;;
(if (args:any? "-run" "-runall" "-remove-runs" "-set-state-status")
(if (args:any? "-run" "-runall" "-remove-runs" "-set-state-status" "-redo-logpro")
    (debug:print 0 *default-log-port* (string-intersperse (argv) " ")))

;; some switches imply homehost. Exit here if not on homehost
;;
(let ((homehost-required  (list "-cleanup-db" "-server")))
  (if (apply args:any? homehost-required)
      (if (not (common:on-homehost?))
1091
1092
1093
1094
1095
1096
1097







1098
1099
1100
1101
1102
1103
1104
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112







+
+
+
+
+
+
+







(if (args:get-arg "-set-state-status")
    (general-run-call 
     "-set-state-status"
     "set state and status"
     (lambda (target runname keys keyvals)
       (operate-on 'set-state-status))))

(if (args:get-arg "-redo-logpro")
    (general-run-call 
     "-redo-logpro"
     "rerun logpro in ezsteps"
     (lambda (target runname keys keyvals)
       (operate-on 'redo-logpro))))

(if (or (args:get-arg "-set-run-status")
	(args:get-arg "-get-run-status"))
    (general-run-call
     "-set-run-status"
     "set run status"
     (lambda (target runname keys keyvals)
       (let* ((runsdat  (rmt:get-runs-by-patt keys runname 
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1253
1254
1255
1256
1257
1258
1259











1260
1261
1262
1263
1264
1265
1266







-
-
-
-
-
-
-
-
-
-
-







                    
                            table-rows))))
  (set! *didsomething* #t)
  (set! *time-to-exit* #t))



;; redo logpro
(if (args:get-arg "-redo-logpro")
  (let* ((result (redo-logpro:redo-logpro)))
    (set! *didsomething* #t)
    (set! *time-to-exit* #t)
    (cond
     ((not result)
      
      (exit 1))
     (else
      #t))))


;; NOTE: list-runs and list-db-targets operate on local db!!!
;;
;; IDEA: megatest list -runname blah% ...
;;
(if (or (args:get-arg "-list-runs")

Modified runs.scm from [18e897116f] to [88f46cb537].

2021
2022
2023
2024
2025
2026
2027
2028

2029
2030
2031
2032
2033
2034
2035
2021
2022
2023
2024
2025
2026
2027

2028
2029
2030
2031
2032
2033
2034
2035







-
+







	 (states       (if state  (string-split state  ",") '()))
	 (statuses     (if status (string-split status ",") '()))
	 (state-status (if (string? new-state-status) (string-split new-state-status ",") '(#f #f)))
	 (rp-mutex     (make-mutex))
	 (bup-mutex    (make-mutex))
         (keep-records (args:get-arg "-keep-records"))) ;; used in conjunction with -remove-runs to keep the records, TODO: consolidate this with "mode".

    (let* ((write-access-actions '(remove-runs set-state-status archive run-wait))
    (let* ((write-access-actions '(remove-runs set-state-status archive run-wait redo-logpro))
           (dbfile             (conc  *toppath* "/megatest.db"))
           (readonly-mode      (not (file-write-access? dbfile))))
      (when (and readonly-mode
                 (member action write-access-actions))
        (debug:print-error 0 *default-log-port* "megatest.db is readonly.  Cannot proceed with action ["action"] in which write-access isrequired .")
        (exit 1)))
    
2066
2067
2068
2069
2070
2071
2072


2073
2074
2075
2076
2077
2078
2079
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081







+
+







		    ;; (if (tasks:need-server run-id)(tasks:start-and-wait-for-server tdbdat run-id 10))
		    ;; seek and kill in flight -runtests with % as testpatt here
		    ;; (if (equal? testpatt "%")
		    (tasks:kill-runner target run-name testpatt)
		    
		    ;; (debug:print 0 *default-log-port* "not attempting to kill any run launcher processes as testpatt is " testpatt))
		    (debug:print 1 *default-log-port* "Removing tests for run: " runkey " " (db:get-value-by-header run header "runname")))
                   ((redo-logpro)
                    (debug:print 1 *default-log-port* "Re-applying new logpro rules without rerun for tests for run: " runkey " " (db:get-value-by-header run header "runname")))
		   ((set-state-status)
		    ;; (if (tasks:need-server run-id)(tasks:start-and-wait-for-server tdbdat run-id 10))
		    (debug:print 1 *default-log-port* "Modifying state and staus for tests for run: " runkey " " (db:get-value-by-header run header "runname")))
		   ((print-run)
		    (debug:print 1 *default-log-port* "Printing info for run " runkey ", run=" run ", tests=" tests ", header=" header)
		    action)
		   ((run-wait)
2130
2131
2132
2133
2134
2135
2136

2137
2138
2139
2140
2141
2142
2143
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146







+







                                  (has-subrun    (and (subrun:subrun-test-initialized? run-dir)
                                                      (not (subrun:subrun-removed? run-dir))))
				  (test-state    (db:test-get-state new-test-dat))
				  (test-fulln    (db:test-get-fullname new-test-dat))
				  (uname         (db:test-get-uname    new-test-dat))
				  (toplevel-with-children (and (db:test-get-is-toplevel test)
							       (> (rmt:test-toplevel-num-items run-id test-name) 0))))
                             (BB> "arrived here 2")
			     (case action
			       ((remove-runs)
				;; if the test is a toplevel-with-children issue an error and do not remove
				(cond
                                 (toplevel-with-children
                                  (debug:print 0 *default-log-port* "WARNING: skipping removal of " test-fulln " with run-id " run-id " as it has sub tests")
                                  (hash-table-set! toplevel-retries test-fulln (+ (hash-table-ref/default toplevel-retries test-fulln 0) 1))
2200
2201
2202
2203
2204
2205
2206

2207
2208
2209
2210
2211
2212
2213
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217







+







						(loop (car tal)(cdr tal))))))
				       )
				      ) ; end case rem-status
                                    ) ; end let
                                  ); end cond has-subrun

                                 (else
                                  (BB> "arrived 1")
                                  ;; BB - TODO - consider backgrounding to threads to delete tests (work below) 
                                  (debug:print-info 0 *default-log-port* "test: " test-name " itest-state: " test-state)
                                  (if (member test-state (list "RUNNING" "LAUNCHED" "REMOTEHOSTSTART" "KILLREQ"))
                                      (begin
                                        (if (not (hash-table-ref/default test-retry-time test-fulln #f))
                                            (begin
                                              ;; want to set to REMOVING BUT CANNOT do it here?
2228
2229
2230
2231
2232
2233
2234




2235
2236
2237
2238
2239
2240
2241
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249







+
+
+
+







                                            (loop new-test-dat tal)
                                            (loop (car tal)(append tal (list new-test-dat)))))
                                      (begin
                                        (runs:remove-test-directory new-test-dat mode) ;; 'remove-all)
                                        (if (not (null? tal))
                                            (loop (car tal)(cdr tal)))))))
				(rmt:update-run-stats run-id (rmt:get-raw-run-stats run-id)))
                               ((redo-logpro)
                                (debug:print-error 0 "redo-logpro unimplemented")
                                ;;(exit 1)
                                )
			       ((set-state-status)
                                (let* ((new-state (car state-status))
                                       (new-status (cadr state-status))
                                       (test-id (db:test-get-id test))
                                       (test-run-dir (db:test-get-rundir new-test-dat))
                                       (has-subrun (and (subrun:subrun-test-initialized? test-run-dir)
                                                      (not (subrun:subrun-removed? test-run-dir)))))