Megatest

Check-in [5de2fd17b5]
Login
Overview
Comment:Get data from archive partially working
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.65-broken
Files: files | file ages | folders
SHA1: 5de2fd17b5348d5283f874e7ec9bf2c53cad12a0
User & Date: mrwellan on 2020-04-21 04:13:36
Other Links: branch diff | manifest | tags
Context
2020-04-27
17:46
Archive get now works well check-in: 0645166f9c user: mrwellan tags: v1.65-broken
2020-04-21
04:13
Get data from archive partially working check-in: 5de2fd17b5 user: mrwellan tags: v1.65-broken
2020-04-20
23:38
Fixed typo check-in: 2a858c5054 user: mrwellan tags: v1.65-broken
Changes

Modified archive.scm from [618f9a591e] to [bb0fd0a689].

384
385
386
387
388
389
390
391













































		 (debug:print-info 0 *default-log-port* "Restoring archived data to " new-test-physical-path " from archive in " archive-path " ... " archive-internal-path)
		 ;; (mutex-lock! bup-mutex)
		 (run-n-wait bup-exe params: bup-restore-params print-cmd: #f)
		 ;; (mutex-unlock! bup-mutex)
		 (mt:test-set-state-status-by-id run-id test-id "COMPLETED" #f #f)))
	     (debug:print-error 0 *default-log-port* "No archive path in the record for run-id=" run-id " test-id=" test-id))))
     (filter vector? tests))))
	 




















































|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
		 (debug:print-info 0 *default-log-port* "Restoring archived data to " new-test-physical-path " from archive in " archive-path " ... " archive-internal-path)
		 ;; (mutex-lock! bup-mutex)
		 (run-n-wait bup-exe params: bup-restore-params print-cmd: #f)
		 ;; (mutex-unlock! bup-mutex)
		 (mt:test-set-state-status-by-id run-id test-id "COMPLETED" #f #f)))
	     (debug:print-error 0 *default-log-port* "No archive path in the record for run-id=" run-id " test-id=" test-id))))
     (filter vector? tests))))

;; from an archive get a specific path - works ONLY with bup for now
;;
(define (archive:bup-get-data archive-command run-id run-name tests rp-mutex bup-mutex)
  (if (null? tests)
      (debug:print-info 0 *default-log-port* "get-data called with no matching tests to operate on.")
      (let* ((bup-exe      (or (configf:lookup *configdat* "archive" "bup") "bup"))
	     (linktree     (common:get-linktree)) ;; (configf:lookup *configdat* "setup" "linktree")))
	     (test-dat     (car tests))
	     (destpath     (args:get-arg "-dest")))

	;; When restoring test-dat will initially contain an old and invalid path to the test
	(let* ((item-path         (db:test-get-item-path test-dat))
	       (test-name         (db:test-get-testname  test-dat))
	       (test-id           (db:test-get-id        test-dat))
	       (run-id            (db:test-get-run_id    test-dat))
	       (keyvals           (rmt:get-key-val-pairs run-id))
	       (target            (string-intersperse (map cadr keyvals) "/"))
	       
	       (toplevel/children (and (db:test-get-is-toplevel test-dat)
				       (> (rmt:test-toplevel-num-items run-id test-name) 0)))
	       (test-partial-path (conc target "/" run-name "/" (db:test-make-full-name test-name item-path)))
	       ;; note the trailing slash to get the dir inspite of it being a link
	       (test-path         (conc linktree "/" test-partial-path))
	       (mutex-lock! rp-mutex)
	       (mutex-unlock! rp-mutex)
	       (archive-block-id        (db:test-get-archived test-dat))
	       (archive-block-info      (rmt:test-get-archive-block-info archive-block-id))
	       (archive-path            (if (vector? archive-block-info)
					    (vector-ref archive-block-info 2) ;; look in db.scm for test-get-archive-block-info for the vector record info
					    #f)) ;; no archive found?
	       (archive-internal-path   (conc (common:get-testsuite-name) "-" run-id "/latest/" test-partial-path)))
	  
	  (if (and archive-path ;; no point in proceeding if there is no actual archive
		   (not toplevel/children))
	      (begin
		;; bup -d /tmp/matt/adisk1/2015_q1/fullrun_e1a40/ restore -C /tmp/seeme fullrun-30/latest/ubuntu/nfs/none/w02.1.20.54_b/
		
		;; DO BUP RESTORE
		(let* (;; new-test-path won't work - must use best-disk instead? Nope, new-test-path but tack on /..
		       (bup-restore-params  (list "-d" archive-path "restore" "-C" (or destpath "data") " " archive-internal-path)))
		  (debug:print-info 0 *default-log-port* "Restoring archived data to " (or destpath "data") " from archive in " archive-path " ... " archive-internal-path)
		  ;; (mutex-lock! bup-mutex)
		  (run-n-wait bup-exe params: bup-restore-params print-cmd: #f)))
	      (debug:print-error 0 *default-log-port* "No archive path in the record for run-id=" run-id " test-id=" test-id))))))

Modified megatest.scm from [23508e9e3f] to [cd4b9c8593].

238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
  -refdb2dat refdb        : convert refdb to sexp or to format specified by s-dumpmode
                            formats: perl, ruby, sqlite3, csv (for csv the -o param
                            will substitute %s for the sheet name in generating 
                            multiple sheets)
  -o                      : output file for refdb2dat (defaults to stdout)
  -archive cmd            : archive runs specified by selectors to one of disks specified
                            in the [archive-disks] section.
                            cmd: keep-html, restore, save, save-remove
  -generate-html          : create a simple html dashboard for browsing your runs
  -generate-html-structure  : create a top level html veiw to list targets/runs and a Run view within each run directory.  
  -list-run-time          : list time requered to complete runs. It supports following switches
                            -run-patt <patt> -target-patt <patt> -dumpmode <csv,json,plain-text>
  -list-test-time	  : list time requered to complete each test in a run. It following following arguments
                            -runname <patt> -target <patt> -dumpmode <csv,json,plain-text>
  -syscheck               : do some very basic checks; write access and space in tmp, home, runs, links and 







|







238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
  -refdb2dat refdb        : convert refdb to sexp or to format specified by s-dumpmode
                            formats: perl, ruby, sqlite3, csv (for csv the -o param
                            will substitute %s for the sheet name in generating 
                            multiple sheets)
  -o                      : output file for refdb2dat (defaults to stdout)
  -archive cmd            : archive runs specified by selectors to one of disks specified
                            in the [archive-disks] section.
                            cmd: keep-html, restore, save, save-remove, get (use -dest to set destination)
  -generate-html          : create a simple html dashboard for browsing your runs
  -generate-html-structure  : create a top level html veiw to list targets/runs and a Run view within each run directory.  
  -list-run-time          : list time requered to complete runs. It supports following switches
                            -run-patt <patt> -target-patt <patt> -dumpmode <csv,json,plain-text>
  -list-test-time	  : list time requered to complete each test in a run. It following following arguments
                            -runname <patt> -target <patt> -dumpmode <csv,json,plain-text>
  -syscheck               : do some very basic checks; write access and space in tmp, home, runs, links and 
310
311
312
313
314
315
316

317
318
319
320
321
322
323
			"-logpro"
			"-m"
			"-rerun"

			"-days"
			"-rename-run"
			"-to"

			;; values and messages
			":category"
			":variable"
			":value"
			":expected"
			":tol"
			":units"







>







310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
			"-logpro"
			"-m"
			"-rerun"

			"-days"
			"-rename-run"
			"-to"
			"-dest"
			;; values and messages
			":category"
			":variable"
			":value"
			":expected"
			":tol"
			":units"

Modified runs.scm from [38b916e9a8] to [f48ffb3bcd].

2167
2168
2169
2170
2171
2172
2173
2174

2175
2176
2177
2178


2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
		   ((run-wait)
		    (debug:print 1 *default-log-port* "Waiting for run " runkey ", run=" runnamepatt " to complete"))
		   ((archive)
		    (debug:print 1 *default-log-port* "Archiving/restoring (" (args:get-arg "-archive") ") data for run: " runkey " " (db:get-value-by-header run header "runname"))
		    (set! worker-thread
			  (make-thread
			   (lambda ()
			     (case (string->symbol (args:get-arg "-archive"))

			       ((save save-remove keep-html)
				(archive:run-bup (args:get-arg "-archive") run-id run-name tests rp-mutex bup-mutex))
			       ((restore)
				(archive:bup-restore (args:get-arg "-archive") run-id run-name tests rp-mutex bup-mutex))


			       (else 
				(debug:print-error 0 *default-log-port* "unrecognised sub command to -archive. Run \"megatest\" to see help")
				(exit))))
			   "archive-bup-thread"))
		    (thread-start! worker-thread))
		   (else
		    (debug:print-info 0 *default-log-port* "action not recognised " action)))
		 
		 ;; actions that operate on one test at a time can be handled below
		 ;;







|
>

|

|
>
>

|
|







2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
		   ((run-wait)
		    (debug:print 1 *default-log-port* "Waiting for run " runkey ", run=" runnamepatt " to complete"))
		   ((archive)
		    (debug:print 1 *default-log-port* "Archiving/restoring (" (args:get-arg "-archive") ") data for run: " runkey " " (db:get-value-by-header run header "runname"))
		    (set! worker-thread
			  (make-thread
			   (lambda ()
			     (let ((op (string->symbol (args:get-arg "-archive"))))
			       (case op
			       ((save save-remove keep-html)
				(archive:run-bup op run-id run-name tests rp-mutex bup-mutex))
			       ((restore)
				(archive:bup-restore op run-id run-name tests rp-mutex bup-mutex))
			       ((get)
				(archive:bup-get-data op run-id run-name tests rp-mutex bup-mutex))
			       (else 
				(debug:print-error 0 *default-log-port* "unrecognised sub command " op " for -archive. Run \"megatest\" to see help")
				(exit)))))
			   "archive-bup-thread"))
		    (thread-start! worker-thread))
		   (else
		    (debug:print-info 0 *default-log-port* "action not recognised " action)))
		 
		 ;; actions that operate on one test at a time can be handled below
		 ;;