Overview
| Comment: | Added tar and 7z to archiving methods |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | v1.65 |
| Files: | files | file ages | folders |
| SHA1: |
e15d743b85df021b3c3a8af77165c81d |
| User & Date: | matt on 2018-05-24 23:56:55 |
| Other Links: | branch diff | manifest | tags |
Context
|
2018-05-25
| ||
| 16:42 | Fixed typo, wrapped get-normalized-load to enable always returning alist. check-in: c64db4cd71 user: mrwellan tags: v1.65 | |
|
2018-05-24
| ||
| 23:56 | Added tar and 7z to archiving methods check-in: e15d743b85 user: matt tags: v1.65 | |
| 17:42 | Initial setup for archive user controlled areas. check-in: 8a75426772 user: mrwellan tags: v1.65 | |
Changes
Modified archive.scm from [2906655364] to [2b8c7ad3e5].
| ︙ | ︙ | |||
129 130 131 132 133 134 135 | ;; 3. gen index ;; 4. save ;; (define (archive:run-bup archive-command run-id run-name tests rp-mutex bup-mutex) ;; move the getting of archive space down into the below block so that a single run can ;; allocate as needed should a disk fill up ;; | | | | > | | > | | | | | > > > > > | 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 |
;; 3. gen index
;; 4. save
;;
(define (archive:run-bup archive-command run-id run-name tests rp-mutex bup-mutex)
;; move the getting of archive space down into the below block so that a single run can
;; allocate as needed should a disk fill up
;;
(let* ((blockid-cache (make-hash-table))
(tsname (common:get-testsuite-name))
(min-space (string->number (or (configf:lookup *configdat* "archive" "minspace") "1000")))
(arch-groups (make-hash-table)) ;; archive groups, each corrosponds to a bup area
(disk-groups (make-hash-table)) ;;
(test-groups (make-hash-table)) ;; these two (disk and test groups) could be combined nicely
(test-dirs (make-hash-table))
(bup-exe (or (configf:lookup *configdat* "archive" "bup") "bup"))
(compress (or (configf:lookup *configdat* "archive" "compress") "9"))
(linktree (common:get-linktree)) ;; (configf:lookup *configdat* "setup" "linktree")))
(archiver (let ((s (configf:lookup *configdat* "archive" "archiver")))
(if s (string->symbol s) 'bup)))
(archiver-cmd (case archiver
((tar) "tar cfj ARCHIVE_NAME.tar.bz2 ")
((7z) " 7z u -t7z -m0=lzma -mx=9 -mfb=64 -md=32m -ms=on ARCHIVE_NAME.7z ")
(else #f)))
(print-prefix "Running: ")) ;; change to #f to turn off printing
;; from the test info bin the path to the test by stem
;;
(for-each
(lambda (test-dat)
(let* ((item-path (db:test-get-item-path test-dat))
(test-name (db:test-get-testname test-dat))
(test-id (db:test-get-id test-dat))
|
| ︙ | ︙ | |||
202 203 204 205 206 207 208 | "test-physical-path = " test-physical-path "\n" "partial-path-index = " partial-path-index "\n" "test-base = " test-base) (hash-table-set! disk-groups test-base (cons test-physical-path (hash-table-ref/default disk-groups test-base '()))) (hash-table-set! test-groups test-base (cons test-dat (hash-table-ref/default test-groups test-base '()))) | > > > | > | | > > > > > | | < > > > > | | | | | | > | < < < | | | | | | | | | | | | | > > > > > > > > > > > > > > > > > > > > > > | | | | | | | | | | | 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 |
"test-physical-path = " test-physical-path "\n"
"partial-path-index = " partial-path-index "\n"
"test-base = " test-base)
(hash-table-set! disk-groups test-base
(cons test-physical-path (hash-table-ref/default disk-groups test-base '())))
(hash-table-set! test-groups test-base
(cons test-dat (hash-table-ref/default test-groups test-base '())))
(hash-table-set! arch-groups test-base
(cons archive-info (hash-table-ref/default arch-groups test-base '())))
(hash-table-set! test-dirs test-id test-path)))))
;; test-path))))
tests)
(debug:print 0 *default-log-port* "INFO: DISK GROUPS=" (hash-table->alist disk-groups))
;; for each disk-group, initialize the bup area if needed
(for-each
(lambda (test-base)
(let* ((disk-group (hash-table-ref disk-groups test-base))
(arch-group (hash-table-ref arch-groups test-base))
(arch-info (car arch-group)) ;; don't know yet how this will work, can I get more than one possibility?
(archive-id (car arch-info))
(archive-dir (cdr arch-info)))
(debug:print 0 *default-log-port* "Processing disk-group " test-base)
(let* ((test-paths (hash-table-ref disk-groups test-base)))
(if (not (common:file-exists? archive-dir))
(create-directory archive-dir #t))
(case archiver
((bup) ;; Archive using bup
(let* ((bup-init-params (list "-d" archive-dir "init"))
(bup-index-params (append (list "-d" archive-dir "index") test-paths))
(bup-save-params (append (list "-d" archive-dir "save" ;; (conc "--strip-path=" linktree)
(conc "-" compress) ;; or (conc "--compress=" compress)
"-n" (conc (common:get-testsuite-name) "-" run-id)
(conc "--strip-path=" test-base) ;; if we push to the directory do we need this?
)
test-paths)))
(if (not (common:file-exists? (conc archive-dir "/HEAD")))
(begin
;; replace this with jobrunner stuff enventually
(debug:print-info 0 *default-log-port* "Init bup in " archive-dir)
;; (mutex-lock! bup-mutex)
(run-n-wait bup-exe params: bup-init-params print-cmd: print-prefix)
;; (mutex-unlock! bup-mutex)
))
(debug:print-info 0 *default-log-port* "Indexing data to be archived")
;; (mutex-lock! bup-mutex)
(run-n-wait bup-exe params: bup-index-params print-cmd: print-prefix)
(debug:print-info 0 *default-log-port* "Archiving data with bup")
(run-n-wait bup-exe params: bup-save-params print-cmd: print-prefix)))
((7z tar)
(for-each
(lambda (test-dat)
(let* ((test-id (db:test-get-id test-dat))
(test-name (db:test-get-testname test-dat))
(item-path (db:test-get-item-path test-dat))
(test-full-name (db:test-make-full-name test-name item-path))
(run-id (db:test-get-run_id test-dat))
(target (string-intersperse (map cadr (rmt:get-key-val-pairs run-id)) "/"))
(run-name (rmt:get-run-name-from-id run-id))
(source-dir (hash-table-ref test-dirs test-id)) ;; (conc test-base "/" test-name "/" item-path))
(target-dir (string-substitute "/$" "" (conc archive-dir "/" target "/" run-name "/" test-full-name))))
;; create the test and item-path levels under archive-dir
(create-directory (pathname-directory target-dir) #t)
(run-n-wait
(conc
(string-substitute "ARCHIVE_NAME" target-dir archiver-cmd) " "
"."
)
print-cmd: print-prefix
run-dir: source-dir)))
(hash-table-ref test-groups test-base))))
;; (mutex-unlock! bup-mutex)
(for-each
(lambda (test-dat)
(let ((test-id (db:test-get-id test-dat))
(run-id (db:test-get-run_id test-dat)))
(rmt:test-set-archive-block-id run-id test-id archive-id)
(if (member archive-command '("save-remove"))
(runs:remove-test-directory test-dat 'archive-remove))))
(hash-table-ref test-groups test-base)))))
(hash-table-keys disk-groups))
#t))
(define (archive:bup-restore archive-command run-id run-name tests rp-mutex bup-mutex) ;; move the getting of archive space down into the below block so that a single run can
;; allocate as needed should a disk fill up
;;
(let* ((bup-exe (or (configf:lookup *configdat* "archive" "bup") "bup"))
(linktree (common:get-linktree))) ;; (configf:lookup *configdat* "setup" "linktree")))
|
| ︙ | ︙ |
Modified configf.scm from [4863d68fd9] to [14aabdff7b].
| ︙ | ︙ | |||
440 441 442 443 444 445 446 |
settings)
curr-section-name key #f)))
(configf:key-val-pr ( x key unk1 val unk2 )
(let* ((alist (hash-table-ref/default res curr-section-name '()))
(envar (and environ-patt
(string-search (regexp environ-patt) curr-section-name) ;; does the section match the envionpatt?
| | > > | 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 |
settings)
curr-section-name key #f)))
(configf:key-val-pr ( x key unk1 val unk2 )
(let* ((alist (hash-table-ref/default res curr-section-name '()))
(envar (and environ-patt
(string-search (regexp environ-patt) curr-section-name) ;; does the section match the envionpatt?
(and (not (string-null? key))(equal? "!" (substring key 0 1))) ;; ! as leading character is a signature to NOT export to the environment
;; (string-match "^.*:.*:.*$" key) ;; ;; something:something:something reserved for triggers in runconfigs
))
(realval (if envar
(config:eval-string-in-environment val)
val)))
(debug:print-info 6 *default-log-port* "read-config env setting, envar: " envar " realval: " realval " val: " val " key: " key " curr-section-name: " curr-section-name)
(if envar (safe-setenv key realval))
(debug:print 10 *default-log-port* " setting: [" curr-section-name "] " key " = " val)
(hash-table-set! res curr-section-name
|
| ︙ | ︙ |
Modified process.scm from [3945f219f6] to [ba823d2c36].
| ︙ | ︙ | |||
18 19 20 21 22 23 24 | ;;====================================================================== ;;====================================================================== ;; Process convience utils ;;====================================================================== | | | 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 |
;;======================================================================
;;======================================================================
;; Process convience utils
;;======================================================================
(use regex directory-utils)
(declare (unit process))
(define (process:conservative-read port)
(let loop ((res ""))
(if (not (eof-object? (peek-char port)))
(loop (conc res (read-char port)))
res)))
|
| ︙ | ︙ | |||
137 138 139 140 141 142 143 |
(loop (let ((l (read-line fh)))
(if (eof-object? l) l (proc l)))
(append result (list curr)))
result))))
;; here is an example line where the shell is sh or bash
;; "find / -print 2&>1 > findall.log"
| | > | > > > > > > > | | 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 |
(loop (let ((l (read-line fh)))
(if (eof-object? l) l (proc l)))
(append result (list curr)))
result))))
;; here is an example line where the shell is sh or bash
;; "find / -print 2&>1 > findall.log"
(define (run-n-wait cmdline #!key (params #f)(print-cmd #f)(run-dir #f))
(if print-cmd
(debug:print 0 *default-log-port*
(if (string? print-cmd)
print-cmd
"")
(if run-dir (conc "Run in " run-dir ";") "")
cmdline
(if params
(conc " " (string-intersperse params " "))
"")))
(if (and run-dir
(directory-exists? run-dir))
(push-directory run-dir))
(let ((pid (if params
(process-run cmdline params)
(process-run cmdline))))
(let loop ((i 0))
(let-values (((pid-val exit-status exit-code) (process-wait pid #t)))
(if (eq? pid-val 0)
(begin
(thread-sleep! 2)
(loop (+ i 1)))
(begin
(if (and run-dir
(directory-exists? run-dir))
(pop-directory))
(values pid-val exit-status exit-code)))))))
;;======================================================================
;; MISC PROCESS RELATED STUFF
;;======================================================================
(define (process:children proc)
(with-input-from-pipe
|
| ︙ | ︙ |