Comment: | Merged development to v1.54 for release as v1.5418 |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | v1.54 |
Files: | files | file ages | folders |
SHA1: |
90fd97673de9b0914baf46c1d380522a |
User & Date: | mrwellan on 2013-04-29 09:31:59 |
Other Links: | branch diff | manifest | tags |
2013-04-29
| ||
10:12 | Fixed typo in rsync call for populating test directories check-in: d25cf88e14 user: mrwellan tags: v1.54, v1.5418 | |
09:31 | Merged development to v1.54 for release as v1.5418 check-in: 90fd97673d user: mrwellan tags: v1.54 | |
2013-04-28
| ||
23:19 | Redirected stdout to mt_launch.log in test rsync calls for fdktestqa test check-in: 2829751795 user: matt tags: dev | |
08:49 | Couple refinements to test launch speedups check-in: 71105b561e user: matt tags: v1.54 | |
Modified db.scm from [1c5c537dce] to [9df9f1f05f].
︙ | |||
239 240 241 242 243 244 245 | 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 | - - - - - - + + + + + + - + - + - + - + - - + + + + | )) ;;====================================================================== ;; T E S T S P E C I F I C D B ;;====================================================================== ;; Create the sqlite db for the individual test(s) |
︙ | |||
840 841 842 843 844 845 846 | 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 | - + - + | db qry ) (debug:print-info 11 "db:get-tests-for-run START run-ids=" run-ids ", testpatt=" testpatt ", states=" states ", statuses=" statuses ", not-in=" not-in ", sort-by=" sort-by) res)) ;; this one is a bit broken BUG FIXME |
︙ | |||
979 980 981 982 983 984 985 | 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 | + + + - - + + | run-id testname item-path) res)) (define db:get-test-id db:get-test-id-not-cached) ;; given a test-info record, patch in the latest data from the testdat.db file ;; found in the test run directory ;; ;; NOT USED ;; |
︙ | |||
1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 | 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 | + | (debug:print-info 11 "Sent " zdat ", received " rawdat) (set! tmp (db:string->obj rawdat)) (vector-ref tmp 2)))) ((zmq) (handle-exceptions exn (begin (debug:print-info 0 "cdb:client-call timeout or error. Trying again in 5 seconds") (thread-sleep! 5) (if (> numretries 0)(apply cdb:client-call serverdat qtype immediate (- numretries 1) params))) (let* ((push-socket (vector-ref serverdat 0)) (sub-socket (vector-ref serverdat 1)) (client-sig (client:get-signature)) (query-sig (message-digest-string (md5-primitive) (conc qtype immediate params))) (zdat (db:obj->string (vector client-sig qtype immediate query-sig params (current-seconds)))) ;; (with-output-to-string (lambda ()(serialize params)))) |
︙ | |||
1242 1243 1244 1245 1246 1247 1248 | 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 | - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + + - + | ;; this should match (client:get-signature) ;; we will need to process "all" messages here some day (receive-message* sub-socket) ;; now get the actual message (let ((myres (db:string->obj (receive-message* sub-socket)))) (if (equal? query-sig (vector-ref myres 1)) (set! res (vector-ref myres 2)) |
︙ | |||
1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 | 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 | + | (set! res (cons (vector id itempath state status run_duration logf comment) res))) db "SELECT id,item_path,state,status,run_duration,final_logf,comment FROM tests WHERE run_id=? AND testname=? AND item_path != '';" run-id test-name) res)) ;; Rollup the pass/fail counts from itemized tests into fail_count and pass_count ;; NOTE: Is this duplicating (db:test-data-rollup db test-id status) ???? (define (db:roll-up-pass-fail-counts db run-id test-name item-path status) ;; (cdb:flush-queue *runremote*) (if (and (not (equal? item-path "")) (member status '("PASS" "WARN" "FAIL" "WAIVED" "RUNNING" "CHECK" "SKIP"))) (begin (sqlite3:execute db |
︙ | |||
1621 1622 1623 1624 1625 1626 1627 | 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 | - + - + | (define (db:testmeta-update-field db testname field value) (sqlite3:execute db (conc "UPDATE test_meta SET " field "=? WHERE testname=?;") value testname)) ;;====================================================================== ;; T E S T D A T A ;;====================================================================== |
︙ | |||
1677 1678 1679 1680 1681 1682 1683 | 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 | - - + + + - - - + + - + - + - + - - + + | ((<) (if (< value expected) "pass" "fail")) ((>=) (if (>= value expected) "pass" "fail")) ((<=) (if (<= value expected) "pass" "fail")) (else (conc "ERROR: bad tol comparator " tol)))))) (debug:print 4 "AFTER2: category: " category " variable: " variable " value: " value ", expected: " expected " tol: " tol " units: " units " status: " status " comment: " comment) (sqlite3:execute tdb "INSERT OR REPLACE INTO test_data (test_id,category,variable,value,expected,tol,units,comment,status,type) VALUES (?,?,?,?,?,?,?,?,?,?);" |
︙ | |||
1766 1767 1768 1769 1770 1771 1772 | 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 | - - + + - - + + | ;; S T E P S ;;====================================================================== (define (db:step-get-time-as-string vec) (seconds->time-string (db:step-get-event_time vec))) ;; db-get-test-steps-for-run |
︙ | |||
1843 1844 1845 1846 1847 1848 1849 | 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 | - - + + | ((eq? (db:step-get-event_time a)(db:step-get-event_time b)) (< (db:step-get-id a) (db:step-get-id b))) (else #f))))) res))) ;; get a pretty table to summarize steps ;; |
︙ | |||
1985 1986 1987 1988 1989 1990 1991 | 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 | - + - + | ;; if the test is not found then clearly the waiton is not met... ;; (if (not ever-seen)(set! result (cons waitontest-name result))))) (if (not ever-seen) (set! result (append (if (null? tests)(list waitontest-name) tests) result))))) waitons) (delete-duplicates result)))) |
︙ |
Modified http-transport.scm from [ebc2f5cd40] to [b400e69336].
︙ | |||
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | + + + + + + + + + + + + - + + | ;;====================================================================== ;; Call this to start the actual server ;; (define *db:process-queue-mutex* (make-mutex)) (define (server:get-best-guess-address hostname) (let ((res #f)) (for-each (lambda (adr) (if (not (eq? (u8vector-ref adr 0) 127)) (set! res adr))) (vector->list (hostinfo-addresses (hostname->hostinfo hostname)))) (string-intersperse (map number->string (u8vector->list (if res res (hostname->ip hostname)))) "."))) (define (http-transport:run hostn) (debug:print 2 "Attempting to start the server ...") (if (not *toppath*) (if (not (setup-for-run)) (begin (debug:print 0 "ERROR: cannot find megatest.config, cannot start server, exiting") (exit)))) (let* (;; (iface (if (string=? "-" hostn) ;; #f ;; (get-host-name) ;; hostn)) (db #f) ;; (open-db)) ;; we don't want the server to be opening and closing the db unnecesarily (hostname (get-host-name)) (ipaddrstr (let ((ipstr (if (string=? "-" hostn) |
︙ | |||
132 133 134 135 136 137 138 | 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 | + - + | ;; (open-run-close tasks:remove-server-records tasks:open-db) (open-run-close tasks:server-register tasks:open-db (current-process-id) ipaddrstr portnum 0 'live 'http) (print "INFO: Trying to start server on " ipaddrstr ":" portnum) ;; This starts the spiffy server ;; NEED WAY TO SET IP TO #f TO BIND ALL |
︙ |
Modified launch.scm from [ea08ddd374] to [d2d20103be].
︙ | |||
51 52 53 54 55 56 57 | 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | - + - + | '()))) (define (launch:execute encoded-cmd) (let* ((cmdinfo (read (open-input-string (base64:base64-decode encoded-cmd))))) (setenv "MT_CMDINFO" encoded-cmd) (if (list? cmdinfo) ;; ((testpath /tmp/mrwellan/jazzmind/src/example_run/tests/sqlitespeed) ;; (test-name sqlitespeed) (runscript runscript.rb) (db-host localhost) (run-id 1)) |
︙ | |||
131 132 133 134 135 136 137 | 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 | - + | (open-run-close set-run-config-vars #f run-id keys keyvals) ;; environment overrides are done *before* the remaining critical envars. (alist->env-vars env-ovrd) (set-megatest-env-vars run-id) (set-item-env-vars itemdat) (save-environment-as-files "megatest") ;; open-run-close not needed for test-set-meta-info |
︙ | |||
206 207 208 209 210 211 212 | 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 | - + - + | ;; (set! script (conc script "source " prev-env)))) ;; call the command using mt_ezstep (set! script (conc "mt_ezstep " stepname " " (if prevstep prevstep "-") " " stepcmd)) (debug:print 4 "script: " script) ;; DO NOT remote |
︙ | |||
274 275 276 277 278 279 280 | 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 | - + | (current-seconds) start-seconds))))) (kill-tries 0)) (let loop ((minutes (calc-minutes))) (begin (set! kill-job? (test-get-kill-request test-id)) ;; run-id test-name itemdat)) ;; open-run-close not needed for test-set-meta-info |
︙ |
Modified megatest-version.scm from [803199c761] to [b40c0d60de].
1 2 3 4 5 | 1 2 3 4 5 6 7 | - + | ;; Always use two digit decimal ;; 1.01, 1.02...1.10,1.11 ... 1.99,2.00.. (declare (unit megatest-version)) |
Modified megatest.scm from [541d0bde0d] to [df389d567a].
︙ | |||
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | + + + + | (declare (uses margs)) (declare (uses runs)) (declare (uses launch)) (declare (uses server)) (declare (uses client)) (declare (uses tests)) (declare (uses genexample)) (declare (uses daemon)) (define *db* #f) ;; this is only for the repl, do not use in general!!!! (include "common_records.scm") (include "key_records.scm") (include "db_records.scm") (include "megatest-fossil-hash.scm") ;; (use trace dot-locking) ;; (trace ;; db:teststep-set-status! ;; db:open-test-db-by-test-id ;; db:test-get-rundir-from-test-id ;; cdb:tests-register-test ;; cdb:tests-update-uname-host ;; cdb:tests-update-run-duration ;; ;; cdb:client-call ;; ;; cdb:remote-run ;; ) ;; cdb:test-set-status-state |
︙ | |||
120 121 122 123 124 125 126 127 128 129 130 131 132 133 | 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 | + | patterns. -list-disks : list the disks available for storing runs -list-targets : list the targets in runconfigs.config -list-db-targets : list the target combinations used in the db -show-config : dump the internal representation of the megatest.config file -show-runconfig : dump the internal representation of the runconfigs.config file -dumpmode json : dump in json format instead of sexpr -show-cmdinfo : dump the command info for a test (run in test environment) Misc -rebuild-db : bring the database schema up to date -update-meta : update the tests metadata for all tests -env2file fname : write the environment to fname.csh and fname.sh -setvars VAR1=val1,VAR2=val2 : Add environment variables to a run NB// these are overwritten by values set in config files. |
︙ | |||
232 233 234 235 236 237 238 239 240 241 242 243 244 245 | 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 | + | "-list-servers" ;; mist queries "-list-disks" "-list-targets" "-list-db-targets" "-show-runconfig" "-show-config" "-show-cmdinfo" ;; queries "-test-paths" ;; get path(s) to a test, ordered by youngest first "-runall" ;; run all tests "-remove-runs" "-rebuild-db" "-rollup" |
︙ | |||
315 316 317 318 319 320 321 | 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 | - + + + + - - - - - + + + + + + + + + + + - | (if (not (null? (lset-intersection equal? (hash-table-keys args:arg-hash) '("-runtests" "-list-runs" "-rollup" "-remove-runs" "-lock" "-unlock" "-update-meta" "-extract-ods")))) (if (setup-for-run) |
︙ | |||
372 373 374 375 376 377 378 | 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 | - + - | (if (equal? id sid) (begin (debug:print-info 0 "Attempting to stop server with pid " pid) (tasks:kill-server status hostname pullport pid transport))))) servers) (debug:print-info 1 "Done with listservers") (set! *didsomething* #t) |
︙ | |||
425 426 427 428 429 430 431 432 433 434 435 436 437 438 | 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 | + + + + + + + | ((not (args:get-arg "-dumpmode")) (pp (hash-table->alist data))) ((string=? (args:get-arg "-dumpmode") "json") (json-write data)) (else (debug:print 0 "ERROR: -dumpmode of " (args:get-arg "-dumpmode") " not recognised"))) (set! *didsomething* #t))) (if (args:get-arg "-show-cmdinfo") (let ((data (read (open-input-string (base64:base64-decode (getenv "MT_CMDINFO")))))) (if (equal? (args:get-arg "-dumpmode") "json") (json-write data) (pp data)) (set! *didsomething* #t))) ;;====================================================================== ;; Remove old run(s) ;;====================================================================== ;; since several actions can be specified on the command line the removal ;; is done first |
︙ | |||
791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 | 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 | + - + - + + | (testpath (assoc/default 'testpath cmdinfo)) (test-name (assoc/default 'test-name cmdinfo)) (runscript (assoc/default 'runscript cmdinfo)) (db-host (assoc/default 'db-host cmdinfo)) (run-id (assoc/default 'run-id cmdinfo)) (test-id (assoc/default 'test-id cmdinfo)) (itemdat (assoc/default 'itemdat cmdinfo)) (work-area (assoc/default 'work-area cmdinfo)) (db #f)) (change-directory testpath) ;; (set! *runremote* runremote) (set! *transport-type* (string->symbol transport)) (if (not (setup-for-run)) (begin (debug:print 0 "Failed to setup, exiting") (exit 1))) (if (and state status) ;; DO NOT remote run, makes calls to the testdat.db test db. |
︙ | |||
839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 | 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 | + - + | (testpath (assoc/default 'testpath cmdinfo)) (test-name (assoc/default 'test-name cmdinfo)) (runscript (assoc/default 'runscript cmdinfo)) (db-host (assoc/default 'db-host cmdinfo)) (run-id (assoc/default 'run-id cmdinfo)) (test-id (assoc/default 'test-id cmdinfo)) (itemdat (assoc/default 'itemdat cmdinfo)) (work-area (assoc/default 'work-area cmdinfo)) (db #f) ;; (open-db)) (state (args:get-arg ":state")) (status (args:get-arg ":status"))) (change-directory testpath) ;; (set! *runremote* runremote) (set! *transport-type* (string->symbol transport)) (if (not (setup-for-run)) (begin (debug:print 0 "Failed to setup, exiting") (exit 1))) ;; can setup as client for server mode now ;; (client:setup) (if (args:get-arg "-load-test-data") ;; has sub commands that are rdb: ;; DO NOT put this one into either cdb:remote-run or open-run-close |
︙ | |||
888 889 890 891 892 893 894 | 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 | - + - + | ((zsh bash sh ash) "2>&1 >") (else ">&"))) (fullcmd (conc "(" (string-intersperse (cons cmd params) " ") ") " redir " " logfile))) ;; mark the start of the test ;; DO NOT run remote |
︙ | |||
935 936 937 938 939 940 941 | 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 | - + | (begin (debug:print 0 "ERROR: You must specify :state and :status with every call to -test-status\n" help) ;; (sqlite3:finalize! db) (exit 6))) (let* ((msg (args:get-arg "-m")) (numoth (length (hash-table-keys otherdata)))) ;; Convert to rpc inside the tests:test-set-status! call, not here |
︙ |
Modified runs.scm from [bb46712ad5] to [e0752394d2].
︙ | |||
442 443 444 445 446 447 448 | 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 | + + - + | (eq? (hash-table-ref/default test-registery x #f) 'start)) (hash-table-keys test-registery)) ", ")) (thread-sleep! 0.1) (loop hed tal reruns)) ((not have-resources) ;; simply try again after waiting a second (debug:print-info 1 "no resources to run new tests, waiting ...") ;; Have gone back and forth on this but db starvation is an issue. ;; wait one second before looking again to run jobs. |
︙ |
Modified tests.scm from [db49b893d4] to [02eaa01dcc].
︙ | |||
242 243 244 245 246 247 248 | 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 | - + | (loop (car tal)(cdr tal))) #f)))))) (pop-directory) result))) ;; Do not rpc this one, do the underlying calls!!! |
︙ | |||
288 289 290 291 292 293 294 | 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 | - + | ;; update the primary record IF state AND status are defined (if (and state status) (cdb:test-set-status-state *runremote* test-id real-status state (if waived waived comment))) ;; if status is "AUTO" then call rollup (note, this one modifies data in test ;; run area, it does remote calls under the hood. (if (and test-id state status (equal? status "AUTO")) |
︙ | |||
322 323 324 325 326 327 328 | 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 | + - + | variable "," value "," expected "," tol "," units "," dcomment ",," ;; extra comma for status type ))) ;; This was run remote, don't think that makes sense. |
︙ | |||
569 570 571 572 573 574 575 | 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 | - + - + - + + | ;; (sqlite3:execute db "UPDATE tests SET run_duration=? WHERE id=?;" minutes test-id)) (if (eq? num-records 0) (cdb:tests-update-uname-host *runremote* test-id uname hostname)) ;;(sqlite3:execute db "UPDATE tests SET uname=?,host=? WHERE id=?;" uname hostname test-id)) ;;(sqlite3:finalize! db)) ) |
Modified tests/fdktestqa/testqa/megatest.config from [c04381f809] to [5cdeb94e1e].
1 | 1 2 3 4 5 6 7 | - + + + | [setup] |
Added tests/fslsync/megatest.config version [6aa39fa6b6].
|
Added tests/fslsync/runconfigs.config version [ba5882eb76].
|
Added tests/fslsync/sites.dat.template version [7e1f28d5f7].
1 | + | site1 |
Added tests/fslsync/tests/setup/mkdirs.logpro version [e453c6e331].
|
Added tests/fslsync/tests/setup/mkdirs.sh version [5b853fc39d].
|
Added tests/fslsync/tests/setup/seedcache.logpro version [e453c6e331].
|
Added tests/fslsync/tests/setup/seedcache.sh version [a18d5e99f5].
|
Added tests/fslsync/tests/setup/testconfig version [3c9aac8422].
|
Added tests/fslsync/tests/sync/fsync.logpro version [e453c6e331].
|
Added tests/fslsync/tests/sync/fsync.sh version [bf21db2120].
|
Added tests/fslsync/tests/sync/rsync.logpro version [e453c6e331].
|
Added tests/fslsync/tests/sync/rsync.sh version [035d9817f0].
|
Added tests/fslsync/tests/sync/testconfig version [518d98790f].
|
Modified tests/fullrun/config/mt_include_1.config from [4d0b8a5d7e] to [8ae9c17ecf].
1 2 | 1 2 3 4 5 6 7 8 9 10 | - + | [setup] # exectutable /path/to/megatest |
︙ |