Overview
Comment: | pgdb selective sync, copy last update time in tables as is and use the smallest of thoes as area update |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | v1.65 |
Files: | files | file ages | folders |
SHA1: |
ac0148ceafdaced62f5dfa03eaabb657 |
User & Date: | pjhatwal on 2019-01-25 13:21:05 |
Other Links: | branch diff | manifest | tags |
Context
2019-01-25
| ||
13:28 | changed prints to debug:print-info check-in: d26d705d6d user: pjhatwal tags: v1.65 | |
13:21 | pgdb selective sync, copy last update time in tables as is and use the smallest of thoes as area update check-in: ac0148ceaf user: pjhatwal tags: v1.65 | |
2019-01-23
| ||
15:16 | fixed bug check-in: 430f66247c user: bjbarcla tags: v1.65, v1.6519 | |
Changes
Modified api.scm from [6b1deaf36f] to [1541791de9].
︙ | |||
76 77 78 79 80 81 82 | 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 | - + + | read-test-data read-test-data* login tasks-get-last testmeta-get-record have-incompletes? synchash-get |
︙ | |||
314 315 316 317 318 319 320 | 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 | - - + + | ((login) (apply db:login dbstruct params)) ((general-call) (let ((stmtname (car params)) (run-id (cadr params)) (realparams (cddr params))) (db:general-call dbstruct stmtname realparams))) ((sdb-qry) (apply sdb:qry params)) ((ping) (current-process-id)) |
︙ |
Modified cgisetup/models/pgdb.scm from [63b07b285d] to [77a1401512].
︙ | |||
135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 | 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 | + + + + + + + - + - + - + - + - - - + + + + + + + + + - + - - - + + + - + - + - + + + + + + + - + - - - + + + - + - + - + + + + + + + + - + - - + + - + - + - + - + | ;; given a target spec id, target and run-name return the run-id ;; if no run found return #f ;; (define (pgdb:get-run-id dbh spec-id target run-name area-id) (dbi:get-one dbh "SELECT id FROM runs WHERE ttype_id=? AND target=? AND run_name=? and area_id=?;" spec-id target run-name area-id)) ;; given a target spec id, target and run-name return the run-id ;; if no run found return #f ;; (define (pgdb:get-run-last-update dbh id ) (dbi:get-one dbh "SELECT last_update FROM runs WHERE id=?;" id)) ;; given a run-id return all the run info ;; (define (pgdb:get-run-info dbh run-id ) ;; to join ttype or not? (dbi:get-one-row dbh ;; 0 1 2 3 4 5 6 7 8 9 10 11 12 "SELECT id,target,ttype_id,run_name,state,status,owner,event_time,comment,fail_count,pass_count,last_update,area_id FROM runs WHERE id=? ;" run-id )) ;; refresh the data in a run record ;; |
︙ |
Modified db.scm from [911f2e610c] to [bac3051f9d].
︙ | |||
2530 2531 2532 2533 2534 2535 2536 | 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 | - + | ;; this is inconsistent with get-runs but it makes some sense. ;; (define (db:get-run-info dbstruct run-id) ;;(if (hash-table-ref/default *run-info-cache* run-id #f) ;; (hash-table-ref *run-info-cache* run-id) (let* ((res (vector #f #f #f #f)) (keys (db:get-keys dbstruct)) |
︙ | |||
3070 3071 3072 3073 3074 3075 3076 | 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 | - + | db "SELECT attemptnum FROM tests WHERE id=?;" #f test-id)))) (define db:test-record-fields '("id" "run_id" "testname" "state" "status" "event_time" "host" "cpuload" "diskfree" "uname" "rundir" "item_path" |
︙ | |||
3175 3176 3177 3178 3179 3180 3181 | 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 | - + - + | (db:with-db dbstruct #f ;; run-id #f (lambda (db) (let ((res #f)) (sqlite3:for-each-row ;; attemptnum added to hold pid of top process (not Megatest) controlling a test |
︙ | |||
3302 3303 3304 3305 3306 3307 3308 | 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 | - + - - + + - + | (define (db:get-steps-info-by-id dbstruct test-step-id) (db:with-db dbstruct #f #f (lambda (db) |
︙ | |||
3336 3337 3338 3339 3340 3341 3342 | 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 | - + - - + + - + | (define (db:get-data-info-by-id dbstruct test-data-id) (db:with-db dbstruct #f #f (lambda (db) |
︙ | |||
3526 3527 3528 3529 3530 3531 3532 | 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 | - + | (sqlite3:for-each-row (lambda (id test_id category variable value expected tol units comment status type) (set! res (cons (vector id test_id category variable value expected tol units comment status type) res))) db "SELECT id,test_id,category,variable,value,expected,tol,units,comment,status,type FROM test_data WHERE test_id=? AND category LIKE ? ORDER BY category,variable;" test-id categorypatt) (reverse res))))) |
︙ | |||
4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 | 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 | + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + - + | (set! result (append (if (null? waiton-tests) (list waitontest-name) waiton-tests) result))) ;; appends the string if the full record is not available ;; if the test is not found then clearly the waiton is not met... ;; (if (not ever-seen)(set! result (cons waitontest-name result))))) ((not ever-seen) (set! result (append (if (null? waiton-tests)(list waitontest-name) waiton-tests) result)))))) waitons) (delete-duplicates result))))) ;;====================================================================== ;; To sync individual run ;;====================================================================== (define (db:get-run-record-ids dbstruct target run keynames test-patt) (let ((backcons (lambda (lst item)(cons item lst)))) (db:with-db dbstruct #f #f (lambda (db) (let* ((keystr (string-intersperse (map (lambda (key val) (conc key " like '" val "'")) keynames (string-split target "/")) " AND ")) (run-qry (conc "SELECT id FROM runs WHERE " keystr " and runname='" run"'")) (test-qry (conc "SELECT id FROM tests WHERE run_id in (" run-qry ") and testname like '" test-patt "'"))) ;(print run-qry) `((runs . ,(fold-row backcons '() db run-qry)) (tests . ,(fold-row backcons '() db test-qry)) (test_steps . ,(fold-row backcons '() db (conc "SELECT id FROM test_steps WHERE test_id in (" test-qry ")"))) (test_data . ,(fold-row backcons '() db (conc "SELECT id FROM test_data WHERE test_id in (" test-qry ")" ))) )))))) ;;====================================================================== ;; Just for sync, procedures to make sync easy ;;====================================================================== ;; get an alist of record ids changed since time since-time ;; '((runs . (1 2 3 ...))(steps . (5 6 7 ...) ...)) ;; (define (db:get-changed-record-ids dbstruct since-time) ;; no transaction, allow the db to be accessed between the big queries (let ((backcons (lambda (lst item)(cons item lst)))) (db:with-db dbstruct #f #f (lambda (db) |
︙ |
Modified db_records.scm from [706558dc8a] to [37c233f08b].
︙ | |||
100 101 102 103 104 105 106 107 108 109 110 111 112 113 | 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 | + | (define-inline (db:test-get-rundir vec) (vector-ref vec 10)) (define-inline (db:test-get-item-path vec) (vector-ref vec 11)) (define-inline (db:test-get-run_duration vec) (vector-ref vec 12)) (define-inline (db:test-get-final_logf vec) (vector-ref vec 13)) (define-inline (db:test-get-comment vec) (vector-ref vec 14)) (define-inline (db:test-get-process_id vec) (vector-ref vec 16)) (define-inline (db:test-get-archived vec) (vector-ref vec 17)) (define-inline (db:test-get-last_update vec) (vector-ref vec 18)) ;; (define-inline (db:test-get-pass_count vec) (vector-ref vec 15)) ;; (define-inline (db:test-get-fail_count vec) (vector-ref vec 16)) (define-inline (db:test-get-fullname vec) (conc (db:test-get-testname vec) "/" (db:test-get-item-path vec))) ;; replace runs:make-full-test-name with this routine |
︙ | |||
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 | 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 | + - + + - + | (define-inline (db:test-data-get-value vec) (vector-ref vec 4)) (define-inline (db:test-data-get-expected vec) (vector-ref vec 5)) (define-inline (db:test-data-get-tol vec) (vector-ref vec 6)) (define-inline (db:test-data-get-units vec) (vector-ref vec 7)) (define-inline (db:test-data-get-comment vec) (vector-ref vec 8)) (define-inline (db:test-data-get-status vec) (vector-ref vec 9)) (define-inline (db:test-data-get-type vec) (vector-ref vec 10)) (define-inline (db:test-data-get-last_update vec) (vector-ref vec 11)) (define-inline (db:test-data-set-id! vec val)(vector-set! vec 0 val)) (define-inline (db:test-data-set-test_id! vec val)(vector-set! vec 1 val)) (define-inline (db:test-data-set-category! vec val)(vector-set! vec 2 val)) (define-inline (db:test-data-set-variable! vec val)(vector-set! vec 3 val)) (define-inline (db:test-data-set-value! vec val)(vector-set! vec 4 val)) (define-inline (db:test-data-set-expected! vec val)(vector-set! vec 5 val)) (define-inline (db:test-data-set-tol! vec val)(vector-set! vec 6 val)) (define-inline (db:test-data-set-units! vec val)(vector-set! vec 7 val)) (define-inline (db:test-data-set-comment! vec val)(vector-set! vec 8 val)) (define-inline (db:test-data-set-status! vec val)(vector-set! vec 9 val)) (define-inline (db:test-data-set-type! vec val)(vector-set! vec 10 val)) ;;====================================================================== ;; S T E P S ;;====================================================================== ;; Run steps ;; make-vector-record "Run steps" db step id test_id stepname step_complete step_pass event_time |
︙ |
Modified megatest-version.scm from [b0f8f20fa8] to [1094c4a239].
︙ | |||
16 17 18 19 20 21 22 | 16 17 18 19 20 21 22 23 | - + | ;; along with Megatest. If not, see <http://www.gnu.org/licenses/>. ;; Always use two or four digit decimal ;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00.. (declare (unit megatest-version)) |
Modified mt-pg.sql from [1ebd25a5ad] to [1d0cdb4bfc].
︙ | |||
40 41 42 43 44 45 46 | 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 | - - - + + + + + + + | DROP TABLE IF EXISTS archives; DROP TABLE IF EXISTS session_vars; DROP TABLE IF EXISTS sessions; DROP TABLE IF EXISTS tags; DROP TABLE IF EXISTS users; DROP TABLE IF EXISTS webviews; DROP TABLE IF EXISTS area_tags; |
︙ | |||
260 261 262 263 264 265 266 267 268 269 270 271 272 273 | 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 | + | CREATE TABLE IF NOT EXISTS users( id SERIAL PRIMARY KEY , username TEXT NOT NULL, fullname TEXT NOT NULL, email TEXT NOT NULL, default_view TEXT default '', is_admin boolean default 'f', deleted INTEGER default 0 ); CREATE TABLE IF NOT EXISTS base_paths( id SERIAL PRIMARY KEY , path TEXT NOT NULL, deleted INTEGER default 0 |
︙ | |||
294 295 296 297 298 299 300 301 302 303 304 305 306 307 | 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 | + | owner_id INTEGER NOT NULL, name TEXT NOT NULL, ttype_id INTEGER DEFAULT 0, view_specifics TEXT , col TEXT NOT NULL, row TEXT NOT NULL, public INTEGER DEFAULT 0, search_patt TEXT default '.*', deleted INTEGER default 0 ); CREATE TABLE IF NOT EXISTS users_webviews( id SERIAL PRIMARY KEY , |
︙ | |||
320 321 322 323 324 325 326 327 328 329 330 331 332 333 | 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 | + - | result_file TEXT Default NULL, chksum TEXT ); CREATE TABLE IF NOT EXISTS cctrl_config( id SERIAL PRIMARY KEY , area_type Text, metadata text default '', cmd TEXT ); CREATE TABLE IF NOT EXISTS platforms( id SERIAL PRIMARY KEY , name Text ); |
Modified mtut.scm from [51190d4e0c] to [848d0d5914].
︙ | |||
1731 1732 1733 1734 1735 1736 1737 | 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 | - - + + | (system (conc "kill " pid)))) (begin (debug:print 0 *default-log-port* ctime " received " instr ) ;(nn-send rep "ok") (if (not (equal? instr "ping")) (begin (debug:print 0 *default-log-port* ctime " running \"" script " " instr "\"") |
︙ |
Modified rmt.scm from [e82f79a582] to [0a05f35135].
︙ | |||
463 464 465 466 467 468 469 470 471 472 473 474 475 476 | 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 | + + + | ;; add caching if qry is 'getid or 'getstr (rmt:send-receive 'sdb-qry run-id (list qry val))) ;; NOT COMPLETED (define (rmt:runtests user run-id testpatt params) (rmt:send-receive 'runtests run-id testpatt)) (define (rmt:get-run-record-ids target run keynames test-patt) (rmt:send-receive 'get-run-record-ids #f (list target run keynames test-patt))) (define (rmt:get-changed-record-ids since-time) (rmt:send-receive 'get-changed-record-ids #f (list since-time)) ) ;;====================================================================== ;; T E S T M E T A ;;====================================================================== |
︙ |
Modified tasks.scm from [358b0b74f6] to [bf64194ab9].
︙ | |||
736 737 738 739 740 741 742 | 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 | - + | (task:print-testtime test-times ",") (task:print-testtime test-times " "))))) ;; gets mtpg-run-id and syncs the record if different ;; |
︙ | |||
760 761 762 763 764 765 766 767 768 769 770 771 | 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 | + - + - - - - - + + + + + + + + - + - - - - - - - - - + + + + + + + + + - - + + + + + - + + + - - - + + + + + + - + + + - + - + - + + + + + + + - + - + + + - + - + - + + - - + + - - + + + + + - - - - - + + + + + + + | (db-contour (db:get-value-by-header row header "contour")) (contour (if (args:get-arg "-prepend-contour") (if (and db-contour (not (equal? db-contour "")) (string? db-contour )) (begin (debug:print-info 1 *default-log-port* "db-contour") db-contour) (args:get-arg "-contour")))) (last-update (db:get-value-by-header row header "last_update")) (keytarg (if (or (args:get-arg "-prepend-contour") (args:get-arg "-prefix-target")) (conc "MT_CONTOUR/MT_AREA/" (string-intersperse (rmt:get-keys) "/")) (string-intersperse (rmt:get-keys) "/"))) ;; e.g. version/iteration/platform (target (if (or (args:get-arg "-prepend-contour") (args:get-arg "-prefix-target")) (conc (or (args:get-arg "-prefix-target") (conc contour "/" (common:get-area-name) "/")) (rmt:get-target run-id)) (rmt:get-target run-id))) ;; e.g. v1.63/a3e1/ubuntu (spec-id (pgdb:get-ttype dbh keytarg)) |
︙ | |||
950 951 952 953 954 955 956 | 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 | - + - + - + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - + + + + - - - - - + + + + + + + + + + | exn (begin (debug:print-info 1 *default-log-port* ((condition-property-accessor 'exn 'message) exn)) #f) (if (not (pgdb:is-area-taged-with-a-tag dbh (vector-ref tag-info 0) (vector-ref area-info 0))) (pgdb:insert-area-tag dbh (vector-ref tag-info 0) (vector-ref area-info 0)))))) |