Megatest

Check-in [67b2c7d27d]
Login
☰Home Timeline Files Branches Tags Tickets Wiki Login
Overview
Comment:fix to sync runs independent to changes in tests
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.65
Files: files | file ages | folders
SHA1: 67b2c7d27d32bc86e5c85546911865e6d53cbbd2
User & Date: pjhatwal on 2018-08-23 18:33:25
Other Links: branch diff | manifest | tags
Context
2018-08-24
15:07
Fix cleanup-db to retain DELETED records for some time. default 30d. configurable via key delete-record-age in setup section check-in: 2a51bc4669 user: pjhatwal tags: v1.65
2018-08-23
18:33
fix to sync runs independent to changes in tests check-in: 67b2c7d27d user: pjhatwal tags: v1.65
2018-08-22
18:04
Updated load waiting so that it scales with the amount above the threshold that the load is check-in: d12eb7e791 user: jmoon18 tags: v1.65, v1.6514
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Modified db.scm from [f030aee63b] to [7a524ea7d5].

︙
517
518
519
520
521
522
523
524

525
526
527
528
529
530
531
532
533
534
535
536
537

538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554

555
556
557
558
559
560
561

517
518
519
520
521
522
523

524

525
526
527
528
529
530
531
532
533
534
535

536

537
538
539
540
541
542
543
544
545
546
547
548
549
550
551

552
553
554
555
556
557
558
559








-
+
-











-
+
-















-
+








	 '("id"             #f)
	 '("test_id"        #f)
	 '("stepname"       #f)
	 '("state"          #f)
	 '("status"         #f)
	 '("event_time"     #f)
	 '("comment"        #f)
	 '("logfile"        #f)
	 '("logfile"        #f))
         '("last_update"    #f))
   (list "test_data"
	 '("id"             #f)
	 '("test_id"        #f)
	 '("category"       #f)
	 '("variable"       #f)
	 '("value"          #f)
	 '("expected"       #f)
	 '("tol"            #f)
	 '("units"          #f)
	 '("comment"        #f)
	 '("status"         #f)
	 '("type"           #f)
	 '("type"           #f))))
         '("last_update"    #f))))

;; needs db to get keys, this is for syncing all tables
;;
(define (db:sync-main-list dbstruct)
  (let ((keys  (db:get-keys dbstruct)))
    (list
     (list "keys"
	   '("id"        #f)
	   '("fieldname" #f)
	   '("fieldtype" #f))
     (list "metadat" '("var" #f) '("val" #f))
     (append (list "runs" 
		   '("id"  #f))
	     (map (lambda (k)(list k #f))
		  (append keys
			  (list "runname" "state" "status" "owner" "event_time" "comment" "fail_count" "pass_count" "contour" "last_update"))))
			  (list "runname" "state" "status" "owner" "event_time" "comment" "fail_count" "pass_count" "contour" ))))
     (list "test_meta"
	   '("id"             #f)
	   '("testname"       #f)
	   '("owner"          #f)
	   '("description"    #f)
	   '("reviewed"       #f)
	   '("iterated"       #f)
︙

Modified tasks.scm from [192be37388] to [c5107e3a44].

︙
757
758
759
760
761
762
763
764

765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782

783

784
785
786

787
788
789
790
791
792
793

757
758
759
760
761
762
763

764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787

788
789
790
791
792
793
794
795








-
+


















+

+


-
+








	       (comment    (db:get-value-by-header row header "comment"))
	       (fail-count (db:get-value-by-header row header "fail_count"))
	       (pass-count (db:get-value-by-header row header "pass_count"))
               (db-contour (db:get-value-by-header row header "contour"))
	       (contour    (if (args:get-arg "-prepend-contour") 
                                 (if (and db-contour (not (equal? db-contour ""))) 
                                           (begin 
                                            (print "db-contour") 
                                            (debug:print-info 1 *default-log-port*  "db-contour") 
 						db-contour)
					    (args:get-arg "-contour"))))
	       (keytarg    (if (or (args:get-arg "-prepend-contour") (args:get-arg "-prefix-target"))
	       			(conc "MT_CONTOUR/MT_AREA/" (string-intersperse (rmt:get-keys) "/")) (string-intersperse (rmt:get-keys) "/"))) ;; e.g. version/iteration/platform
	       (target     (if (or (args:get-arg "-prepend-contour") (args:get-arg "-prefix-target")) 
	       			(conc (or (args:get-arg "-prefix-target") (conc contour "/" (common:get-area-name) "/")) (rmt:get-target run-id)) (rmt:get-target run-id)))                 ;; e.g. v1.63/a3e1/ubuntu
	       (spec-id    (pgdb:get-ttype dbh keytarg))
	       (new-run-id (pgdb:get-run-id dbh spec-id target run-name area-id))
	       ;; (area-id    (db:get-value-by-header row header "area_id)"))
	       )
              (if new-run-id
	      (begin ;; let ((run-record (pgdb:get-run-info dbh new-run-id))
		(hash-table-set! runs-ht run-id new-run-id)
		;; ensure key fields are up to date
		(pgdb:refresh-run-info
		 dbh
		 new-run-id
		 state status owner event-time comment fail-count pass-count area-id)
     (debug:print-info 1 *default-log-port* "Working on run-id " run-id " pgdb-id"  new-run-id )
		new-run-id)
      
	      (if (equal? state "deleted")
                 (begin 
                 (print "Warning: Run with id " run-id " was created after previous sync and deleted before the sync") #f)
                 (debug:print-info 1 *default-log-port*  "Warning: Run with id " run-id " was created after previous sync and deleted before the sync") #f)
               (if (handle-exceptions
		      exn
		      (begin (print-call-chain)
                              (print ((condition-property-accessor 'exn 'message) exn))     
			#f)
                     
                     (pgdb:insert-run
︙
818
819
820
821
822
823
824
825

826
827
828

829
830
831
832
833


834
835
836
837
838
839
840

820
821
822
823
824
825
826

827
828
829

830
831
832
833


834
835
836
837
838
839
840
841
842








-
+


-
+



-
-
+
+








                                  #f)))
    (if step-id
      (begin  
        (if pgdb-test-id
           (begin 
                (if  pgdb-step-id
                   (begin
                    (print "Updating existing test-step with test-id: " test-id " and step-id " step-id " pgdb test id: " pgdb-test-id " pgdb step id " pgdb-step-id )
                    (debug:print-info 1 *default-log-port*  "Updating existing test-step with test-id: " test-id " and step-id " step-id " pgdb test id: " pgdb-test-id " pgdb step id " pgdb-step-id )
                    (pgdb:update-test-step dbh pgdb-step-id pgdb-test-id stepname state status event_time comment logfile))
                    (begin
 		      (print "Inserting test-step with test-id: " test-id " and step-id " step-id  " pgdb test id: " pgdb-test-id)
 		      (debug:print-info 1 *default-log-port*  "Inserting test-step with test-id: " test-id " and step-id " step-id  " pgdb test id: " pgdb-test-id)
                      (pgdb:insert-test-step dbh pgdb-test-id stepname state status event_time comment logfile )
                      (set! pgdb-step-id  (pgdb:get-test-step-id dbh pgdb-test-id stepname state))))
                (hash-table-set! step-ht step-id pgdb-step-id ))
           (print "Error: Test not cashed")))
      (print "Error: Could not get test step info for step id " test-step-id ))))	;; this is a wierd senario need to debug      	
           (debug:print-info 1 *default-log-port*  "Error: Test not cashed")))
      (debug:print-info 1 *default-log-port*  "Error: Could not get test step info for step id " test-step-id ))))	;; this is a wierd senario need to debug      	
   test-step-ids)))

(define (tasks:sync-test-gen-data dbh cached-info test-data-ids)
  (let ((test-ht (hash-table-ref cached-info 'tests))
        (data-ht (hash-table-ref cached-info 'data)))
    (for-each
     (lambda (test-data-id)
︙
856
857
858
859
860
861
862
863

864
865
866

867
868
869
870
871
872
873
874
875
876
877
878
879
880
881

882
883

884
885
886
887
888
889
890

858
859
860
861
862
863
864

865
866
867

868
869
870
871
872
873
874
875
876
877
878
879
880
881
882

883
884

885
886
887
888
889
890
891
892








-
+


-
+














-
+

-
+








                                  #f)))
    (if data-id
      (begin
        (if pgdb-test-id
           (begin 
                (if  pgdb-data-id
                   (begin
                    (print "Updating existing test-data with test-id: " test-id " and  data-id " data-id " pgdb test id: " pgdb-test-id " pgdb data id " pgdb-data-id)
                    (debug:print-info 1 *default-log-port*  "Updating existing test-data with test-id: " test-id " and  data-id " data-id " pgdb test id: " pgdb-test-id " pgdb data id " pgdb-data-id)
                    (pgdb:update-test-data dbh pgdb-data-id pgdb-test-id  category variable value expected tol units comment status type))
                    (begin
 		      (print "Inserting test-data with test-id: " test-id " and data-id " data-id " pgdb test id: " pgdb-test-id)
 		      (debug:print-info 1 *default-log-port*  "Inserting test-data with test-id: " test-id " and data-id " data-id " pgdb test id: " pgdb-test-id)
                       (if (handle-exceptions
		      exn
		      (begin (print-call-chain)
                              (print ((condition-property-accessor 'exn 'message) exn))     
			#f)
                     
                    (pgdb:insert-test-data dbh pgdb-test-id category variable value expected tol units comment status type ))
		       ;(tasks:run-id->mtpg-run-id dbh cached-info run-id area-info)
                      (begin
                      ;(pgdb:insert-test-data dbh pgdb-test-id category variable value expected tol units comment status type )
                      (set! pgdb-data-id  (pgdb:get-test-data-id dbh pgdb-test-id  category variable)))
		  (exit))))
                (hash-table-set! data-ht data-id pgdb-data-id ))
             (begin
                 (print "Error: Test not in pgdb"))))
                 (debug:print-info 1 *default-log-port*  "Error: Test not in pgdb"))))

      (print "Error: Could not get test data info for data id " test-data-id ))))	;; this is a wierd senario need to debug      	
      (debug:print-info 1 *default-log-port*  "Error: Could not get test data info for data id " test-data-id ))))	;; this is a wierd senario need to debug      	
   test-data-ids)))



(define (tasks:sync-tests-data dbh cached-info test-ids area-info)
  (let ((test-ht (hash-table-ref cached-info 'tests)))
    (for-each
︙
917
918
919
920
921
922
923
924

925
926
927

928
929
930
931

932
933
934
935
936
937
938
939
940
941

942
943
944
945
946
947
948
949
950

951
952
953








954
955
956
957
958
959
960

919
920
921
922
923
924
925

926
927
928

929
930
931
932

933
934
935
936
937
938
939
940
941
942

943
944
945
946
947
948
949
950
951

952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970








-
+


-
+



-
+









-
+








-
+



+
+
+
+
+
+
+
+








	 ;; "id"           "run_id"        "testname"  "state"      "status"      "event_time"
	 ;; "host"         "cpuload"       "diskfree"  "uname"      "rundir"      "item_path"
	 ;; "run_duration" "final_logf"    "comment"   "shortdir"   "attemptnum"  "archived"
         (if pgdb-run-id
           (begin
	   (if pgdb-test-id ;; have a record
	     (begin ;; let ((key-name (conc run-id "/" test-name "/" item-path)))
	       (print "Updating existing test with run-id: " run-id " and test-id: " test-id " pgdb run id: " pgdb-run-id "  pgdb-test-id "  pgdb-test-id)
	       (debug:print-info 1 *default-log-port*  "Updating existing test with run-id: " run-id " and test-id: " test-id " pgdb run id: " pgdb-run-id "  pgdb-test-id "  pgdb-test-id)
	       (pgdb:update-test dbh pgdb-test-id pgdb-run-id test-name item-path state status host cpuload diskfree uname run-dir log-file run-duration comment event-time archived))
	     (begin 
                 (print "Inserting test with run-id: " run-id " and test-id: " test-id  " pgdb run id: " pgdb-run-id)
                 (debug:print-info 1 *default-log-port*  "Inserting test with run-id: " run-id " and test-id: " test-id  " pgdb run id: " pgdb-run-id)
                (pgdb:insert-test dbh pgdb-run-id test-name item-path state status host cpuload diskfree uname run-dir log-file run-duration comment event-time archived)
                (set! pgdb-test-id (pgdb:get-test-id dbh pgdb-run-id test-name item-path))))
               (hash-table-set! test-ht test-id pgdb-test-id))
              (print "WARNING: Skipping run with run-id:" run-id ". This run was created after privious sync and removed before this sync."))))
              (debug:print-info 1 *default-log-port*  "WARNING: Skipping run with run-id:" run-id ". This run was created after privious sync and removed before this sync."))))
     test-ids)))

(define (task:add-area-tag dbh area-info tag) 
  (let* ((tag-info (pgdb:get-tag-info-by-name dbh tag)))
   (if (not tag-info)
     (begin   
     (if (handle-exceptions
	   exn
	   (begin 
               (print ((condition-property-accessor 'exn 'message) exn))     
               (debug:print-info 1 *default-log-port*  ((condition-property-accessor 'exn 'message) exn))     
	   #f)
	   (pgdb:insert-tag  dbh   tag))
                       (set! tag-info (pgdb:get-tag-info-by-name dbh tag))
		  #f)))
     ;;add to area_tags
     (handle-exceptions
	   exn
	   (begin 
               (print ((condition-property-accessor 'exn 'message) exn))     
               (debug:print-info 1 *default-log-port*  ((condition-property-accessor 'exn 'message) exn))     
	   #f)
           (if (not (pgdb:is-area-taged-with-a-tag dbh (vector-ref tag-info 0)  (vector-ref area-info 0)))  
	   (pgdb:insert-area-tag  dbh   (vector-ref tag-info 0)  (vector-ref area-info 0))))))

(define (tasks:sync-run-data dbh cached-info run-ids area-info) 
  (for-each
     (lambda (run-id)
      (debug:print-info 1 *default-log-port*   "Check if run with " run-id " needs to be synced" )
       (tasks:run-id->mtpg-run-id dbh cached-info run-id area-info))
run-ids))


;; get runs changed since last sync
;; (define (tasks:sync-test-data dbh cached-info area-info)
;;   (let* ((

(define (tasks:sync-to-postgres configdat dest)
  (let* ((dbh         (pgdb:open configdat dbname: dest))
︙
969
970
971
972
973
974
975
976

977
978
979
980
981
982
983
984
985
986
987
988
989
990

991
992
993
994



995
996
997

998
999
1000
1001
1002
1003

979
980
981
982
983
984
985

986
987
988





989
990
991
992
993
994

995
996
997


998
999
1000
1001
1002

1003
1004
1005
1006
1007
1008
1009








-
+


-
-
-
-
-






-
+


-
-
+
+
+


-
+







	(let* ((last-sync-time (vector-ref area-info 3))
	       (changed        (rmt:get-changed-record-ids last-sync-time))
	       (run-ids        (alist-ref 'runs       changed))
	       (test-ids       (alist-ref 'tests      changed))
	       (test-step-ids  (alist-ref 'test_steps changed))
	       (test-data-ids  (alist-ref 'test_data  changed))
	       (run-stat-ids   (alist-ref 'run_stats  changed))
               (area-tag    (if (args:get-arg "-area-tag") 
         (area-tag    (if (args:get-arg "-area-tag") 
                                 (args:get-arg "-area-tag")
                                  "")))
     ;(print "last-sync-time " last-sync-time)
  
	  ;(print "test-ids: " test-ids)
    ;(print "--------------------------------------")
    ;(print "run-ids: " run-ids)  
           (if (and (equal? area-tag "") (not (pgdb:is-area-taged dbh (vector-ref area-info 0))))
            (set! area-tag *default-area-tag*)) 
           (if (not (equal? area-tag "")) 
             (task:add-area-tag dbh area-info area-tag)) 
	  (if (not (null? test-ids))
	      (begin
		(print "Syncing " (length test-step-ids) " changed tests")
		(debug:print-info 1 *default-log-port*  "Syncing " (length test-step-ids) " changed tests")
                ;;Assumption here is that if test-step or test data is changed then the test last update time is changed 
                ;; not syncing run stats at this time as they can be derived from tests table.
		(tasks:sync-tests-data dbh cached-info test-ids area-info)
                ;(exit)   
		            (tasks:sync-tests-data dbh cached-info test-ids area-info)
                ;(exit)  
                (tasks:sync-run-data dbh cached-info run-ids area-info) 
                (tasks:sync-test-steps dbh cached-info test-step-ids)
                (tasks:sync-test-gen-data dbh cached-info test-data-ids)))
	  (pgdb:write-sync-time dbh area-info start))
	  (pgdb:write-sync-time dbh area-info (- start 1)))
	(if (tasks:set-area dbh configdat)
	    (tasks:sync-to-postgres configdat dest)
	    (begin
	      (debug:print 0 *default-log-port* "ERROR: unable to create an area record")
	      #f)))))

This page was generated in about 0.113s by Fossil 2.24 [452f8e1ffc] 2024-03-26 15:36:00