Megatest

Check-in [297a374249]
Login
Overview
Comment:Added back use of mutex for transactions
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v2.0001-matt-test-edits
Files: files | file ages | folders
SHA1: 297a374249962e547176b8787434d22119fb97ff
User & Date: mrwellan on 2022-02-09 09:56:16
Other Links: branch diff | manifest | tags
Context
2022-02-09
11:18
Added some expiration handling for run servers Closed-Leaf check-in: 1b4e76c0e0 user: mrwellan tags: v2.0001-matt-test-edits
09:56
Added back use of mutex for transactions check-in: 297a374249 user: mrwellan tags: v2.0001-matt-test-edits
08:02
Initialize placeholder record to correct length in db:get-run-info check-in: dbdd2c33cc user: mrwellan tags: v2.0001-matt-test-edits
Changes

Modified commonmod.scm from [6aaa47a003] to [bcea10d67c].

381
382
383
384
385
386
387
388

389
390
391
392
393
394
395
381
382
383
384
385
386
387

388
389
390
391
392
393
394
395







-
+







*db-write-access*
*db-last-sync*
*db-sync-in-progress*
*db-multi-sync-mutex*
*task-db*
*db-access-allowed*
*db-access-mutex*
*db-transaction-mutex*
;; *db-transaction-mutex*
*db-cache-path*
*db-with-db-mutex*
*db-api-call-time*
*didsomething*
*no-sync-db*
*my-signature*
*transport-type*
960
961
962
963
964
965
966

967
968
969
970
971
972

973
974
975
976
977
978
979
960
961
962
963
964
965
966
967
968
969
970
971
972

973
974
975
976
977
978
979
980







+





-
+







(define *db-stats-mutex*      (make-mutex))
;; db access
(define *db-last-access*      (current-seconds)) ;; last db access, used in server
(define *db-write-access*     #t)
;; db sync
(define *db-last-sync*        0)                 ;; last time the sync to megatest.db happened
(define *db-sync-in-progress* #f)                ;; if there is a sync in progress do not try to start another
;; multi-sync mutex used in both dbmod and launchmod
(define *db-multi-sync-mutex* (make-mutex))      ;; protect access to *db-sync-in-progress*, *db-last-sync*
;; task db
(define *task-db*             #f) ;; (vector db path-to-db)
(define *db-access-allowed*   #t) ;; flag to allow access
(define *db-access-mutex*     (make-mutex))
(define *db-transaction-mutex* (make-mutex))
;; (define *db-transaction-mutex* (make-mutex))
(define *db-cache-path*       #f)
(define *db-with-db-mutex*    (make-mutex))
(define *db-api-call-time*    (make-hash-table)) ;; hash of command => (list of times)
;; no sync db
(define *no-sync-db*          #f)

;; SERVER

Modified dbmod.scm from [0e76e5785b] to [0dbe869d65].

343
344
345
346
347
348
349
350



351
352
353
354
355
356
357
343
344
345
346
347
348
349

350
351
352
353
354
355
356
357
358
359







-
+
+
+







(defstruct dbr:dbdat
  (db          #f)    ;; should rename this to oddb for on disk db
  (inmem       #f)
  (last-sync   0)
  (last-write  (current-seconds))
  (run-id      #f)
  (fname       #f))
  

(define *db-transaction-mutex* (make-mutex))

;; Returns the dbdat for a particular dbfile inside the area
;;
(define (dbr:dbstruct-get-dbdat dbstruct dbfile)
  (hash-table-ref/default (dbr:dbstruct-dbdats dbstruct) dbfile #f))

(define (dbr:dbstruct-dbdat-put! dbstruct dbfile db)
  (hash-table-set! (dbr:dbstruct-dbdats dbstruct) dbfile db))
524
525
526
527
528
529
530

531
532
533
534
535
536
537









538
539
540
541
542
543
544
526
527
528
529
530
531
532
533







534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549







+
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+







	 (res (proc dbh dbfile)))
    ;; (sqlite3:finalize! dbh)
    res))

;; called before db is open?
;;
(define (db:get-iam-server-lock dbh dbfname host port)
  (mutex-lock! *db-transaction-mutex*)
  (sqlite3:with-transaction
   dbh
   (lambda ()
     (let* ((locker (db:get-locker dbh dbfname)))
       (if locker
	   locker
	   (db:take-lock dbh dbfname port))))))
  (let ((res (sqlite3:with-transaction
	      dbh
	      (lambda ()
		(let* ((locker (db:get-locker dbh dbfname)))
		  (if locker
		      locker
		      (db:take-lock dbh dbfname port)))))))
    (mutex-unlock! *db-transaction-mutex*)
    res))
	     
;; (exn sqlite3) 
(define (db:get-locker dbh dbfname)
  (condition-case
   (sqlite3:first-row dbh "SELECT owner_pid,owner_host,owner_port,event_time FROM locks WHERE lockname=?;" dbfname)
   (exn (sqlite3) #f)))

1007
1008
1009
1010
1011
1012
1013

1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034


1035
1036
1037
1038
1039
1040
1041
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039

1040
1041
1042
1043
1044
1045
1046
1047
1048







+




















-
+
+







				   #f)) 
	   (stmth  (sqlite3:prepare db full-ins)))
      ;; (db:delay-if-busy targdb) ;; NO WAITING
      (if (member "last_update" field-names)
	  (debug:print-info 3 *default-log-port* "is-trigger-dropped: " is-trigger-dropped)) 
      (for-each
       (lambda (fromdat-lst)
	 (mutex-lock! *db-transaction-mutex*)
	 (sqlite3:with-transaction
	  db
	  (lambda ()
	    (for-each ;; 
	     (lambda (fromrow)
	       (let* ((a    (vector-ref fromrow 0))
		      (curr (hash-table-ref/default todat a #f))
		      (same #t))
		 (let loop ((i 0))
		   (if (or (not curr)
			   (not (equal? (vector-ref fromrow i)(vector-ref curr i))))
		       (set! same #f))
		   (if (and same
			    (< i (- num-fields 1)))
		       (loop (+ i 1))))
		 (if (not same)
		     (begin
		       (debug:print 0 *default-log-port* "applying data "fromrow"to table "tablename", numrecs="numrecs)
		       (apply sqlite3:execute stmth (vector->list fromrow))
		       (hash-table-set! numrecs tablename (+ 1 (hash-table-ref/default numrecs tablename 0)))))))
	     fromdat-lst))))
	     fromdat-lst)))
	 (mutex-unlock! *db-transaction-mutex*))
       fromdats)
      (sqlite3:finalize! stmth)
      (if (member "last_update" field-names)
	  (db:create-trigger db tablename)))))

;; tbls is ( ("tablename" ( "field1" [#f|proc1] ) ( "field2" [#f|proc2] ) .... ) )
;; db's are sqlite3 handles
1513
1514
1515
1516
1517
1518
1519

1520
1521
1522
1523
1524
1525
1526
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534







+







  (assert *configinfo* "ERROR: db:initialize-db called before configfiles loaded. This is fatal.")
  (let* ((configdat (car *configinfo*))  ;; tut tut, global warning...
	 (keys     (keys:config-get-fields configdat))
	 (havekeys (> (length keys) 0))
	 (keystr   (keys->keystr keys))
	 (fieldstr (keys:make-key/field-string configdat))
	 #;(db       (dbr:dbdat-db dbdat)))
    (mutex-lock! *db-transaction-mutex*)
    (for-each (lambda (key)
		(let ((keyn key))
		  (if (member (string-downcase keyn)
			      (list "runname" "state" "status" "owner" "event_time" "comment" "fail_count"
				    "pass_count" "contour"))
		      (begin
			(print "ERROR: your key cannot be named " keyn " as this conflicts with the same named field in the runs table, you must remove your megatest.db and <linktree>/.db before trying again.")
1639
1640
1641
1642
1643
1644
1645

1646
1647
1648
1649
1650
1651
1652
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661







+







	(sqlite3:execute db "CREATE TABLE IF NOT EXISTS extradat (id INTEGER PRIMARY KEY, run_id INTEGER, key TEXT, val TEXT);")
	(sqlite3:execute db "CREATE TABLE IF NOT EXISTS metadat (id INTEGER PRIMARY KEY, var TEXT, val TEXT,
                                  CONSTRAINT metadat_constraint UNIQUE (var));")
	(sqlite3:execute db "CREATE TABLE IF NOT EXISTS access_log (id INTEGER PRIMARY KEY, user TEXT, accessed TIMESTAMP, args TEXT);")
	;; Must do this *after* running patch db !! No more. 
	;; cannot use db:set-var since it will deadlock, hardwire the code here
	(sqlite3:execute db "INSERT OR REPLACE INTO metadat (var,val) VALUES (?,?);" "MEGATEST_VERSION" (common:version-signature))
	(mutex-unlock! *db-transaction-mutex*)
	(debug:print-info 11 *default-log-port* "db:initialize END") ;; ))))

	;;======================================================================
	;; R U N   S P E C I F I C   D B 
	;;======================================================================
	
	(sqlite3:execute db "CREATE TABLE IF NOT EXISTS tests 
2769
2770
2771
2772
2773
2774
2775
2776

2777
2778
2779
2780
2781
2782
2783
2778
2779
2780
2781
2782
2783
2784

2785
2786
2787
2788
2789
2790
2791
2792







-
+







	"SELECT state,status,count(id) AS count FROM tests WHERE run_id=? AND NOT(uname='n/a' AND item_path='') GROUP BY state,status;"
	run-id))))

;; Update run_stats for given run_id
;; input data is a list (state status count)
;;
(define (db:update-run-stats dbstruct run-id stats)
  ;; (mutex-lock! *db-transaction-mutex*)
  (mutex-lock! *db-transaction-mutex*)
  (db:with-db
   dbstruct
   run-id
   #f

   (lambda (db)
     ;; remove previous data
2791
2792
2793
2794
2795
2796
2797
2798

2799
2800
2801
2802
2803
2804
2805
2800
2801
2802
2803
2804
2805
2806

2807
2808
2809
2810
2811
2812
2813
2814







-
+







		(for-each
		 (lambda (dat)
		   (sqlite3:execute stmt1 run-id (car dat)(cadr dat))
		   (apply sqlite3:execute stmt2 run-id dat))
		 stats)))))
       (sqlite3:finalize! stmt1)
       (sqlite3:finalize! stmt2)
       ;; (mutex-unlock! *db-transaction-mutex*)
       (mutex-unlock! *db-transaction-mutex*)
       res))))

(define (db:get-main-run-stats dbstruct run-id)
  (db:with-db
   dbstruct
   #f ;; this data comes from main
   #f