Megatest

Check-in [3b54f22608]
Login
Overview
Comment:Added suicide mode to db:with-db for development. It has been very hard to isolate threaded crashes under the tcp-server.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.80
Files: files | file ages | folders
SHA1: 3b54f22608062d635e43c3b422f6aa852d259320
User & Date: mrwellan on 2023-02-27 10:54:27
Other Links: branch diff | manifest | tags
Context
2023-02-27
21:25
Added function for using ATTACH for sync between db's. Not yet in use. check-in: dca3a45c98 user: mrwellan tags: v1.80
10:54
Added suicide mode to db:with-db for development. It has been very hard to isolate threaded crashes under the tcp-server. check-in: 3b54f22608 user: mrwellan tags: v1.80
2023-02-23
11:53
Fixed get-target check-in: 78fc9c5443 user: matt tags: v1.80
Changes

Modified dbfile.scm from [5dd6613875] to [0905d9cb9e].

42
43
44
45
46
47
48






49
50
51
52
53
54
55
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61







+
+
+
+
+
+







	;; debugprint
	)

(define keep-age-param (make-parameter 10)) ;; qif file age, if over move to attic
(define num-run-dbs    (make-parameter 10))     ;; number of db's in .megatest
(define dbfile:testsuite-name (make-parameter #f))

;; 'original     - use old condition code
;; 'suicide-mode - create mtrah/stop-the-train with info on what went wrong
;; else use no condition code (should be production mode)
;;
(define no-condition-db-with-db (make-parameter 'suicide-mode))

;;======================================================================
;;  R E C O R D S
;;======================================================================

;; a single Megatest area with it's multiple dbs is
;; managed in a dbstruct
;;
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1047
1048
1049
1050
1051
1052
1053


1054
1055
1056
1057
1058
1059
1060







-
-







	      (loop (+ count 1))))))
    (with-output-to-file crumbn
      (lambda ()
	(print fname" run-id="run-id" params="params)
	))
    crumbn))

(define no-condition-db-with-db (make-parameter #t))

;; (db:with-db dbstruct run-id sqlite3:exec "select blah fgrom blaz;")
;; r/w is a flag to indicate if the db is modified by this query #t = yes, #f = no
;;
(define (dbfile:with-db dbstruct run-id r/w proc params)
  (assert dbstruct "FATAL: db:with-db called with dbstruct "#f)
  (assert (dbr:dbstruct? dbstruct) "FATAL: dbstruct is "dbstruct)
  (let* ((use-mutex (> *api-process-request-count* 25)) ;; risk of db corruption
1069
1070
1071
1072
1073
1074
1075
1076


1077
1078


1079
1080
1081
1082
1083
1084
1085


1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101





























1102
1103
1104
1105
1106
1107
1108
1073
1074
1075
1076
1077
1078
1079

1080
1081
1082

1083
1084
1085
1086
1087
1088
1089
1090

1091
1092
1093















1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129







-
+
+

-
+
+






-
+
+

-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







		      (if use-mutex (mutex-lock! *db-with-db-mutex*))
		      (let ((res (apply proc dbdat db params))) ;; the actual call is here.
			(if use-mutex (mutex-unlock! *db-with-db-mutex*))
			;; (if (vector? dbstruct)(db:done-with dbstruct run-id r/w))
			(if dbdat
			    (dbfile:add-dbdat dbstruct run-id dbdat))
			;; (delete-file* crumbfile)
			res))))
			res)))
	 (stop-train (conc (dbr:dbstruct-areapath dbstruct)"/stop-the-train")))

    (assert (sqlite3:database? db) "FATAL: db:with-db, db is not a database, db="db", fname="fname)
    (assert (sqlite3:database? db) "FATAL: db:with-db, db is not a database, db="db
	    ", fname="fname)
    (if (file-exists? jfile)
	(begin
	  (dbfile:print-err "INFO: "jfile" exists, delaying to reduce database load")
	  (thread-sleep! 0.2)))
    (if (and use-mutex
	     (common:low-noise-print 120 "over-50-parallel-api-requests"))
	(dbfile:print-err *api-process-request-count* " parallel api requests being processed in process "
	(dbfile:print-err *api-process-request-count*
			  " parallel api requests being processed in process "
			  (current-process-id))) ;;  ", throttling access"))
    (if (no-condition-db-with-db)
	(qryproc)
	(condition-case
	 (qryproc)
	 (exn (io-error)
	      (db:generic-error-printout exn "ERROR: i/o error with " fname ". Check permissions, disk space etc. and try again."))
	 (exn (corrupt)
	      (db:generic-error-printout exn "ERROR: database " fname " is corrupt. Repair it to proceed."))
	 (exn (busy)
	      (db:generic-error-printout exn "ERROR: database " fname
					 " is locked. Try copying to another location, remove original and copy back."))
	 (exn (permission)(db:generic-error-printout exn "ERROR: database " fname " has some permissions problem."))
	 (exn ()
	      (db:generic-error-printout exn "ERROR: Unknown error with database " fname " message: "
					 ((condition-property-accessor 'exn 'message) exn)))))))
    (case (no-condition-db-with-db)
      ((production)(qryproc))
      ((suicide-mode)
       (handle-exceptions
	exn
	(with-output-to-file stop-train
	  (lambda ()
	    (db:generic-error-printout exn "Stop train mode, run-id: "run-id
				       " params: "params" proc: "proc)))
	(qryproc)))
      (else
       (condition-case
	(qryproc)
	(exn (io-error)
	     (db:generic-error-printout exn "ERROR: i/o error with "fname
					". Check permissions, disk space etc. and try again."))
	(exn (corrupt)
	     (db:generic-error-printout exn "ERROR: database "fname
					" is corrupt. Repair it to proceed."))
	(exn (busy)
	     (db:generic-error-printout exn "ERROR: database "fname
					" is locked. Try copying to another location,"
					" remove original and copy back."))
	(exn (permission)(db:generic-error-printout exn "ERROR: database "fname
						    " has some permissions problem."))
	(exn ()
	     (db:generic-error-printout exn "ERROR: Unknown error with database "fname
					" message: "
					((condition-property-accessor 'exn 'message) exn))))))))

;;======================================================================
;; another attempt at a transactionized queue
;;======================================================================

;; ;; ;; (define *transaction-queues* (make-hash-table))
;; ;; ;;