Megatest

Check-in [f1ce492c36]
Login
Overview
Comment:Configurable storage of path to test via link tree or direct. Partial edits to get iterated test results rollup working properly
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | testdir-vs-linksdir
Files: files | file ages | folders
SHA1: f1ce492c366d03139edbbd688965b205480c35e6
User & Date: mrwellan on 2013-04-11 16:26:11
Other Links: branch diff | manifest | tags
Context
2013-04-11
16:26
Configurable storage of path to test via link tree or direct. Partial edits to get iterated test results rollup working properly Closed-Leaf check-in: f1ce492c36 user: mrwellan tags: testdir-vs-linksdir
09:39
Corrected issue due to inconsistent output from pathname-directory, expected . but was getting #f check-in: cf42fc1304 user: mrwellan tags: development, v1.5407
Changes

Modified launch.scm from [4243d5cf59] to [5fcbf41d44].

334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
				     ((eq? rollup-status 2)
				      ;; if the current status is AUTO the defer to the calculated value but qualify (i.e. make this AUTO-WARN)
				      (if (equal? (db:test-get-status testinfo) "AUTO") "AUTO-WARN" "WARN"))
				     (else "FAIL"))
				    (args:get-arg "-m") #f)))
	      ;; for automated creation of the rollup html file this is a good place...
	      (if (not (equal? item-path ""))
		  (open-run-close tests:summarize-items #f run-id test-name #f)) ;; don't force - just update if no
	      )
	    (mutex-unlock! m)
	    ;; (exec-results (cmd-run->list fullrunscript)) ;;  (list ">" (conc test-name "-run.log"))))
	    ;; (success      exec-results)) ;; (eq? (cadr exec-results) 0)))
	    (debug:print 2 "Output from running " fullrunscript ", pid " (vector-ref exit-info 0) " in work area " 
			 work-area ":\n====\n exit code " (vector-ref exit-info 2) "\n" "====\n")
	    ;; (sqlite3:finalize! db)







|







334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
				     ((eq? rollup-status 2)
				      ;; if the current status is AUTO the defer to the calculated value but qualify (i.e. make this AUTO-WARN)
				      (if (equal? (db:test-get-status testinfo) "AUTO") "AUTO-WARN" "WARN"))
				     (else "FAIL"))
				    (args:get-arg "-m") #f)))
	      ;; for automated creation of the rollup html file this is a good place...
	      (if (not (equal? item-path ""))
		  (tests:summarize-items run-id test-name #f)) ;; don't force - just update if no
	      )
	    (mutex-unlock! m)
	    ;; (exec-results (cmd-run->list fullrunscript)) ;;  (list ">" (conc test-name "-run.log"))))
	    ;; (success      exec-results)) ;; (eq? (cadr exec-results) 0)))
	    (debug:print 2 "Output from running " fullrunscript ", pid " (vector-ref exit-info 0) " in work area " 
			 work-area ":\n====\n exit code " (vector-ref exit-info 2) "\n" "====\n")
	    ;; (sqlite3:finalize! db)
433
434
435
436
437
438
439
440



441
442
443
444
445
446
447
		      (if rd rd (conc *toppath* "/runs"))))

	 (lnkbase  (conc linktree "/" target "/" runname))
	 (lnkpath  (conc lnkbase "/" testname))
	 (lnkpathf (conc lnkpath (if not-iterated "" "/") item-path)))

    ;; Update the rundir path in the test record for all
    (cdb:test-set-rundir-by-test-id *runremote* test-id lnkpathf)




    (debug:print 2 "INFO:\n       lnkbase=" lnkbase "\n       lnkpath=" lnkpath "\n  toptest-path=" toptest-path "\n     test-path=" test-path)
    (if (not (file-exists? linktree))
	(begin
	  (debug:print 0 "WARNING: linktree did not exist! Creating it now at " linktree)
	  (create-directory linktree #t))) ;; (system (conc "mkdir -p " linktree))))
    ;; create the directory for the tests dir links, this is needed no matter what...







|
>
>
>







433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
		      (if rd rd (conc *toppath* "/runs"))))

	 (lnkbase  (conc linktree "/" target "/" runname))
	 (lnkpath  (conc lnkbase "/" testname))
	 (lnkpathf (conc lnkpath (if not-iterated "" "/") item-path)))

    ;; Update the rundir path in the test record for all
    (cdb:test-set-rundir-by-test-id *runremote* test-id 
				    (if (equal? (configf:lookup *configdat* "setup" "testpath") "runpath")
					test-path
					lnkpathf))

    (debug:print 2 "INFO:\n       lnkbase=" lnkbase "\n       lnkpath=" lnkpath "\n  toptest-path=" toptest-path "\n     test-path=" test-path)
    (if (not (file-exists? linktree))
	(begin
	  (debug:print 0 "WARNING: linktree did not exist! Creating it now at " linktree)
	  (create-directory linktree #t))) ;; (system (conc "mkdir -p " linktree))))
    ;; create the directory for the tests dir links, this is needed no matter what...
456
457
458
459
460
461
462
463



464
465
466
467
468
469
470
    ;; NB - This is not working right - some top tests are not getting the path set!!!

    (if (not (hash-table-ref/default *toptest-paths* testname #f))
	(let* ((testinfo       (cdb:get-test-info-by-id *runremote* test-id)) ;;  run-id testname item-path))
	       (curr-test-path (if testinfo (db:test-get-rundir testinfo) #f)))
	  (hash-table-set! *toptest-paths* testname curr-test-path)
	  ;; NB// Was this for the test or for the parent in an iterated test?
	  (cdb:test-set-rundir! *runremote* run-id testname "" lnkpath) ;; toptest-path)



	  (if (or (not curr-test-path)
		  (not (directory-exists? toptest-path)))
	      (begin
		(debug:print-info 2 "Creating " toptest-path " and link " lnkpath)
		(create-directory toptest-path #t)
		(hash-table-set! *toptest-paths* testname toptest-path)))))








|
>
>
>







459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
    ;; NB - This is not working right - some top tests are not getting the path set!!!

    (if (not (hash-table-ref/default *toptest-paths* testname #f))
	(let* ((testinfo       (cdb:get-test-info-by-id *runremote* test-id)) ;;  run-id testname item-path))
	       (curr-test-path (if testinfo (db:test-get-rundir testinfo) #f)))
	  (hash-table-set! *toptest-paths* testname curr-test-path)
	  ;; NB// Was this for the test or for the parent in an iterated test?
	  (cdb:test-set-rundir! *runremote* run-id testname "" 
				(if (equal? (configf:lookup *configdat* "setup" "testpath") "runpath")
				    toptest-path
				    lnkpath))
	  (if (or (not curr-test-path)
		  (not (directory-exists? toptest-path)))
	      (begin
		(debug:print-info 2 "Creating " toptest-path " and link " lnkpath)
		(create-directory toptest-path #t)
		(hash-table-set! *toptest-paths* testname toptest-path)))))

Modified megatest.scm from [6d5064c07e] to [9827aa0182].

849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
	      ;; DO NOT put this one into either cdb:remote-run or open-run-close
	      (db:load-test-data db test-id))
	  (if (args:get-arg "-setlog")
	      (let ((logfname (args:get-arg "-setlog")))
		(cdb:test-set-log! *runremote* test-id logfname)))
	  (if (args:get-arg "-set-toplog")
	      ;; DO NOT run remote
	      (tests:test-set-toplog! db run-id test-name (args:get-arg "-set-toplog")))
	  (if (args:get-arg "-summarize-items")
	      ;; DO NOT run remote
	      (tests:summarize-items db run-id test-name #t)) ;; do force here
	  (if (args:get-arg "-runstep")
	      (if (null? remargs)
		  (begin
		    (debug:print 0 "ERROR: nothing specified to run!")
		    (if db (sqlite3:finalize! db))
		    (exit 6))
		  (let* ((stepname   (args:get-arg "-runstep"))







|


|







849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
	      ;; DO NOT put this one into either cdb:remote-run or open-run-close
	      (db:load-test-data db test-id))
	  (if (args:get-arg "-setlog")
	      (let ((logfname (args:get-arg "-setlog")))
		(cdb:test-set-log! *runremote* test-id logfname)))
	  (if (args:get-arg "-set-toplog")
	      ;; DO NOT run remote
	      (tests:test-set-toplog! run-id test-name (args:get-arg "-set-toplog")))
	  (if (args:get-arg "-summarize-items")
	      ;; DO NOT run remote
	      (tests:summarize-items run-id test-name #t)) ;; do force here
	  (if (args:get-arg "-runstep")
	      (if (null? remargs)
		  (begin
		    (debug:print 0 "ERROR: nothing specified to run!")
		    (if db (sqlite3:finalize! db))
		    (exit 6))
		  (let* ((stepname   (args:get-arg "-runstep"))

Modified tests.scm from [40dd9ff166] to [c09cbf5a9d].

337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
		 (string-match (regexp "\\S+") comment))
	    waived)
	(let ((cmt  (if waived waived comment)))
	  (cdb:remote-run db:test-set-comment #f test-id cmt)))
    ))


(define (tests:test-set-toplog! db run-id test-name logf) 
  (cdb:client-call *runremote* 'tests:test-set-toplog #t 2 logf run-id test-name))

(define (tests:summarize-items db run-id test-name force)
  ;; if not force then only update the record if one of these is true:
  ;;   1. logf is "log/final.log
  ;;   2. logf is same as outputfilename
  (let* ((outputfilename (conc "megatest-rollup-" test-name ".html"))
	 (orig-dir       (current-directory))
	 (logf-info      (cdb:remote-run db:test-get-logfile-info #f run-id test-name))
	 (logf           (if logf-info (cadr logf-info) #f))







|


|







337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
		 (string-match (regexp "\\S+") comment))
	    waived)
	(let ((cmt  (if waived waived comment)))
	  (cdb:remote-run db:test-set-comment #f test-id cmt)))
    ))


(define (tests:test-set-toplog! run-id test-name logf) 
  (cdb:client-call *runremote* 'tests:test-set-toplog #t 2 logf run-id test-name))

(define (tests:summarize-items run-id test-name force)
  ;; if not force then only update the record if one of these is true:
  ;;   1. logf is "log/final.log
  ;;   2. logf is same as outputfilename
  (let* ((outputfilename (conc "megatest-rollup-" test-name ".html"))
	 (orig-dir       (current-directory))
	 (logf-info      (cdb:remote-run db:test-get-logfile-info #f run-id test-name))
	 (logf           (if logf-info (cadr logf-info) #f))
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
		(print "<table cellspacing=\"0\" border=\"1\">" 
		       "<tr><td>Item</td><td>State</td><td>Status</td><td>Comment</td>"
		       outtxt "</table></body></html>")
		(release-dot-lock outputfilename)))
	    (close-output-port oup)
	    (change-directory orig-dir)
	    ;; NB// tests:test-set-toplog! is remote internal...
	    (tests:test-set-toplog! db run-id test-name outputfilename)
	    )))))

(define (get-all-legal-tests)
  (let* ((tests  (glob (conc *toppath* "/tests/*")))
	 (res    '()))
    (debug:print-info 4 "Looking at tests " (string-intersperse tests ","))
    (for-each (lambda (testpath)







|







425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
		(print "<table cellspacing=\"0\" border=\"1\">" 
		       "<tr><td>Item</td><td>State</td><td>Status</td><td>Comment</td>"
		       outtxt "</table></body></html>")
		(release-dot-lock outputfilename)))
	    (close-output-port oup)
	    (change-directory orig-dir)
	    ;; NB// tests:test-set-toplog! is remote internal...
	    (tests:test-set-toplog! run-id test-name outputfilename)
	    )))))

(define (get-all-legal-tests)
  (let* ((tests  (glob (conc *toppath* "/tests/*")))
	 (res    '()))
    (debug:print-info 4 "Looking at tests " (string-intersperse tests ","))
    (for-each (lambda (testpath)

Modified tests/fullrun/megatest.config from [63f2370192] to [1a54689003].

25
26
27
28
29
30
31






32
33
34
35
36
37
38
# FULL or 2, NORMAL or 1, OFF or 0
synchronous OFF
# Throttle roughly scales the db access milliseconds to seconds delay
throttle 0.2
# Max retries allows megatest to re-check that a tests status has changed
# as tests can have transient FAIL status occasionally
maxretries 20







[validvalues]
state start end 0 1 - 2
status pass fail n/a 0 1 running - 2

# These are set before all tests, override them 
# in the testconfig [pre-launch-env-overrides] section







>
>
>
>
>
>







25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
# FULL or 2, NORMAL or 1, OFF or 0
synchronous OFF
# Throttle roughly scales the db access milliseconds to seconds delay
throttle 0.2
# Max retries allows megatest to re-check that a tests status has changed
# as tests can have transient FAIL status occasionally
maxretries 20

# Store the linkpath or the runpath in the database for tests. Default is
# linkpath
#
# testpath linkpath
# testpath runpath

[validvalues]
state start end 0 1 - 2
status pass fail n/a 0 1 running - 2

# These are set before all tests, override them 
# in the testconfig [pre-launch-env-overrides] section

Added tests/fullrun/tests/sqlitespeed/logcheck.logpro version [26375d6495].



>
1
;; An empty file with no rules is legit ...

Modified tests/fullrun/tests/sqlitespeed/runscript.rb from [45705f52bd] to [cf4678be52].

1
2
3
4
5
6
7
8
9
10
11
12


13
14
15
16
17
18
19
#! /usr/bin/env ruby

require "#{ENV['MT_RUN_AREA_HOME']}/../supportfiles/ruby/librunscript.rb"

# run_record(stepname, cmd) - will record in db if exit code of script was zero or not
run_and_record('create db',"sqlite3 testing.db << EOF\ncreate table if not exists blah(id INTEGER PRIMARY KEY,name TEXT);\n.q\nEOF","")

if (! File.exists?("../../runfirst/I_was_here"))
    puts "ERROR: This test was started before the prerequisites ran!"
    system "megatest -test-status :state INCOMPLETE :status FAIL"
    exit 1
end



# file_size_checker(stepname, filename, minsize, maxsize) - negative means ignore
# file_size_checker('create db','testing.db',100,-1)

num_records=rand(5) # 0000
record_step("add #{num_records}","start","n/a")
status=false












>
>







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#! /usr/bin/env ruby

require "#{ENV['MT_RUN_AREA_HOME']}/../supportfiles/ruby/librunscript.rb"

# run_record(stepname, cmd) - will record in db if exit code of script was zero or not
run_and_record('create db',"sqlite3 testing.db << EOF\ncreate table if not exists blah(id INTEGER PRIMARY KEY,name TEXT);\n.q\nEOF","")

if (! File.exists?("../../runfirst/I_was_here"))
    puts "ERROR: This test was started before the prerequisites ran!"
    system "megatest -test-status :state INCOMPLETE :status FAIL"
    exit 1
end

system "megatest -setlog logcheck.html"

# file_size_checker(stepname, filename, minsize, maxsize) - negative means ignore
# file_size_checker('create db','testing.db',100,-1)

num_records=rand(5) # 0000
record_step("add #{num_records}","start","n/a")
status=false
28
29
30
31
32
33
34
35
36
37
38
if status==0
  status='pass'
else
  status='fail'
end

record_step("add #{num_records}","end",status)












<
<
<
30
31
32
33
34
35
36
37



if status==0
  status='pass'
else
  status='fail'
end

record_step("add #{num_records}","end",status)




Modified tests/fullrun/tests/sqlitespeed/testconfig from [a64305aaab] to [b3f8ccd259].

1
2

3
4
5
6
7
8
9
[setup]
runscript runscript.rb

tags non important,dumb junk

[requirements]
waiton    runfirst

[items]
MANYITEMS [system (env > envfile.txt;echo aa ab ac ad ae af ag ah ai)]

|
>







1
2
3
4
5
6
7
8
9
10
[setup]
runscript runscript.rb | logpro logcheck.logpro logcheck.html

tags non important,dumb junk

[requirements]
waiton    runfirst

[items]
MANYITEMS [system (env > envfile.txt;echo aa ab ac ad ae af ag ah ai)]