Changes In Branch v1.65-telemetry-sideeye Excluding Merge-Ins
This is equivalent to a diff from 3797a8bab4 to 48c73df9dc
2019-02-20
| ||
16:21 | added loop to wait for testconfig to show up (combat nfs cache latency) ; added testconfig path derived from launch:execute payload to search path in case tests:get-all has gaps due to nfs cache latency check-in: 71819d884d user: bjbarcla tags: v1.65-telemetry | |
2019-02-19
| ||
12:06 | dead end Closed-Leaf check-in: 48c73df9dc user: bjbarcla tags: v1.65-telemetry-sideeye | |
2019-02-15
| ||
18:03 | wip check-in: e9f1734405 user: bjbarcla tags: v1.65-telemetry-sideeye | |
2019-02-14
| ||
15:04 | pulled in docs/manual changes check-in: 3797a8bab4 user: bjbarcla tags: v1.65-telemetry | |
15:04 | allowed for nfs latency when reading testconfig in launch:execute check-in: d75080c8ce user: bjbarcla tags: v1.65-telemetry | |
15:02 | merged in trunk to get docs/manual updates check-in: 73cb4bf58e user: bjbarcla tags: v1.65 | |
Modified common.scm from [91f2f49c15] to [df6616f1cb].
︙ | ︙ | |||
160 161 162 163 164 165 166 167 168 169 170 171 172 173 | (define *transport-type* 'http) ;; override with [server] transport http|rpc|nmsg (define *runremote* #f) ;; if set up for server communication this will hold <host port> ;; (define *max-cache-size* 0) (define *logged-in-clients* (make-hash-table)) (define *server-id* #f) (define *server-info* #f) ;; good candidate for easily convert to non-global (define *time-to-exit* #f) (define *server-run* #t) (define *run-id* #f) (define *server-kind-run* (make-hash-table)) (define *home-host* #f) ;; (define *total-non-write-delay* 0) (define *heartbeat-mutex* (make-mutex)) (define *api-process-request-count* 0) | > | 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 | (define *transport-type* 'http) ;; override with [server] transport http|rpc|nmsg (define *runremote* #f) ;; if set up for server communication this will hold <host port> ;; (define *max-cache-size* 0) (define *logged-in-clients* (make-hash-table)) (define *server-id* #f) (define *server-info* #f) ;; good candidate for easily convert to non-global (define *time-to-exit* #f) (define *time-to-exit-cleanup-thunk-list* '()) (define *server-run* #t) (define *run-id* #f) (define *server-kind-run* (make-hash-table)) (define *home-host* #f) ;; (define *total-non-write-delay* 0) (define *heartbeat-mutex* (make-mutex)) (define *api-process-request-count* 0) |
︙ | ︙ | |||
888 889 890 891 892 893 894 895 896 897 898 899 900 901 | (debug:print-info 13 *default-log-port* "watchdog done.")) (debug:print-info 13 *default-log-port* "no need for watchdog on non-homehost")))) (define (std-exit-procedure) ;;(common:telemetry-log-close) (on-exit (lambda () 0)) ;;(debug:print-info 13 *default-log-port* "std-exit-procedure called; *time-to-exit*="*time-to-exit*) (let ((no-hurry (if *time-to-exit* ;; hurry up #f (begin (set! *time-to-exit* #t) #t)))) (debug:print-info 4 *default-log-port* "starting exit process, finalizing databases.") | > > > > > > > > | 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 | (debug:print-info 13 *default-log-port* "watchdog done.")) (debug:print-info 13 *default-log-port* "no need for watchdog on non-homehost")))) (define (std-exit-procedure) ;;(common:telemetry-log-close) (on-exit (lambda () 0)) (for-each (lambda (thunk) (handle-exceptions exn (debug:print 0 *default-log-port* "WARNING: exception while running on-exit thunk item") (thunk))) *time-to-exit-cleanup-thunk-list*) ;;(debug:print-info 13 *default-log-port* "std-exit-procedure called; *time-to-exit*="*time-to-exit*) (let ((no-hurry (if *time-to-exit* ;; hurry up #f (begin (set! *time-to-exit* #t) #t)))) (debug:print-info 4 *default-log-port* "starting exit process, finalizing databases.") |
︙ | ︙ |
Added docs/manual/debug_techniques.txt version [9c5fdaf595].
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | // This file is part of Megatest. // // Megatest is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // Megatest is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with Megatest. If not, see <http://www.gnu.org/licenses/>. // // Copyright 2006-2012, Matthew Welland. Study Plan ---------- Megatest is an extensive program with a lot to learn. Following are some paths through the material to smooth the learning path. Basic Concepts (suggest you pick these up on the way) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * Components of automation; run, test, iteration * Selectors; target, runname, and testpatt Running Testsuites or Automation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * Using the dashboard gui (recommended) ** Using the "Runs" panel. ** Using the "Run Control" panel. ** Using a test control panel ** The Right Mouse Button menu ** Debug features *** xterm **** pstree **** log files; mt_copy.log, mt_launch.log **** variables; megatest.csh, megatest.sh **** testconfig dump, *testconfig *** State/status buttons *** Run, Clean, KillReq *** ReRunClean * Using the command line ** Getting help; megatest -h, megatest -manual ** Starting runs; megatest -run *** Selection controls; -target, -runname and -testpatt Writing Tests and Flows ~~~~~~~~~~~~~~~~~~~~~~~ * environment variables (table 5) * tests/_testname_/testconfig link:megatest_manual.html#_the_testconfig_file[testconfig details] ** ezsteps and logpro section ** iteration (one test applied to many inputs), items, itemstable link:megatest_manual.html#_iteration[test iteration] ** dependencies, waiton, itemmatch, itemwait link:megatest_manual.html#_requirements_section[test requirements] ** miscellaneous; mode toplevel, runtimelim, skip on file or on running, waiver propagation * megatest areas ** megatest.config ** runconfigs.config ** config language features; include, shell, system, scheme, rp|realpath, getenv, get, rget, scriptinc link:megatest.html#_config_file_helpers[config file helpers] Advanced Topics ~~~~~~~~~~~~~~~ * Removing and keeping runs selectively link:megatest_manual.html#_managing_old_runs[managing runs] * Subruns link:megatest_manual.html#_nested_runs[nested runs] * Config file features link:megatest_manual.html#_config_file_helpers[config file features] * HTML output with -generate-html * Triggers, post run, state/status * MTLOWESTLOAD * flexilauncher * env delta and testconfig * capturing test data, extracting values from logpro and using them for pass/fail * mtutil, postgres connection, packets for cross-site/cross-user control (e.g. mcrun). Maintenance and Troubleshooting ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * cleanup-db, database structure of Megatest 1.6x * archiving * homehost management * show-runconfig * show-config * show with -debug 0,9 * load management |
Modified docs/manual/megatest_manual.html from [453b3aeb82] to [defaff4299].
︙ | ︙ | |||
896 897 898 899 900 901 902 | <div class="paragraph"><p>Tests are launched using the launching system available for the distributed compute platform in use. A template script is provided which can launch jobs on local and remote Linux hosts. Currently megatest uses the network filesystem to call home to your master sqlite3 database. Megatest has been used with the Intel Netbatch and lsf (also known as openlava) batch systems and it should be straightforward to use it with other similar systems.</p></div> | < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 | <div class="paragraph"><p>Tests are launched using the launching system available for the distributed compute platform in use. A template script is provided which can launch jobs on local and remote Linux hosts. Currently megatest uses the network filesystem to call home to your master sqlite3 database. Megatest has been used with the Intel Netbatch and lsf (also known as openlava) batch systems and it should be straightforward to use it with other similar systems.</p></div> <div class="paragraph"><p>this page intentionally left blank</p></div> </div> </div> </div> <div class="sect1"> <h2 id="_installation">Installation</h2> <div class="sectionbody"> <div class="sect2"> |
︙ | ︙ | |||
1397 1398 1399 1400 1401 1402 1403 | owner myname description An example test reviewed never</pre> </div></div> <div class="paragraph"><p>This test runs a single step called "stepname1" which runs a script "stepname.sh". Note that although it is common to put the actions needed for a test step into a script it is not necessary.</p></div> | < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 | owner myname description An example test reviewed never</pre> </div></div> <div class="paragraph"><p>This test runs a single step called "stepname1" which runs a script "stepname.sh". Note that although it is common to put the actions needed for a test step into a script it is not necessary.</p></div> <div class="paragraph"><p>include:howto.txt[]</p></div> </div> </div> </div> <div class="sect1"> <h2 id="_reference">Reference</h2> <div class="sectionbody"> <div class="sect2"> |
︙ | ︙ | |||
1787 1788 1789 1790 1791 1792 1793 | <div class="content monospaced"> <pre>[configf:settings trim-trailing-spaces yes]</pre> </div></div> </div> <div class="sect2"> <h3 id="_job_submission_control">Job Submission Control</h3> <div class="sect3"> | | | 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 | <div class="content monospaced"> <pre>[configf:settings trim-trailing-spaces yes]</pre> </div></div> </div> <div class="sect2"> <h3 id="_job_submission_control">Job Submission Control</h3> <div class="sect3"> <h4 id="_submit_jobs_to_host_types_based_on_test_name">Submit jobs to Host Types based on Test Name</h4> <div class="listingblock"> <div class="title">In megatest.config</div> <div class="content monospaced"> <pre>[host-types] general nbfake remote bsub |
︙ | ︙ | |||
2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 | <td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td> <td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">( key1 key2 … )</p></td> <td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td> </tr> </tbody> </table> </div> </div> <div class="sect1"> <h2 id="_megatest_internals">Megatest Internals</h2> <div class="sectionbody"> <div class="imageblock graphviz"> <div class="content"> <img src="server.png" alt="server.png"> | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 | <td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td> <td class="tableblock halign-left valign-top" ><p class="tableblock monospaced">( key1 key2 … )</p></td> <td class="tableblock halign-left valign-top" ><p class="tableblock monospaced"></p></td> </tr> </tbody> </table> </div> </div> <div class="sect1"> <h2 id="_study_plan_2">Study Plan</h2> <div class="sectionbody"> <div class="paragraph"><p>Megatest is an extensive program with a lot to learn. Following are some paths through the material to smooth the learning path.</p></div> <div class="sect2"> <h3 id="_basic_concepts_suggest_you_pick_these_up_on_the_way_2">Basic Concepts (suggest you pick these up on the way)</h3> <div class="ulist"><ul> <li> <p> Components of automation; run, test, iteration </p> </li> <li> <p> Selectors; target, runname, and testpatt </p> </li> </ul></div> </div> <div class="sect2"> <h3 id="_running_testsuites_or_automation_2">Running Testsuites or Automation</h3> <div class="ulist"><ul> <li> <p> Using the dashboard gui (recommended) </p> <div class="ulist"><ul> <li> <p> Using the "Runs" panel. </p> </li> <li> <p> Using the "Run Control" panel. </p> </li> <li> <p> Using a test control panel </p> </li> <li> <p> The Right Mouse Button menu </p> </li> <li> <p> Debug features </p> <div class="ulist"><ul> <li> <p> xterm </p> <div class="ulist"><ul> <li> <p> pstree </p> </li> <li> <p> log files; mt_copy.log, mt_launch.log </p> </li> <li> <p> variables; megatest.csh, megatest.sh </p> </li> <li> <p> testconfig dump, *testconfig </p> </li> </ul></div> </li> <li> <p> State/status buttons </p> </li> <li> <p> Run, Clean, KillReq </p> </li> <li> <p> ReRunClean </p> </li> </ul></div> </li> </ul></div> </li> <li> <p> Using the command line </p> <div class="ulist"><ul> <li> <p> Getting help; megatest -h, megatest -manual </p> </li> <li> <p> Starting runs; megatest -run </p> <div class="ulist"><ul> <li> <p> Selection controls; -target, -runname and -testpatt </p> </li> </ul></div> </li> </ul></div> </li> </ul></div> </div> <div class="sect2"> <h3 id="_writing_tests_and_flows_2">Writing Tests and Flows</h3> <div class="ulist"><ul> <li> <p> environment variables (table 5) </p> </li> <li> <p> tests/<em>testname</em>/testconfig <a href="megatest_manual.html#_the_testconfig_file">testconfig details</a> </p> <div class="ulist"><ul> <li> <p> ezsteps and logpro section </p> </li> <li> <p> iteration (one test applied to many inputs), items, itemstable <a href="megatest_manual.html#_iteration">test iteration</a> </p> </li> <li> <p> dependencies, waiton, itemmatch, itemwait <a href="megatest_manual.html#_requirements_section">test requirements</a> </p> </li> <li> <p> miscellaneous; mode toplevel, runtimelim, skip on file or on running, waiver propagation </p> </li> </ul></div> </li> <li> <p> megatest areas </p> <div class="ulist"><ul> <li> <p> megatest.config </p> </li> <li> <p> runconfigs.config </p> </li> <li> <p> config language features; include, shell, system, scheme, rp|realpath, getenv, get, rget, scriptinc <a href="megatest.html#_config_file_helpers">config file helpers</a> </p> </li> </ul></div> </li> </ul></div> </div> <div class="sect2"> <h3 id="_advanced_topics_2">Advanced Topics</h3> <div class="ulist"><ul> <li> <p> Removing and keeping runs selectively <a href="megatest_manual.html#_managing_old_runs">managing runs</a> </p> </li> <li> <p> Subruns <a href="megatest_manual.html#_nested_runs">nested runs</a> </p> </li> <li> <p> Config file features <a href="megatest_manual.html#_config_file_helpers">config file features</a> </p> </li> <li> <p> HTML output with -generate-html </p> </li> <li> <p> Triggers, post run, state/status </p> </li> <li> <p> MTLOWESTLOAD </p> </li> <li> <p> flexilauncher </p> </li> <li> <p> env delta and testconfig </p> </li> <li> <p> capturing test data, extracting values from logpro and using them for pass/fail </p> </li> <li> <p> mtutil, postgres connection, packets for cross-site/cross-user control (e.g. mcrun). </p> </li> </ul></div> </div> <div class="sect2"> <h3 id="_maintenance_and_troubleshooting_2">Maintenance and Troubleshooting</h3> <div class="ulist"><ul> <li> <p> cleanup-db, database structure of Megatest 1.6x </p> </li> <li> <p> archiving </p> </li> <li> <p> homehost management </p> </li> <li> <p> show-runconfig </p> </li> <li> <p> show-config </p> </li> <li> <p> show with -debug 0,9 </p> </li> <li> <p> load management </p> </li> </ul></div> </div> </div> </div> <div class="sect1"> <h2 id="_megatest_internals">Megatest Internals</h2> <div class="sectionbody"> <div class="imageblock graphviz"> <div class="content"> <img src="server.png" alt="server.png"> |
︙ | ︙ | |||
3031 3032 3033 3034 3035 3036 3037 | </div> </div> </div> <div id="footnotes"><hr></div> <div id="footer"> <div id="footer-text"> Version 1.0<br> | | | 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 | </div> </div> </div> <div id="footnotes"><hr></div> <div id="footer"> <div id="footer-text"> Version 1.0<br> Last updated 2019-02-14 14:07:31 PST </div> </div> </body> </html> |
Modified docs/manual/megatest_manual.txt from [2589d78a96] to [4a326e3c90].
︙ | ︙ | |||
108 109 110 111 112 113 114 | include::getting_started.txt[] include::study_plan.txt[] // :leveloffset: 0 include::writing_tests.txt[] | | > > | 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 | include::getting_started.txt[] include::study_plan.txt[] // :leveloffset: 0 include::writing_tests.txt[] include:howto.txt[] include::reference.txt[] include::debug_techniques.txt[] Megatest Internals ------------------ ["graphviz", "server.png"] ---------------------------------------------------------------------- |
︙ | ︙ |
Added docs/manual/overview.txt version [b16c60428e].
> | 1 | this page intentionally left blank |
Modified launch.scm from [6dd1993f7c] to [47e44d66d4].
︙ | ︙ | |||
259 260 261 262 263 264 265 266 267 268 269 | ((skip) (launch:einf-rollup-status-set! exit-info 6) ;; (vector-set! exit-info 3 4) ;; rollup-status ;; NB// test-set-status! does rdb calls under the hood (tests:test-set-status! run-id test-id next-state "SKIP" (if (eq? this-step-status 'skip) "Logpro skip found" #f) #f)) ((pass) (tests:test-set-status! run-id test-id next-state "PASS" #f #f)) (else ;; 'fail (launch:einf-rollup-status-set! exit-info 1) ;; (vector-set! exit-info 3 1) ;; force fail, this used to be next-state but that doesn't make sense. should always be "COMPLETED" (tests:test-set-status! run-id test-id "COMPLETED" "FAIL" (conc "Failed at step " stepname) #f) | > > | > > | 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 | ((skip) (launch:einf-rollup-status-set! exit-info 6) ;; (vector-set! exit-info 3 4) ;; rollup-status ;; NB// test-set-status! does rdb calls under the hood (tests:test-set-status! run-id test-id next-state "SKIP" (if (eq? this-step-status 'skip) "Logpro skip found" #f) #f)) ((pass) (tests:test-set-status! run-id test-id next-state "PASS" #f #f)) (else ;; 'fail (launch:einf-rollup-status-set! exit-info 1) ;; (vector-set! exit-info 3 1) ;; force fail, this used to be next-state but that doesn't make sense. should always be "COMPLETED" (tests:test-set-status! run-id test-id "COMPLETED" "FAIL" (conc "Failed at step " stepname) #f) )) (set! *launch:execute:needs-cleanup* #f)) logpro-used)) (define (launch:manage-steps run-id test-id item-path fullrunscript ezsteps subrun test-name tconfigreg exit-info m) ;; (let-values ;; (((pid exit-status exit-code) ;; (run-n-wait fullrunscript))) ;; (tests:test-set-status! test-id "RUNNING" "n/a" #f #f) ;; Since we should have a clean slate at this time there is no need to do ;; any of the other stuff that tests:test-set-status! does. Let's just ;; force RUNNING/n/a ;; (thread-sleep! 0.3) ;; (tests:test-force-state-status! run-id test-id "RUNNING" "n/a") (set! *launch:execute:needs-cleanup* #t) (rmt:set-state-status-and-roll-up-items run-id test-name item-path "RUNNING" #f #f) ;; (thread-sleep! 0.3) ;; NFS slowness has caused grief here ;; if there is a runscript do it first (if fullrunscript (let ((pid (process-run fullrunscript))) (rmt:test-set-top-process-pid run-id test-id pid) |
︙ | ︙ | |||
458 459 460 461 462 463 464 | (if (hash-table-ref/default misc-flags 'keep-going #f) ;; keep originals for cpu-load and disk-free unless they change more than the allowed delta (loop (calc-minutes) (or new-cpu-load cpu-load) (or new-disk-free disk-free) (if do-sync (current-seconds) last-sync))))))) (tests:update-central-meta-info run-id test-id (get-cpu-load) (get-df (current-directory))(calc-minutes) #f #f))) ;; NOTE: Checking twice for keep-going is intentional | | | 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 | (if (hash-table-ref/default misc-flags 'keep-going #f) ;; keep originals for cpu-load and disk-free unless they change more than the allowed delta (loop (calc-minutes) (or new-cpu-load cpu-load) (or new-disk-free disk-free) (if do-sync (current-seconds) last-sync))))))) (tests:update-central-meta-info run-id test-id (get-cpu-load) (get-df (current-directory))(calc-minutes) #f #f))) ;; NOTE: Checking twice for keep-going is intentional (define *launch:execute:needs-cleanup* #f) (define (launch:execute encoded-cmd) (let* ((cmdinfo (common:read-encoded-string encoded-cmd)) (tconfigreg #f)) (setenv "MT_CMDINFO" encoded-cmd) ;;(bb-check-path msg: "launch:execute incoming") (if (list? cmdinfo) ;; ((testpath /tmp/mrwellan/jazzmind/src/example_run/tests/sqlitespeed) ;; (test-name sqlitespeed) (runscript runscript.rb) (db-host localhost) (run-id 1)) |
︙ | ︙ | |||
481 482 483 484 485 486 487 488 489 490 491 492 493 494 | ;; (transport (assoc/default 'transport cmdinfo)) ;; not used ;; (serverinf (assoc/default 'serverinf cmdinfo)) ;; (port (assoc/default 'port cmdinfo)) (serverurl (assoc/default 'serverurl cmdinfo)) (homehost (assoc/default 'homehost cmdinfo)) (run-id (assoc/default 'run-id cmdinfo)) (test-id (assoc/default 'test-id cmdinfo)) (target (assoc/default 'target cmdinfo)) (areaname (assoc/default 'areaname cmdinfo)) (itemdat (assoc/default 'itemdat cmdinfo)) (env-ovrd (assoc/default 'env-ovrd cmdinfo)) (set-vars (assoc/default 'set-vars cmdinfo)) ;; pre-overrides from -setvar (runname (assoc/default 'runname cmdinfo)) (megatest (assoc/default 'megatest cmdinfo)) | > > > | 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 | ;; (transport (assoc/default 'transport cmdinfo)) ;; not used ;; (serverinf (assoc/default 'serverinf cmdinfo)) ;; (port (assoc/default 'port cmdinfo)) (serverurl (assoc/default 'serverurl cmdinfo)) (homehost (assoc/default 'homehost cmdinfo)) (run-id (assoc/default 'run-id cmdinfo)) (test-id (assoc/default 'test-id cmdinfo)) (cleanup-thunk (lambda () (if *launch:execute:needs-cleanup* (rmt:test-set-state-status run-id test-id "INCOMPLETE" "KILLED" "either explicitly killed or other test execution abort failure")))) (target (assoc/default 'target cmdinfo)) (areaname (assoc/default 'areaname cmdinfo)) (itemdat (assoc/default 'itemdat cmdinfo)) (env-ovrd (assoc/default 'env-ovrd cmdinfo)) (set-vars (assoc/default 'set-vars cmdinfo)) ;; pre-overrides from -setvar (runname (assoc/default 'runname cmdinfo)) (megatest (assoc/default 'megatest cmdinfo)) |
︙ | ︙ | |||
574 575 576 577 578 579 580 581 582 583 584 585 586 587 | ;; (signal-mask! signum) ;; to mask or not? seems to cause issues in exiting (if (eq? signum signal/stop) (debug:print-error 0 *default-log-port* "attempt to STOP process. Exiting.")) (set! *time-to-exit* #t) (print "Received signal " signum ", cleaning up before exit. Please wait...") (let ((th1 (make-thread (lambda () (rmt:test-set-state-status run-id test-id "INCOMPLETE" "KILLED" #f) (print "Killed by signal " signum ". Exiting") (thread-sleep! 1) (exit 1)))) (th2 (make-thread (lambda () (thread-sleep! 2) (debug:print 0 *default-log-port* "Done") (exit 4))))) | > | 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 | ;; (signal-mask! signum) ;; to mask or not? seems to cause issues in exiting (if (eq? signum signal/stop) (debug:print-error 0 *default-log-port* "attempt to STOP process. Exiting.")) (set! *time-to-exit* #t) (print "Received signal " signum ", cleaning up before exit. Please wait...") (let ((th1 (make-thread (lambda () (rmt:test-set-state-status run-id test-id "INCOMPLETE" "KILLED" #f) (set! *launch:execute:needs-cleanup* #f) (print "Killed by signal " signum ". Exiting") (thread-sleep! 1) (exit 1)))) (th2 (make-thread (lambda () (thread-sleep! 2) (debug:print 0 *default-log-port* "Done") (exit 4))))) |
︙ | ︙ | |||
603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 | (exit)))) (test-pid (db:test-get-process_id test-info))) (cond ;; -mrw- I'm removing KILLREQ from this list so that a test in KILLREQ state is treated as a "do not run" flag. ((member (db:test-get-state test-info) '("INCOMPLETE" "KILLED" "UNKNOWN" "STUCK")) ;; prior run of this test didn't complete, go ahead and try to rerun (debug:print 0 *default-log-port* "INFO: test is INCOMPLETE or KILLED, treat this execute call as a rerun request") ;; (tests:test-force-state-status! run-id test-id "REMOTEHOSTSTART" "n/a") (rmt:test-set-state-status run-id test-id "REMOTEHOSTSTART" "n/a" #f) ) ;; prime it for running ((member (db:test-get-state test-info) '("RUNNING" "REMOTEHOSTSTART")) (if (process:alive-on-host? test-host test-pid) (debug:print-error 0 *default-log-port* "test state is " (db:test-get-state test-info) " and process " test-pid " is still running on host " test-host ", cannot proceed") (exit))) ((not (member (db:test-get-state test-info) '("REMOVING" "REMOTEHOSTSTART" "RUNNING" "KILLREQ"))) ;; (tests:test-force-state-status! run-id test-id "REMOTEHOSTSTART" "n/a") (rmt:test-set-state-status run-id test-id "REMOTEHOSTSTART" "n/a" #f) ) (else ;; (member (db:test-get-state test-info) '("REMOVING" "REMOTEHOSTSTART" "RUNNING" "KILLREQ")) (debug:print-error 0 *default-log-port* "test state is " (db:test-get-state test-info) ", cannot proceed") (exit)))) (debug:print 2 *default-log-port* "Exectuing " test-name " (id: " test-id ") on " (get-host-name)) | > > > | 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 | (exit)))) (test-pid (db:test-get-process_id test-info))) (cond ;; -mrw- I'm removing KILLREQ from this list so that a test in KILLREQ state is treated as a "do not run" flag. ((member (db:test-get-state test-info) '("INCOMPLETE" "KILLED" "UNKNOWN" "STUCK")) ;; prior run of this test didn't complete, go ahead and try to rerun (debug:print 0 *default-log-port* "INFO: test is INCOMPLETE or KILLED, treat this execute call as a rerun request") ;; (tests:test-force-state-status! run-id test-id "REMOTEHOSTSTART" "n/a") ;;BB(set! *time-to-exit-cleanup-thunk-list* (set! *launch:execute:needs-cleanup* #t) (rmt:test-set-state-status run-id test-id "REMOTEHOSTSTART" "n/a" #f) ) ;; prime it for running ((member (db:test-get-state test-info) '("RUNNING" "REMOTEHOSTSTART")) (if (process:alive-on-host? test-host test-pid) (debug:print-error 0 *default-log-port* "test state is " (db:test-get-state test-info) " and process " test-pid " is still running on host " test-host ", cannot proceed") (exit))) ((not (member (db:test-get-state test-info) '("REMOVING" "REMOTEHOSTSTART" "RUNNING" "KILLREQ"))) ;; (tests:test-force-state-status! run-id test-id "REMOTEHOSTSTART" "n/a") (set! *launch:execute:needs-cleanup* #t) (rmt:test-set-state-status run-id test-id "REMOTEHOSTSTART" "n/a" #f) ) (else ;; (member (db:test-get-state test-info) '("REMOVING" "REMOTEHOSTSTART" "RUNNING" "KILLREQ")) (debug:print-error 0 *default-log-port* "test state is " (db:test-get-state test-info) ", cannot proceed") (exit)))) (debug:print 2 *default-log-port* "Exectuing " test-name " (id: " test-id ") on " (get-host-name)) |
︙ | ︙ | |||
1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 | (begin (debug:print-info 0 *default-log-port* "attempting to preclean directory " (db:test-get-rundir testinfo) " for test " test-name "/" item-path) (runs:remove-test-directory testinfo 'remove-data-only))) ;; remove data only, do not perturb the record ;; prevent overlapping actions - set to LAUNCHED as early as possible ;; ;; the following call handles waiver propogation. cannot yet condense into roll-up-pass-fail (tests:test-set-status! run-id test-id "LAUNCHED" "n/a" #f #f) ;; (if launch-results launch-results "FAILED")) (rmt:set-state-status-and-roll-up-items run-id test-name item-path #f "LAUNCHED" #f) ;; (pp (hash-table->alist tconfig)) (set! diskpath (get-best-disk *configdat* tconfig)) (if diskpath (let ((dat (create-work-area run-id run-info keyvals test-id test-path diskpath test-name itemdat))) (set! work-area (car dat)) | > | 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 | (begin (debug:print-info 0 *default-log-port* "attempting to preclean directory " (db:test-get-rundir testinfo) " for test " test-name "/" item-path) (runs:remove-test-directory testinfo 'remove-data-only))) ;; remove data only, do not perturb the record ;; prevent overlapping actions - set to LAUNCHED as early as possible ;; ;; the following call handles waiver propogation. cannot yet condense into roll-up-pass-fail (set! *launch:execute:needs-cleanup* #t) (tests:test-set-status! run-id test-id "LAUNCHED" "n/a" #f #f) ;; (if launch-results launch-results "FAILED")) (rmt:set-state-status-and-roll-up-items run-id test-name item-path #f "LAUNCHED" #f) ;; (pp (hash-table->alist tconfig)) (set! diskpath (get-best-disk *configdat* tconfig)) (if diskpath (let ((dat (create-work-area run-id run-info keyvals test-id test-path diskpath test-name itemdat))) (set! work-area (car dat)) |
︙ | ︙ | |||
1531 1532 1533 1534 1535 1536 1537 | (conc cmdstr " >> mt_launch.log 2>&1 &"))) (car fullcmd)) (if useshell '() (cdr fullcmd)))) (success (if launchwait (equal? 0 (cadr launch-results-prev)) #t)) (launch-results (if launchwait (car launch-results-prev) launch-results-prev))) | | | > | 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 | (conc cmdstr " >> mt_launch.log 2>&1 &"))) (car fullcmd)) (if useshell '() (cdr fullcmd)))) (success (if launchwait (equal? 0 (cadr launch-results-prev)) #t)) (launch-results (if launchwait (car launch-results-prev) launch-results-prev))) (when (not success) (tests:test-set-status! run-id test-id "COMPLETED" "DEAD" "launcher failed; exited non-zero; check mt_launch.log" #f) (set! *launch:execute:needs-cleanup* #f)) ;; (if launch-results launch-results "FAILED")) (mutex-unlock! *launch-setup-mutex*) ;; yes, really should mutex all the way to here. Need to put this entire process into a fork. ;; (rmt:no-sync-del! lock-key) ;; release the lock for starting this test (if (not launchwait) ;; give the OS a little time to allow the process to start (thread-sleep! 0.01)) (with-output-to-file "mt_launch.log" (lambda () (print "LAUNCHCMD: " (string-intersperse fullcmd " ")) |
︙ | ︙ |
Modified tests.scm from [82f5f9277a] to [ebe84ac0bd].
︙ | ︙ | |||
1545 1546 1547 1548 1549 1550 1551 | dat ;; no cached data available (let* ((treg (or test-registry (tests:get-all))) (test-path (or (hash-table-ref/default treg test-name #f) (conc *toppath* "/tests/" test-name))) (test-configf (conc test-path "/testconfig")) | | > | > > > | | | | | | | | | | | > > | 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 | dat ;; no cached data available (let* ((treg (or test-registry (tests:get-all))) (test-path (or (hash-table-ref/default treg test-name #f) (conc *toppath* "/tests/" test-name))) (test-configf (conc test-path "/testconfig")) (testexists (let loop ((tries-left 14)) (cond ( (and (common:file-exists? test-configf)(file-read-access? test-configf)) #t) ( (common:file-exists? test-configf) (debug:print 0 *default-log-port* "WARNING: Cannot read testconfig file: "test-configf) #f) ( (and wait-a-minute (> tries-left 0)) (debug:print 0 *default-log-port* "WARNING: Cannot read testconfig. Sleeping 10 and will try again. Tries left: " tries-left) (thread-sleep! 10) (loop (sub1 tries-left))) (else (debug:print 0 *default-log-port* "WARNING: testconfig file does not exist: "test-configf " [wait-a-minute=" wait-a-minute "]") #f)))) (tcfg (if testexists (read-config test-configf #f system-allowed environ-patt: (if system-allowed "pre-launch-env-vars" #f)) #f))) (cond ((testexists (if (and tcfg cache-file) (hash-table-set! tcfg "have fulldata" #t)) ;; mark this as fully read data (if tcfg (hash-table-set! *testconfigs* test-full-name tcfg)) ;; side effect (if (and testexists cache-file (file-write-access? cache-path) allow-write-cache) (let ((tpath (conc cache-path "/.testconfig"))) (debug:print-info 1 *default-log-port* "Caching testconfig for " test-name " in " tpath) (if (and tcfg (not (common:in-running-test?))) (configf:write-alist tcfg tpath)))) tcfg)) (else #f)))))))) ;; sort tests by priority and waiton ;; Move test specific stuff to a test unit FIXME one of these days (define (tests:sort-by-priority-and-waiton test-records) (if (eq? (hash-table-size test-records) 0) '() (let* ((mungepriority (lambda (priority) |
︙ | ︙ |