Index: NOTES ================================================================== --- NOTES +++ NOTES @@ -1,5 +1,28 @@ + +====================================================================== +New way of launching needed to accomodate different target hosttypes +for items +====================================================================== + +[flavors] +general ssh #{getbgesthost general} +nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo + +[hosts] +general cubian xena + +[launchers] +envsetup general +xor/%/n 4C16G +% nbgeneral + +[jobtools] +launcher internal + + + ====================================================================== Try writing to in-memory db and every 2-5 seconds syncing to megatest.db ====================================================================== First, how much time will it take to write back the changes: Index: common.scm ================================================================== --- common.scm +++ common.scm @@ -950,6 +950,55 @@ (define (mddb:get-dashboards) (let ((db (mddb:open-db))) (query fetch-column (sql db "SELECT ipaddr || ':' || portnum FROM dashboards;")))) - +;;====================================================================== +;; T E S T L A U N C H I N G P E R I T E M W I T H H O S T T Y P E S +;;====================================================================== +;; +;; [host-types] +;; general ssh #{getbgesthost general} +;; nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo +;; +;; [hosts] +;; general cubian xena +;; +;; [launchers] +;; envsetup general +;; xor/%/n 4C16G +;; % nbgeneral +;; +;; [jobtools] +;; launcher bsub +;; # if defined and not "no" flexi-launcher will bypass launcher unless there is no +;; # match. +;; flexi-launcher yes + +(define (common:get-launcher configdat testname itempath) + (let ((fallback-launcher (configf:lookup configdat "jobtools" "launcher"))) + (if (and (configf:lookup configdat "jobtools" "flexi-launcher") ;; overrides launcher + (not (equal? (configf:lookup configdat "jobtools" "flexi-launcher") "no"))) + (let* ((launchers (hash-table-ref/default configdat "launchers" '()))) + (if (null? launchers) + fallback-launcher + (let loop ((hed (car launchers)) + (tal (cdr launchers))) + (let ((patt (car hed)) + (host-type (cadr hed))) + (if (tests:match patt testname itempath) + (begin + (debug:print-info 0 "Have flexi-launcher match for " testname "/" itempath " = " host-type) + (let ((launcher (configf:lookup configdat "host-types" host-type))) + (if launcher + launcher + (begin + (debug:print-info 0 "WARNING: no launcher found for host-type " host-type) + (if (null? tal) + fallback-launcher + (loop (car tal)(cdr tal))))))) + ;; no match, try again + (if (null? tal) + fallback-launcher + (loop (car tal)(cdr tal)))))))) + fallback-launcher))) + Index: docs/manual/howto.txt ================================================================== --- docs/manual/howto.txt +++ docs/manual/howto.txt @@ -49,10 +49,34 @@ Hint: You can browse the archive using bup commands directly. ---------------- bup -d /path/to/bup/archive ftp ---------------- + +Submit jobs to Host Types based on Test Name +-------------------------------------------- + +.In megatest.config +------------------------ +[host-types] +general ssh #{getbgesthost general} +nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo + +[hosts] +general cubian xena + +[launchers] +envsetup general +xor/%/n 4C16G +% nbgeneral + +[jobtools] +launcher bsub +# if defined and not "no" flexi-launcher will bypass launcher unless there is no +# match. +flexi-launcher yes +------------------------ Tricks ------ This section is a compendium of a various useful tricks for debugging, Index: docs/manual/megatest_manual.html ================================================================== --- docs/manual/megatest_manual.html +++ docs/manual/megatest_manual.html @@ -960,10 +960,36 @@
bup -d /path/to/bup/archive ftp
+ +
+

Submit jobs to Host Types based on Test Name

+
+
+
In megatest.config
+
+
[host-types]
+general ssh #{getbgesthost general}
+nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo
+
+[hosts]
+general cubian xena
+
+[launchers]
+envsetup general
+xor/%/n 4C16G
+% nbgeneral
+
+[jobtools]
+launcher bsub
+# if defined and not "no" flexi-launcher will bypass launcher unless there is no
+# match.
+flexi-launcher yes
+
+

Tricks

This section is a compendium of a various useful tricks for debugging, @@ -1069,19 +1095,58 @@

Reference

-

Config File Settings

+

Megatest Config File Settings

Trim trailing spaces

[configf:settings trim-trailing-spaces yes]
+
+
+
+

Submit jobs to Host Types based on Test Name

+
+
+
In megatest.config
+
+
[host-types]
+general   nbfake
+remote    bsub
+
+[launchers]
+runfirst/sum% remote
+% general
+
+[jobtools]
+launcher bsub
+# if defined and not "no" flexi-launcher will bypass launcher unless there is no
+# match.
+flexi-launcher yes
+
+
+

host-types

+

List of host types and the commandline to run a job on that host type.

+
+
host-type ⇒ launch command
+
+
general nbfake
+
+
+
+

launchers

+
+
test/itempath ⇒ host-type
+
+
runfirst/sum% remote
+
+

The testconfig File

Index: docs/manual/reference.txt ================================================================== --- docs/manual/reference.txt +++ docs/manual/reference.txt @@ -1,18 +1,55 @@ Reference ========= -Config File Settings --------------------- +Megatest Config File Settings +----------------------------- Trim trailing spaces ~~~~~~~~~~~~~~~~~~~~ ------------------ [configf:settings trim-trailing-spaces yes] ------------------ + +Submit jobs to Host Types based on Test Name +-------------------------------------------- + +.In megatest.config +------------------------ +[host-types] +general nbfake +remote bsub + +[launchers] +runfirst/sum% remote +% general + +[jobtools] +launcher bsub +# if defined and not "no" flexi-launcher will bypass launcher unless there is no +# match. +flexi-launcher yes +------------------------ + +host-types +~~~~~~~~~~ + +List of host types and the commandline to run a job on that host type. + +.host-type => launch command +------------ +general nbfake +------------ + +launchers +~~~~~~~~~ +.test/itempath => host-type +------------ +runfirst/sum% remote +------------ The testconfig File ------------------- Setup section Index: launch.scm ================================================================== --- launch.scm +++ launch.scm @@ -796,11 +796,10 @@ (if ush (if (equal? ush "no") ;; must use "no" to NOT use shell #f ush) #t))) ;; default is yes - (launcher (config-lookup *configdat* "jobtools" "launcher")) (runscript (config-lookup test-conf "setup" "runscript")) (ezsteps (> (length (hash-table-ref/default test-conf "ezsteps" '())) 0)) ;; don't send all the steps, could be big (diskspace (config-lookup test-conf "requirements" "diskspace")) (memory (config-lookup test-conf "requirements" "memory")) (hosts (config-lookup *configdat* "jobtools" "workhosts")) @@ -818,11 +817,12 @@ (case (string->symbol exe) ((dboard) "../megatest") ((mtest) "../megatest") ((dashboard) "megatest") (else exe))))) - (item-path (item-list->path itemdat)) + (item-path (item-list->path itemdat)) + (launcher (common:get-launcher *configdat* test-name item-path)) ;; (config-lookup *configdat* "jobtools" "launcher")) (test-sig (conc test-name ":" item-path)) ;; (item-list->path itemdat))) ;; test-path is the full path including the item-path (work-area #f) (toptest-work-area #f) ;; for iterated tests the top test contains data relevant for all (diskpath #f) (cmdparms #f) Index: tests/fullrun/megatest.config ================================================================== --- tests/fullrun/megatest.config +++ tests/fullrun/megatest.config @@ -257,6 +257,38 @@ ltest #{scheme (case (string->symbol (conc (getenv "datapath"))) \ ((none) "nbfake") \ ((openlava) "bsub") \ (else "sleeprunner"))} + +#================================================================ +# Flexi-launcher +#================================================================ +# +# [host-types] +# general ssh #{getbgesthost general} +# nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo +# +# [hosts] +# general cubian xena +# +# [launchers] +# envsetup general +# xor/%/n 4C16G +# % nbgeneral +# +# [jobtools] +# launcher bsub +# # if defined and not "no" flexi-launcher will bypass launcher unless there is no +# # match. +# flexi-launcher yes + +[jobtools] +flexi-launcher yes + +[host-types] +general nbfake +remote bsub + +[launchers] +runfirst/sum% remote