Skip to content

Commit ecff885

Browse files
noahvnoahv
committed
Refactored Environment Module Support
 - New global TestDef structure maintains requested modules - Added checkForModules, applyModules, and revertModules helpers - Added tests/bat/Autotools sample autotools based hello world - Added tests/bat/ModuleCmd ini tests for plugins modified - Updated extensive_example.ini to include new modules_swap Signed-off-by: Noah van Dresser <[email protected]>
1 parent a344c9d commit ecff885

34 files changed

+5564
-604
lines changed

‎pylib/System/TestDef.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,10 @@ def __init__(self):
6262
# note that it hasn't yet been defined
6363
self.logger = None
6464
self.modcmd = None
65+
# set aside per stage environment module requests
66+
self.module_unload = {}
67+
self.module_load = {}
68+
self.module_swap = {}
6569
self.execmd = None
6670
self.harasser = None
6771
self.config = None

‎pylib/Tools/Build/Autotools.py‎

Lines changed: 133 additions & 146 deletions
Large diffs are not rendered by default.

‎pylib/Tools/Build/Shell.py‎

Lines changed: 26 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,9 @@
2525
# @param merge_stdout_stderr Merge stdout and stderr into one output stream
2626
# @param stdout_save_lines Number of lines of stdout to save
2727
# @param stderr_save_lines Number of lines of stderr to save
28-
# @param modules Modules to load
2928
# @param modules_unload Modules to unload
29+
# @param modules Modules to load
30+
# @param modules_swap Modules to swap
3031
# @param fail_test Specifies whether this test is expected to fail (value=None means test is expected to succeed)
3132
# @param fail_returncode Specifies the expected failure returncode of this test
3233
# @param allocate_cmd Command to use for allocating nodes from the resource manager
@@ -44,8 +45,9 @@ def __init__(self):
4445
self.options['merge_stdout_stderr'] = (False, "Merge stdout and stderr into one output stream")
4546
self.options['stdout_save_lines'] = (-1, "Number of lines of stdout to save")
4647
self.options['stderr_save_lines'] = (-1, "Number of lines of stderr to save")
47-
self.options['modules'] = (None, "Modules to load")
4848
self.options['modules_unload'] = (None, "Modules to unload")
49+
self.options['modules'] = (None, "Modules to load")
50+
self.options['modules_swap'] = (None, "Modules to swap")
4951
self.options['fail_test'] = (None, "Specifies whether this test is expected to fail (value=None means test is expected to succeed)")
5052
self.options['fail_returncode'] = (None, "Specifies the expected failure returncode of this test")
5153
self.options['allocate_cmd'] = (None, "Command to use for allocating nodes from the resource manager")
@@ -262,52 +264,22 @@ def execute(self, log, keyvals, testDef):
262264
except KeyError:
263265
pass
264266
# check for modules required by the middleware
265-
try:
266-
if midlog['parameters'] is not None:
267-
for md in midlog['parameters']:
268-
if "modules" == md[0]:
269-
try:
270-
if cmds['modules'] is not None:
271-
# append these modules to those
272-
mods = md[1].split(',')
273-
newmods = modules.split(',')
274-
for md in newmods:
275-
mods.append(md)
276-
cmds['modules'] = ','.join(mods)
277-
except KeyError:
278-
cmds['modules'] = md[1]
279-
break
280-
except KeyError:
281-
pass
267+
status,stdout,stderr = testDef.modcmd.checkForModules(log['section'], midlog, cmds, testDef)
268+
if 0 != status:
269+
log['status'] = status
270+
log['stdout'] = stdout
271+
log['stderr'] = stderr
272+
return
282273
except KeyError:
283274
pass
284275

285-
# check to see if they specified a module to use
286-
# where the compiler can be found
287-
usedModuleUnload = False
288-
try:
289-
if cmds['modules_unload'] is not None:
290-
status,stdout,stderr = testDef.modcmd.unloadModules(cmds['modules_unload'], testDef)
291-
if 0 != status:
292-
log['status'] = status
293-
log['stderr'] = stderr
294-
return
295-
usedModuleUnload = True
296-
except KeyError:
297-
# not required to provide a module to unload
298-
pass
299-
usedModule = False
300-
try:
301-
if cmds['modules'] is not None:
302-
status,stdout,stderr = testDef.modcmd.loadModules(cmds['modules'], testDef)
303-
if 0 != status:
304-
log['status'] = status
305-
log['stderr'] = stderr
306-
return
307-
usedModule = True
308-
except KeyError:
309-
# not required to provide a module
310-
pass
276+
# Apply any requested environment module settings
277+
status,stdout,stderr = testDef.modcmd.applyModules(log['section'], cmds, testDef)
278+
if 0 != status:
279+
log['status'] = status
280+
log['stdout'] = stdout
281+
log['stderr'] = stderr
282+
return
311283

312284
# sense and record the compiler being used
313285
plugin = None
@@ -399,21 +371,15 @@ def execute(self, log, keyvals, testDef):
399371
log['stderr'] = stderr
400372
# record this location for any follow-on steps
401373
log['location'] = location
402-
if usedModule:
403-
# unload the modules before returning
404-
status,stdout,stderr = testDef.modcmd.unloadModules(cmds['modules'], testDef)
405-
if 0 != status:
406-
log['status'] = status
407-
log['stderr'] = stderr
408-
os.chdir(cwd)
409-
return
410-
if usedModuleUnload:
411-
status,stdout,stderr = testDef.modcmd.loadModules(cmds['modules_unload'], testDef)
412-
if 0 != status:
413-
log['status'] = status
414-
log['stderr'] = stderr
415-
os.chdir(cwd)
416-
return
374+
375+
# Revert any requested environment module settings
376+
status,stdout,stderr = testDef.modcmd.revertModules(log['section'], testDef)
377+
if 0 != status:
378+
log['status'] = status
379+
log['stdout'] = stdout
380+
log['stderr'] = stderr
381+
return
382+
417383
# if we added middleware to the paths, remove it
418384
if midpath:
419385
os.environ['PATH'] = oldbinpath

‎pylib/Tools/CNC/IPMITool.py‎

Lines changed: 23 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,9 @@ def run(self):
144144
# @param numthreads Number of worker threads to use
145145
# @param dryrun Dryrun - print out commands but do not execute
146146
# @param sudo Use sudo to exeute privilaged comands
147+
# @param modules_unload Modules to unload
148+
# @param modules Modules to load
149+
# @param modules_swap Modules to swap
147150
# @}
148151
class IPMITool(CNCMTTTool):
149152
def __init__(self):
@@ -159,6 +162,9 @@ def __init__(self):
159162
self.options['numthreads'] = (30, "Number of worker threads to use")
160163
self.options['dryrun'] = (False, "Dryrun - print out commands but do not execute")
161164
self.options['sudo'] = (False, "Use sudo to execute privileged commands")
165+
self.options['modules_unload'] = (None, "Modules to unload")
166+
self.options['modules'] = (None, "Modules to load")
167+
self.options['modules_swap'] = (None, "Modules to swap")
162168
self.lock = threading.Lock()
163169
self.threads = []
164170
self.threadID = 0
@@ -184,32 +190,20 @@ def print_options(self, testDef, prefix):
184190

185191
def execute(self, log, keyvals, testDef):
186192
testDef.logger.verbose_print("IPMITool execute")
187-
# check for a modules directive
188-
mods = None
189-
try:
190-
if keyvals['modules'] is not None:
191-
if testDef.modcmd is None:
192-
# cannot execute this request
193-
log['stderr'] = "No module support available"
194-
log['status'] = 1
195-
return
196-
# create a list of the requested modules
197-
mods = keyvals['modules'].split(',')
198-
# have them loaded
199-
status,stdout,stderr = testDef.modcmd.loadModules(mods, testDef)
200-
if 0 != status:
201-
log['status'] = status
202-
log['stdout'] = stdout
203-
log['stderr'] = stderr
204-
return
205-
modloaded = True
206-
except KeyError:
207-
pass
208193

209194
# parse what we were given against our defined options
210195
cmds = {}
211196
testDef.parseOptions(log, self.options, keyvals, cmds)
212197
testDef.logger.verbose_print("IPMITool: " + ' '.join(cmds))
198+
199+
# Apply any requested environment module settings
200+
status,stdout,stderr = testDef.modcmd.applyModules(log['section'], cmds, testDef)
201+
if 0 != status:
202+
log['status'] = status
203+
log['stdout'] = stdout
204+
log['stderr'] = stderr
205+
return
206+
213207
# must have given us at least one controller address
214208
try:
215209
if cmds['controller'] is None:
@@ -335,8 +329,13 @@ def execute(self, log, keyvals, testDef):
335329
if 0 != st[0]:
336330
log['status'] = st[0]
337331
log['stderr'] = st[2]
338-
# reset modules if necessary
339-
if mods is not None:
340-
testDef.modcmd.unloadModules(mods, testDef)
332+
333+
# Revert any requested environment module settings
334+
status,stdout,stderr = testDef.modcmd.revertModules(log['section'], testDef)
335+
if 0 != status:
336+
log['status'] = status
337+
log['stdout'] = stdout
338+
log['stderr'] = stderr
339+
return
341340

342341
return

‎pylib/Tools/Fetch/AlreadyInstalled.py‎

Lines changed: 34 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@
1919
# @section AlreadyInstalled
2020
# No-op plugin for using existing middleware installation
2121
# @param exec Executable that should be in path
22-
# @param module Modules (or lmod modules) to be loaded for accessing this package
22+
# @param modules_unload Modules to unload
23+
# @param modules Modules to load
24+
# @param modules_swap Modules to swap
2325
# @}
2426
class AlreadyInstalled(FetchMTTTool):
2527

@@ -28,7 +30,9 @@ def __init__(self):
2830
FetchMTTTool.__init__(self)
2931
self.options = {}
3032
self.options['exec'] = (None, "Executable that should be in path")
31-
self.options['module'] = (None, "Modules (or lmod modules) to be loaded for accessing this package")
33+
self.options['modules'] = (None, "Modules to load")
34+
self.options['modules_unload'] = (None, "Modules to unload")
35+
self.options['modules_swap'] = (None, "Modules to swap")
3236
return
3337

3438
def activate(self):
@@ -53,44 +57,32 @@ def print_options(self, testDef, prefix):
5357
def execute(self, log, keyvals, testDef):
5458
# if we were given an executable to check for,
5559
# see if we can find it
56-
usedModule = False
57-
try:
58-
if keyvals['exec'] is not None:
59-
# if we were given a module to load, then
60-
# do so prior to checking for the executable
61-
try:
62-
if keyvals['module'] is not None:
63-
status,stdout,stderr = testDef.modcmd.loadModules(keyvals['modules'], testDef)
64-
if 0 != status:
65-
log['status'] = status
66-
log['stderr'] = stderr
67-
return
68-
usedModule = True
69-
except KeyError:
70-
pass
71-
# now look for the executable in our path
72-
if not find_executable(keyvals['exec']):
73-
log['status'] = 1
74-
log['stderr'] = "Executable {0} not found".format(keyvals['exec'])
75-
else:
76-
log['status'] = 0
77-
if usedModule:
78-
# unload the modules before returning
79-
status,stdout,stderr = testDef.modcmd.unloadModules(keyvals['modules'], testDef)
80-
if 0 != status:
81-
log['status'] = status
82-
log['stderr'] = stderr
83-
return
84-
usedModule = False
85-
return
86-
except KeyError:
87-
pass
88-
if usedModule:
89-
# unload the modules before returning
90-
status,stdout,stderr = testDef.modcmd.unloadModules(keyvals['modules'], testDef)
91-
if 0 != status:
92-
log['status'] = status
93-
log['stderr'] = stderr
94-
return
95-
log['status'] = 0
60+
61+
# parse any provided options - these will override the defaults
62+
cmds = {}
63+
testDef.parseOptions(log, self.options, keyvals, cmds)
64+
65+
# Apply any requested environment module settings
66+
status,stdout,stderr = testDef.modcmd.applyModules(log['section'], cmds, testDef)
67+
if 0 != status:
68+
log['status'] = status
69+
log['stdout'] = stdout
70+
log['stderr'] = stderr
71+
return
72+
73+
# now look for the executable in our path
74+
if not find_executable(keyvals['exec']):
75+
log['status'] = 1
76+
log['stderr'] = "Executable " + cmds['exec'] + " not found"
77+
else:
78+
log['status'] = 0
79+
80+
# Revert any requested environment module settings
81+
status,stdout,stderr = testDef.modcmd.revertModules(log['section'], testDef)
82+
if 0 != status:
83+
log['status'] = status
84+
log['stdout'] = stdout
85+
log['stderr'] = stderr
86+
return
87+
9688
return

0 commit comments

Comments
 (0)