From bcf1d9d01747e124185026616fc5a5883881624c Mon Sep 17 00:00:00 2001 From: Zee Date: Wed, 23 Oct 2024 23:02:44 +0000 Subject: [PATCH 01/12] added rtest.py and smoke test --- rtest.py | 324 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ rtest.xml | 4 + 2 files changed, 328 insertions(+) create mode 100644 rtest.py diff --git a/rtest.py b/rtest.py new file mode 100644 index 000000000..c8f8ea791 --- /dev/null +++ b/rtest.py @@ -0,0 +1,324 @@ +#!/usr/bin/python3 +"""Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. +Run tests on build""" + +import re +import os +import sys +import subprocess +import shlex +import argparse +import pathlib +import platform +from genericpath import exists +from fnmatch import fnmatchcase +from xml.dom import minidom +import multiprocessing +import time + +args = {} +OS_info = {} + +timeout = False +test_proc = None +stop = 0 + +test_script = [ 'cd %IDIR%', '%XML%' ] + +def parse_args(): + """Parse command-line arguments""" + parser = argparse.ArgumentParser(description=""" + Checks build arguments + """) + parser.add_argument('-t', '--test', required=True, + help='Test set to run from rtest.xml (required, e.g. osdb)') + parser.add_argument('-g', '--debug', required=False, default=False, action='store_true', + help='Test Debug build (optional, default: false)') + parser.add_argument('-o', '--output', type=str, required=False, default="xml", + help='Test output file (optional, default: test_detail.xml)') + parser.add_argument( '--install_dir', type=str, required=False, default="build", + help='Installation directory where build or release folders are (optional, default: build)') + parser.add_argument( '--fail_test', default=False, required=False, action='store_true', + help='Return as if test failed (optional, default: false)') + # parser.add_argument('-v', '--verbose', required=False, default = False, action='store_true', + # help='Verbose install (optional, default: False)') + return parser.parse_args() + + +def vram_detect(): + global OS_info + OS_info["VRAM"] = 0 + if os.name == "nt": + cmd = "hipinfo.exe" + process = subprocess.run([cmd], stdout=subprocess.PIPE) + for line_in in process.stdout.decode().splitlines(): + if 'totalGlobalMem' in line_in: + OS_info["VRAM"] = float(line_in.split()[1]) + break + else: + cmd = "rocminfo" + process = subprocess.run([cmd], stdout=subprocess.PIPE) + for line_in in process.stdout.decode().splitlines(): + match = re.search(r'.*Size:.*([0-9]+)\(.*\).*KB', line_in, re.IGNORECASE) + if match: + OS_info["VRAM"] = float(match.group(1))/(1024*1024) + break + +def os_detect(): + global OS_info + if os.name == "nt": + OS_info["ID"] = platform.system() + else: + inf_file = "/etc/os-release" + if os.path.exists(inf_file): + with open(inf_file) as f: + for line in f: + if "=" in line: + k,v = line.strip().split("=") + OS_info[k] = v.replace('"','') + OS_info["NUM_PROC"] = os.cpu_count() + vram_detect() + print(OS_info) + + +def create_dir(dir_path): + if os.path.isabs(dir_path): + full_path = dir_path + else: + full_path = os.path.join( os.getcwd(), dir_path ) + return pathlib.Path(full_path).mkdir(parents=True, exist_ok=True) + +def delete_dir(dir_path) : + if (not os.path.exists(dir_path)): + return + if os.name == "nt": + return run_cmd( "RMDIR" , f"/S /Q {dir_path}") + else: + linux_path = pathlib.Path(dir_path).absolute() + return run_cmd( "rm" , f"-rf {linux_path}") + +class TimerProcess(multiprocessing.Process): + + def __init__(self, start, stop, kill_pid): + multiprocessing.Process.__init__(self) + self.quit = multiprocessing.Event() + self.timed_out = multiprocessing.Event() + self.start_time = start + self.max_time = stop + self.kill_pid = kill_pid + + def run(self): + while not self.quit.is_set(): + #print( f'time_stop {self.start_time} limit {self.max_time}') + if (self.max_time == 0): + return + t = time.monotonic() + if ( t - self.start_time > self.max_time ): + print( f'killing {self.kill_pid} t {t}') + if os.name == "nt": + cmd = ['TASKKILL', '/F', '/T', '/PID', str(self.kill_pid)] + proc = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) + else: + os.kill(self.kill_pid, signal.SIGKILL) + self.timed_out.set() + self.stop() + pass + + def stop(self): + self.quit.set() + + def stopped(self): + return self.timed_out.is_set() + + +def time_stop(start, pid): + global timeout, stop + while (True): + print( f'time_stop {start} limit {stop}') + t = time.monotonic() + if (stop == 0): + return + if ( (stop > 0) and (t - start > stop) ): + print( f'killing {pid} t {t}') + if os.name == "nt": + cmd = ['TASKKILL', '/F', '/T', '/PID', str(pid)] + proc = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) + else: + test_proc.kill() + timeout = True + stop = 0 + time.sleep(0) + +def run_cmd(cmd, test = False, time_limit = 0): + global args + global test_proc, timer_thread + global stop + if (cmd.startswith('cd ')): + return os.chdir(cmd[3:]) + if (cmd.startswith('mkdir ')): + return create_dir(cmd[6:]) + cmdline = f"{cmd}" + print(cmdline) + try: + if not test: + proc = subprocess.run(cmdline, check=True, stderr=subprocess.STDOUT, shell=True) + status = proc.returncode + else: + error = False + timeout = False + test_proc = subprocess.Popen(cmdline, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + if time_limit > 0: + start = time.monotonic() + #p = multiprocessing.Process(target=time_stop, args=(start, test_proc.pid)) + p = TimerProcess(start, time_limit, test_proc.pid) + p.start() + while True: + output = test_proc.stdout.readline() + if output == '' and test_proc.poll() is not None: + break + elif output: + outstring = output.strip() + print (outstring) + error = error or re.search(r'FAILED', outstring) + status = test_proc.poll() + if time_limit > 0: + p.stop() + p.join() + timeout = p.stopped() + print(f"timeout {timeout}") + if error: + status = 1 + elif timeout: + status = 2 + else: + status = test_proc.returncode + except: + import traceback + exc = traceback.format_exc() + print( "Python Exception: {0}".format(exc) ) + status = 3 + return status + +def batch(script, xml): + global OS_info + global args + # + cwd = pathlib.os.curdir + rtest_cwd_path = os.path.abspath( os.path.join( cwd, 'rtest.xml') ) + + if os.path.isfile(rtest_cwd_path) and os.path.dirname(rtest_cwd_path).endswith( "staging" ): + # if in a staging directory then test locally + test_dir = cwd + else: + # deal with windows pathing + install_dir = '//'.join(args.install_dir.split('\\')) + + if args.debug: + build_type = "debug" + else: + #check if we have a release folder in build + if os.path.isdir(f'{install_dir}//release//test'): + build_type = "release" + else: + build_type = "" + + if len(build_type) > 0: + test_dir = f"{install_dir}//{build_type}//test" + else: + test_dir = f"{install_dir}//test" + fail = False + for i in range(len(script)): + cmdline = script[i] + xcmd = cmdline.replace('%IDIR%', test_dir) + cmd = xcmd.replace('%ODIR%', args.output) + if cmd.startswith('tdir '): + if pathlib.Path(cmd[5:]).exists(): + return 0 # all further cmds skipped + else: + continue + error = False + if cmd.startswith('%XML%'): + # run the matching tests listed in the xml test file + var_subs = {} + for var in xml.getElementsByTagName('var'): + name = var.getAttribute('name') + val = var.getAttribute('value') + var_subs[name] = val + for test in xml.getElementsByTagName('test'): + sets = test.getAttribute('sets') + runset = sets.split(',') + if args.test in runset: + for run in test.getElementsByTagName('run'): + name = run.getAttribute('name') + vram_limit = run.getAttribute('vram_min') + if vram_limit: + if OS_info["VRAM"] < float(vram_limit): + print( f'***\n*** Skipped: {name} due to VRAM req.\n***') + continue + if name: + print( f'***\n*** Running: {name}\n***') + time_limit = run.getAttribute('time_max') + if time_limit: + timeout = float(time_limit) + else: + timeout = 0 + + raw_cmd = run.firstChild.data + var_cmd = raw_cmd.format_map(var_subs) + error = run_cmd(var_cmd, True, timeout) + if (error == 2): + print( f'***\n*** Timed out when running: {name}\n***') + else: + error = run_cmd(cmd) + fail = fail or error + if (fail): + if (cmd == "%XML%"): + print(f"FAILED xml test suite!") + else: + print(f"ERROR running: {cmd}") + if (os.curdir != cwd): + os.chdir( cwd ) + return 1 + if (os.curdir != cwd): + os.chdir( cwd ) + + return 0 + +def run_tests(): + global test_script + global xmlDoc + + # install + cwd = os.curdir + + xmlPath = os.path.join( cwd, 'rtest.xml') + xmlDoc = minidom.parse( xmlPath ) + + scripts = [] + scripts.append( test_script ) + for i in scripts: + if (batch(i, xmlDoc)): + #print("Failure in script. ABORTING") + if (os.curdir != cwd): + os.chdir( cwd ) + return 1 + if (os.curdir != cwd): + os.chdir( cwd ) + return 0 + +def main(): + global args + global timer_thread + + os_detect() + args = parse_args() + + status = run_tests() + + if args.fail_test: + status = 1 + if (status): + sys.exit(status) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/rtest.xml b/rtest.xml index 431fd28d4..95b12210e 100644 --- a/rtest.xml +++ b/rtest.xml @@ -2,11 +2,15 @@ + {CTEST_FILTER} {CTEST_REGEX} {CTEST_FILTER} {CTEST_REGEX} + + {SMOKE_TEST} + From 633a6b9670f92315b3feb01afc23d88e57d101d5 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Wed, 23 Oct 2024 23:07:14 +0000 Subject: [PATCH 02/12] updated changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 143bbb7c6..736b56eae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ Documentation for rocRAND is available at [https://rocm.docs.amd.com/projects/rocRAND/en/latest/](https://rocm.docs.amd.com/projects/rocRAND/en/latest/) +## (Unreleased) rocRAND-3.2.0 for ROCm 6.4 + +### Added +* rtest.py, a python script to run unit tests + * added a smoke test option. Smoke tests can now be ran with python3 rtest.py -t/--test smoke + ## (Unreleased) rocRAND-3.2.0 for ROCm 6.3.0 ### Added From 73ce3a62bd72ab9dfc22531dbbdf7df6772f5e7f Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Wed, 23 Oct 2024 23:10:49 +0000 Subject: [PATCH 03/12] removed extra line (unreleased) rocRAND 3.2.0 for ROCm 6.3.0 in changelog --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3554144c0..a01c428d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,11 +10,9 @@ Documentation for rocRAND is available at * rtest.py, a python script to run unit tests * added a smoke test option. Smoke tests can now be ran with python3 rtest.py -t/--test smoke -## (Unreleased) rocRAND-3.2.0 for ROCm 6.3.0 ## rocRAND 3.2.0 for ROCm 6.3.0 - ### Added * Added host generator for MT19937 From 1d7c9885285f5e1e79314fd21d4dee72cebea946 Mon Sep 17 00:00:00 2001 From: Di Nguyen Date: Thu, 24 Oct 2024 09:32:40 -0600 Subject: [PATCH 04/12] Update CHANGELOG.md removed extra - Co-authored-by: Jeffrey Novotny --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a01c428d5..1f104c199 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Documentation for rocRAND is available at [https://rocm.docs.amd.com/projects/rocRAND/en/latest/](https://rocm.docs.amd.com/projects/rocRAND/en/latest/) -## (Unreleased) rocRAND-3.2.0 for ROCm 6.4 +## (Unreleased) rocRAND 3.2.0 for ROCm 6.4 ### Added * rtest.py, a python script to run unit tests From 9bdc16036d2b296d12c4bf54ef268ec924edaef7 Mon Sep 17 00:00:00 2001 From: Di Nguyen Date: Thu, 24 Oct 2024 09:33:08 -0600 Subject: [PATCH 05/12] Update CHANGELOG.md reformatted instructions to rtest Co-authored-by: Jeffrey Novotny --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f104c199..dfc2a56af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ Documentation for rocRAND is available at ### Added * rtest.py, a python script to run unit tests - * added a smoke test option. Smoke tests can now be ran with python3 rtest.py -t/--test smoke + * added a smoke test option. To run the smoke tests, use `python3 rtest.py -t/--test smoke`. ## rocRAND 3.2.0 for ROCm 6.3.0 From 139a56c45d8b25ba8e0f36ea4a576665bb6d2b80 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Thu, 24 Oct 2024 22:51:29 +0000 Subject: [PATCH 06/12] added --emulation option --- rtest.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/rtest.py b/rtest.py index c8f8ea791..501c7643d 100644 --- a/rtest.py +++ b/rtest.py @@ -30,8 +30,10 @@ def parse_args(): parser = argparse.ArgumentParser(description=""" Checks build arguments """) - parser.add_argument('-t', '--test', required=True, - help='Test set to run from rtest.xml (required, e.g. osdb)') + parser.add_argument('-e', '--emulation', required=False, default="", + help='Run emulation test sets (optional, eg.smoke). At least one of -e or -t must be set') + parser.add_argument('-t', '--test', required=False, default="", + help='Test set to run from rtest.xml (optional, e.g. osdb). At least one of -e or -t must be set') parser.add_argument('-g', '--debug', required=False, default=False, action='store_true', help='Test Debug build (optional, default: false)') parser.add_argument('-o', '--output', type=str, required=False, default="xml", @@ -247,6 +249,9 @@ def batch(script, xml): for test in xml.getElementsByTagName('test'): sets = test.getAttribute('sets') runset = sets.split(',') + + assert(args.test != '' or args.emulation != '') + if args.test in runset: for run in test.getElementsByTagName('run'): name = run.getAttribute('name') @@ -268,6 +273,28 @@ def batch(script, xml): error = run_cmd(var_cmd, True, timeout) if (error == 2): print( f'***\n*** Timed out when running: {name}\n***') + + if args.emulation in runset: + for run in test.getElementsByTagName('run'): + name = run.getAttribute('name') + vram_limit = run.getAttribute('vram_min') + if vram_limit: + if OS_info["VRAM"] < float(vram_limit): + print( f'***\n*** Skipped: {name} due to VRAM req.\n***') + continue + if name: + print( f'***\n*** Running: {name}\n***') + time_limit = run.getAttribute('time_max') + if time_limit: + timeout = float(time_limit) + else: + timeout = 0 + + raw_cmd = run.firstChild.data + var_cmd = raw_cmd.format_map(var_subs) + error = run_cmd(var_cmd, True, timeout) + if (error == 2): + print( f'***\n*** Timed out when running: {name}\n***') else: error = run_cmd(cmd) fail = fail or error From 754ea9933c59beea6d3582d7f2259a526bda0f3b Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Thu, 24 Oct 2024 22:52:42 +0000 Subject: [PATCH 07/12] updated changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a01c428d5..05a1a666c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ Documentation for rocRAND is available at ### Added * rtest.py, a python script to run unit tests * added a smoke test option. Smoke tests can now be ran with python3 rtest.py -t/--test smoke + * added emulation test option. Smoke tests can now be ran with python3 rtest.py -e/--emulation=smoke + + + ## rocRAND 3.2.0 for ROCm 6.3.0 From 0319c1e1fc1a663ce95c74405169ad99577d4f64 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Fri, 25 Oct 2024 16:52:15 +0000 Subject: [PATCH 08/12] switched to inclusion filter instead of exclusion --- rtest.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtest.xml b/rtest.xml index 95b12210e..ae4c0e1ea 100644 --- a/rtest.xml +++ b/rtest.xml @@ -2,7 +2,7 @@ - + {CTEST_FILTER} {CTEST_REGEX} From 0894a4fb02620ab2ae57d2aa91ddd05ae9054dee Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Mon, 28 Oct 2024 17:22:05 +0000 Subject: [PATCH 09/12] updated so that user can not use --emulation and --test at the same time --- rtest.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rtest.py b/rtest.py index 501c7643d..029b5665d 100644 --- a/rtest.py +++ b/rtest.py @@ -31,9 +31,9 @@ def parse_args(): Checks build arguments """) parser.add_argument('-e', '--emulation', required=False, default="", - help='Run emulation test sets (optional, eg.smoke). At least one of -e or -t must be set') + help='Test set to run from rtest.xml (optional, eg.smoke). At least one but not both of -e or -t must be set') parser.add_argument('-t', '--test', required=False, default="", - help='Test set to run from rtest.xml (optional, e.g. osdb). At least one of -e or -t must be set') + help='Test set to run from rtest.xml (optional, e.g. osdb). At least one but not both of -e or -t must be set') parser.add_argument('-g', '--debug', required=False, default=False, action='store_true', help='Test Debug build (optional, default: false)') parser.add_argument('-o', '--output', type=str, required=False, default="xml", @@ -250,7 +250,9 @@ def batch(script, xml): sets = test.getAttribute('sets') runset = sets.split(',') - assert(args.test != '' or args.emulation != '') + A, B = args.test != '', args.emulation != '' + if not (A ^ B): + raise ValueError('At least one but not both of -e/--emulation or -t/--test must be set') if args.test in runset: for run in test.getElementsByTagName('run'): From c8e452badaa8c33caa940cf3b594b90c84d00ee5 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Fri, 1 Nov 2024 21:22:52 +0000 Subject: [PATCH 10/12] updated changelog to be more consistent with other libraries --- CHANGELOG.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f173b8eb3..4e0301227 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,12 @@ Documentation for rocRAND is available at ## (Unreleased) rocRAND 3.2.0 for ROCm 6.4 ### Added -* rtest.py, a python script to run unit tests - * added a smoke test option. To run the smoke tests, use `python3 rtest.py -t/--test smoke`. - * added emulation test option. I.e `python3 -e/--emulation=smoke` +* Added smoke test options, which runs a subset of the unit tests and ensuring that less than 2gb of VRAM will be used + * Smoke tests can be ran with `[--emulation|-e|--test|-t]=smoke` +* Added `--emulation` option added for `rtest.py` +### Changed +* `--test|-t` no longer a required flag for `rtest.py`, instead user can use either `--emulation|-e` or `--test|-t` but not both ## rocRAND 3.2.0 for ROCm 6.3.0 From a7ee51e2981ed102e54f2a7fddd060b9ad0e8d12 Mon Sep 17 00:00:00 2001 From: Di Nguyen Date: Mon, 4 Nov 2024 11:04:27 -0700 Subject: [PATCH 11/12] Update CHANGELOG.md Co-authored-by: Jeffrey Novotny --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e0301227..376a9fcd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,10 @@ Documentation for rocRAND is available at ## (Unreleased) rocRAND 3.2.0 for ROCm 6.4 ### Added -* Added smoke test options, which runs a subset of the unit tests and ensuring that less than 2gb of VRAM will be used - * Smoke tests can be ran with `[--emulation|-e|--test|-t]=smoke` -* Added `--emulation` option added for `rtest.py` + +* Added smoke test options, which runs a subset of the unit tests and ensures that less than 2gb of VRAM will be used + * Smoke tests can be run using `[--emulation|-e|--test|-t]=smoke` +* Added `--emulation` option for `rtest.py` ### Changed * `--test|-t` no longer a required flag for `rtest.py`, instead user can use either `--emulation|-e` or `--test|-t` but not both From 73d37adad7faa598ed5eaf02cd22a7f1351eb1cd Mon Sep 17 00:00:00 2001 From: Di Nguyen Date: Mon, 4 Nov 2024 11:04:33 -0700 Subject: [PATCH 12/12] Update CHANGELOG.md Co-authored-by: Jeffrey Novotny --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 376a9fcd3..8d8afd686 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,8 @@ Documentation for rocRAND is available at * Added `--emulation` option for `rtest.py` ### Changed -* `--test|-t` no longer a required flag for `rtest.py`, instead user can use either `--emulation|-e` or `--test|-t` but not both + +* `--test|-t` is no longer a required flag for `rtest.py`. Instead, the user can use either `--emulation|-e` or `--test|-t`, but not both. ## rocRAND 3.2.0 for ROCm 6.3.0