From d0a086142b966a70ae0fbcf3813d042ec5a4c55b Mon Sep 17 00:00:00 2001 From: thansen-amd Date: Thu, 12 Oct 2023 10:32:22 -0600 Subject: [PATCH 1/5] add asan setup and change to settings map --- .jenkins/asan.groovy | 80 ++++++++++++++++++++++++++++++++++ .jenkins/codecov.groovy | 4 +- .jenkins/common.groovy | 13 +++--- .jenkins/precheckin.groovy | 4 +- .jenkins/staticanalysis.groovy | 4 +- 5 files changed, 96 insertions(+), 9 deletions(-) create mode 100644 .jenkins/asan.groovy diff --git a/.jenkins/asan.groovy b/.jenkins/asan.groovy new file mode 100644 index 00000000..3be13ab2 --- /dev/null +++ b/.jenkins/asan.groovy @@ -0,0 +1,80 @@ +#!/usr/bin/env groovy +@Library('rocJenkins@pong') _ +import com.amd.project.* +import com.amd.docker.* +import java.nio.file.Path; + +def runCI = +{ + nodeDetails, jobName-> + + def prj = new rocProject('rocRAND', 'address-sanitizer') + + def nodes = new dockerNodes(nodeDetails, jobName, prj) + + def commonGroovy + + def settings = [addressSanitizer: true] + + boolean formatCheck = false + + def compileCommand = + { + platform, project-> + + commonGroovy = load "${project.paths.project_src_prefix}/.jenkins/common.groovy" + commonGroovy.runCompileCommand(platform, project, jobName, settings) + } + + + def testCommand = + { + platform, project-> + + commonGroovy.runTestCommand(platform, project) + } + + def packageCommand = + { + platform, project-> + + commonGroovy.runPackageCommand(platform, project) + } + + buildProject(prj, formatCheck, nodes.dockerArray, compileCommand, testCommand, packageCommand) +} + +ci: { + String urlJobName = auxiliary.getTopJobName(env.BUILD_URL) + + def propertyList = ["compute-rocm-dkms-no-npi-hipclang":[pipelineTriggers([cron('0 1 * * 0')])]] + propertyList = auxiliary.appendPropertyList(propertyList) + + def jobNameList = ["compute-rocm-dkms-no-npi-hipclang":([ubuntu18:['gfx900'],centos7:['gfx906'],centos8:['gfx906'],sles15sp1:['gfx908']])] + jobNameList = auxiliary.appendJobNameList(jobNameList) + + propertyList.each + { + jobName, property-> + if (urlJobName == jobName) + properties(auxiliary.addCommonProperties(property)) + } + + jobNameList.each + { + jobName, nodeDetails-> + if (urlJobName == jobName) + stage(jobName) { + runCI(nodeDetails, jobName) + } + } + + // For url job names that are not listed by the jobNameList i.e. compute-rocm-dkms-no-npi-1901 + if(!jobNameList.keySet().contains(urlJobName)) + { + properties(auxiliary.addCommonProperties([pipelineTriggers([cron('0 1 * * *')])])) + stage(urlJobName) { + runCI([ubuntu16:['gfx906']], urlJobName) + } + } +} \ No newline at end of file diff --git a/.jenkins/codecov.groovy b/.jenkins/codecov.groovy index ef6c6a7d..616f37af 100644 --- a/.jenkins/codecov.groovy +++ b/.jenkins/codecov.groovy @@ -14,6 +14,8 @@ def runCI = def commonGroovy + def settings = [codeCoverage: true] + boolean formatCheck = false def compileCommand = @@ -21,7 +23,7 @@ def runCI = platform, project-> commonGroovy = load "${project.paths.project_src_prefix}/.jenkins/common.groovy" - commonGroovy.runCompileCommand(platform, project, jobName, false, false, true) + commonGroovy.runCompileCommand(platform, project, jobName, settings) } def testCommand = diff --git a/.jenkins/common.groovy b/.jenkins/common.groovy index d157cbb3..a656a014 100644 --- a/.jenkins/common.groovy +++ b/.jenkins/common.groovy @@ -1,15 +1,16 @@ // This file is for internal AMD use. // If you are interested in running your own Jenkins, please raise a github issue for assistance. -def runCompileCommand(platform, project, jobName, boolean debug=false, boolean staticLibrary=false, boolean codeCoverage=false) +def runCompileCommand(platform, project, jobName, settings) { project.paths.construct_build_prefix() project.paths.build_command = './install -c' - String buildTypeArg = debug ? '-DCMAKE_BUILD_TYPE=Debug' : '-DCMAKE_BUILD_TYPE=Release' - String buildTypeDir = debug ? 'debug' : 'release' - String buildStatic = staticLibrary ? '-DBUILD_STATIC_LIBS=ON' : '-DBUILD_SHARED=OFF' - String codeCovFlag = codeCoverage ? '-DCODE_COVERAGE=ON' : '' + String buildTypeArg = settings.debug ? '-DCMAKE_BUILD_TYPE=Debug' : '-DCMAKE_BUILD_TYPE=Release' + String buildTypeDir = settings.debug ?: 'release' + String buildStatic = settings.staticLibrary ? '-DBUILD_STATIC_LIBS=ON' : '-DBUILD_SHARED=OFF' + String codeCovFlag = settings.codeCoverage ? '-DCODE_COVERAGE=ON' : '' + String asanFlag = settings.addressSanitizer ? '-DBUILD_ADDRESS_SANITIZER=ON' : '' String cmake = platform.jenkinsLabel.contains('centos') ? 'cmake3' : 'cmake' //Set CI node's gfx arch as target if PR, otherwise use default targets of the library String amdgpuTargets = env.BRANCH_NAME.startsWith('PR-') ? '-DAMDGPU_TARGETS=\$gfx_arch' : '' @@ -20,7 +21,7 @@ def runCompileCommand(platform, project, jobName, boolean debug=false, boolean s mkdir -p build/${buildTypeDir} && cd build/${buildTypeDir} # gfxTargetParser reads gfxarch and adds target features such as xnack ${auxiliary.gfxTargetParser()} - ${cmake} -DCMAKE_CXX_COMPILER=/opt/rocm/bin/hipcc ${buildTypeArg} ${buildStatic} ${amdgpuTargets} ${codeCovFlag} -DBUILD_TEST=ON -DBUILD_BENCHMARK=ON ../.. + ${cmake} -DCMAKE_CXX_COMPILER=/opt/rocm/bin/hipcc ${buildTypeArg} ${buildStatic} ${amdgpuTargets} ${codeCovFlag} ${asanFlag} -DBUILD_TEST=ON -DBUILD_BENCHMARK=ON ../.. make -j\$(nproc) """ diff --git a/.jenkins/precheckin.groovy b/.jenkins/precheckin.groovy index d067fd85..b31aabc5 100644 --- a/.jenkins/precheckin.groovy +++ b/.jenkins/precheckin.groovy @@ -14,6 +14,8 @@ def runCI = def commonGroovy + def settings = [:] + boolean formatCheck = false def compileCommand = @@ -21,7 +23,7 @@ def runCI = platform, project-> commonGroovy = load "${project.paths.project_src_prefix}/.jenkins/common.groovy" - commonGroovy.runCompileCommand(platform, project, jobName) + commonGroovy.runCompileCommand(platform, project, jobName, settings) } diff --git a/.jenkins/staticanalysis.groovy b/.jenkins/staticanalysis.groovy index 1bb7b4fa..789a6f3f 100644 --- a/.jenkins/staticanalysis.groovy +++ b/.jenkins/staticanalysis.groovy @@ -16,6 +16,8 @@ def runCompileCommand(platform, project, jobName, boolean debug=false) def runCI = { + def settings = [:] + nodeDetails, jobName-> def prj = new rocProject('rocRAND', 'StaticAnalysis') @@ -30,7 +32,7 @@ def runCI = { platform, project-> - runCompileCommand(platform, project, jobName, false) + runCompileCommand(platform, project, jobName, settings) } buildProject(prj , formatCheck, nodes.dockerArray, compileCommand, null, null, staticAnalysis) From 67ddaf9ccf8a3a31a44acb3c92aec20fc7b36726 Mon Sep 17 00:00:00 2001 From: thansen-amd Date: Thu, 12 Oct 2023 11:11:21 -0600 Subject: [PATCH 2/5] staticanalysis settings --- .jenkins/staticanalysis.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/staticanalysis.groovy b/.jenkins/staticanalysis.groovy index 789a6f3f..6a08920f 100644 --- a/.jenkins/staticanalysis.groovy +++ b/.jenkins/staticanalysis.groovy @@ -9,7 +9,7 @@ import com.amd.project.* import com.amd.docker.* import java.nio.file.Path -def runCompileCommand(platform, project, jobName, boolean debug=false) +def runCompileCommand(platform, project, jobName, settings) { project.paths.construct_build_prefix() } From 19b6878c8a8057d50239cdf61b9852a78a0d658f Mon Sep 17 00:00:00 2001 From: thansen-amd Date: Tue, 17 Oct 2023 09:10:59 -0600 Subject: [PATCH 3/5] link test command with asan files --- .jenkins/asan.groovy | 2 +- .jenkins/common.groovy | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.jenkins/asan.groovy b/.jenkins/asan.groovy index 3be13ab2..1ef93ece 100644 --- a/.jenkins/asan.groovy +++ b/.jenkins/asan.groovy @@ -31,7 +31,7 @@ def runCI = { platform, project-> - commonGroovy.runTestCommand(platform, project) + commonGroovy.runASANTestCommand(platform, project) } def packageCommand = diff --git a/.jenkins/common.groovy b/.jenkins/common.groovy index a656a014..4451d8b1 100644 --- a/.jenkins/common.groovy +++ b/.jenkins/common.groovy @@ -46,6 +46,30 @@ def runTestCommand (platform, project) platform.runCommand(this, command) } +def runASANTestCommand (platform, project) +{ + //Temporary workaround due to bug in container + String centos7Workaround = platform.jenkinsLabel.contains('centos7') ? 'export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/opt/rocm/lib64/' : '' + + String sudo = auxiliary.sudo(platform.jenkinsLabel) + // String centos = platform.jenkinsLabel.contains('centos') ? '3' : '' + // Disable xorwow test for now as it is a known failure with gfx90a. + // def testCommand = "ctest${centos} --output-on-failure" + def testCommand = "ctest --output-on-failure" + + def command = """#!/usr/bin/env bash + set -x + cd ${project.paths.project_build_prefix}/build/release + make -j4 + export ASAN_LIB_PATH=\$(/opt/rocm/llvm/bin/clang -print-file-name=libclang_rt.asan-x86_64.so) + export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:\$(dirname "\${ASAN_LIB_PATH}") + ${centos7Workaround} + ${sudo} ${testCommand} + """ + + platform.runCommand(this, command) +} + def runPackageCommand(platform, project) { def packageHelper = platform.makePackage(platform.jenkinsLabel,"${project.paths.project_build_prefix}/build/release") From 16930dd4b8e7b16c66ff6faef8e60af8d5f6efcd Mon Sep 17 00:00:00 2001 From: thansen-amd Date: Tue, 17 Oct 2023 09:50:40 -0600 Subject: [PATCH 4/5] test command to settings map --- .jenkins/asan.groovy | 2 +- .jenkins/common.groovy | 32 ++++++-------------------------- .jenkins/precheckin.groovy | 2 +- 3 files changed, 8 insertions(+), 28 deletions(-) diff --git a/.jenkins/asan.groovy b/.jenkins/asan.groovy index 1ef93ece..71fbc01c 100644 --- a/.jenkins/asan.groovy +++ b/.jenkins/asan.groovy @@ -31,7 +31,7 @@ def runCI = { platform, project-> - commonGroovy.runASANTestCommand(platform, project) + commonGroovy.runTestCommand(platform, project, settings) } def packageCommand = diff --git a/.jenkins/common.groovy b/.jenkins/common.groovy index 4451d8b1..d72e1ed7 100644 --- a/.jenkins/common.groovy +++ b/.jenkins/common.groovy @@ -28,7 +28,7 @@ def runCompileCommand(platform, project, jobName, settings) platform.runCommand(this, command) } -def runTestCommand (platform, project) +def runTestCommand (platform, project, settings) { String sudo = auxiliary.sudo(platform.jenkinsLabel) // String centos = platform.jenkinsLabel.contains('centos') ? '3' : '' @@ -36,40 +36,20 @@ def runTestCommand (platform, project) // def testCommand = "ctest${centos} --output-on-failure" def testCommand = "ctest --output-on-failure" + String LD_PATH = settings.addressSanitizer ? 'export ASAN_LIB_PATH=\$(/opt/rocm/llvm/bin/clang -print-file-name=libclang_rt.asan-x86_64.so) \ + export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:\$(dirname "\${ASAN_LIB_PATH}")' : 'export LD_LIBRARY_PATH=/opt/rocm/lib/' + def command = """#!/usr/bin/env bash set -x cd ${project.paths.project_build_prefix}/build/release make -j4 - ${sudo} LD_LIBRARY_PATH=/opt/rocm/lib/ ${testCommand} + ${LD_PATH} + ${sudo} ${testCommand} """ platform.runCommand(this, command) } -def runASANTestCommand (platform, project) -{ - //Temporary workaround due to bug in container - String centos7Workaround = platform.jenkinsLabel.contains('centos7') ? 'export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/opt/rocm/lib64/' : '' - - String sudo = auxiliary.sudo(platform.jenkinsLabel) - // String centos = platform.jenkinsLabel.contains('centos') ? '3' : '' - // Disable xorwow test for now as it is a known failure with gfx90a. - // def testCommand = "ctest${centos} --output-on-failure" - def testCommand = "ctest --output-on-failure" - - def command = """#!/usr/bin/env bash - set -x - cd ${project.paths.project_build_prefix}/build/release - make -j4 - export ASAN_LIB_PATH=\$(/opt/rocm/llvm/bin/clang -print-file-name=libclang_rt.asan-x86_64.so) - export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:\$(dirname "\${ASAN_LIB_PATH}") - ${centos7Workaround} - ${sudo} ${testCommand} - """ - - platform.runCommand(this, command) -} - def runPackageCommand(platform, project) { def packageHelper = platform.makePackage(platform.jenkinsLabel,"${project.paths.project_build_prefix}/build/release") diff --git a/.jenkins/precheckin.groovy b/.jenkins/precheckin.groovy index b31aabc5..386e56fc 100644 --- a/.jenkins/precheckin.groovy +++ b/.jenkins/precheckin.groovy @@ -31,7 +31,7 @@ def runCI = { platform, project-> - commonGroovy.runTestCommand(platform, project) + commonGroovy.runTestCommand(platform, project, settings) } def packageCommand = From 3712950a4b9cb74fe7eb35b5138707e53fb6dabf Mon Sep 17 00:00:00 2001 From: thansen-amd Date: Tue, 17 Oct 2023 11:11:35 -0600 Subject: [PATCH 5/5] string literal for multiple commands --- .jenkins/common.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.jenkins/common.groovy b/.jenkins/common.groovy index d72e1ed7..9cc362a4 100644 --- a/.jenkins/common.groovy +++ b/.jenkins/common.groovy @@ -36,8 +36,8 @@ def runTestCommand (platform, project, settings) // def testCommand = "ctest${centos} --output-on-failure" def testCommand = "ctest --output-on-failure" - String LD_PATH = settings.addressSanitizer ? 'export ASAN_LIB_PATH=\$(/opt/rocm/llvm/bin/clang -print-file-name=libclang_rt.asan-x86_64.so) \ - export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:\$(dirname "\${ASAN_LIB_PATH}")' : 'export LD_LIBRARY_PATH=/opt/rocm/lib/' + String LD_PATH = settings.addressSanitizer ? """export ASAN_LIB_PATH=\$(/opt/rocm/llvm/bin/clang -print-file-name=libclang_rt.asan-x86_64.so) + export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:\$(dirname "\${ASAN_LIB_PATH}")""" : 'export LD_LIBRARY_PATH=/opt/rocm/lib/' def command = """#!/usr/bin/env bash set -x