From ce4afa8e477b546914c1023a0dce945aa177883e Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Wed, 20 Nov 2024 13:19:26 -0700 Subject: [PATCH 01/14] make rmake compatible with linux --- rmake.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/rmake.py b/rmake.py index 6b7dccae9..27fefeed8 100644 --- a/rmake.py +++ b/rmake.py @@ -10,6 +10,7 @@ import ctypes import pathlib from fnmatch import fnmatchcase +import shutil args = {} param = {} @@ -106,12 +107,13 @@ def config_cmd(): cmake_options.append( generator ) else: rocm_path = os.getenv( 'ROCM_PATH', "/opt/rocm") - if (OS_info["ID"] in ['centos', 'rhel']): - cmake_executable = "cmake3" - else: - cmake_executable = "cmake" + + cmake_executable = 'cmake' + if shutil.which('cmake3') is not None: + cmake_executable = 'cmake3' + toolchain = "toolchain-linux.cmake" - cmake_platform_opts = f"-DROCM_DIR:PATH={rocm_path} -DCPACK_PACKAGING_INSTALL_PREFIX={rocm_path}" + cmake_platform_opts = [f"-DROCM_DIR:PATH={rocm_path}", f"-DCPACK_PACKAGING_INSTALL_PREFIX={rocm_path}"] tools = f"-DCMAKE_TOOLCHAIN_FILE={toolchain}" cmake_options.append( tools ) From 160b85009fe9349c040061abfa4ae56a8640433a Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Wed, 20 Nov 2024 13:48:38 -0700 Subject: [PATCH 02/14] updated tool chain and cmakeList such that CMAKE_PREFIX_PATH is defined --- CMakeLists.txt | 4 +++- toolchain-linux.cmake | 18 +++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b667a43ba..c9616ba2c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -204,7 +204,9 @@ if(WIN32) else() set(CMAKE_INSTALL_PREFIX "$ENV{ROCM_PATH}" CACHE PATH "Install path prefix, prepended onto install directories") #Adding CMAKE_PREFIX_PATH - list( APPEND CMAKE_PREFIX_PATH $ENV{ROCM_PATH}/llvm $ENV{ROCM_PATH} ) + if(NOT DEFINED CMAKE_PREFIX_PATH) + list( APPEND CMAKE_PREFIX_PATH $ENV{ROCM_PATH}/llvm $ENV{ROCM_PATH}) + endif() if(NOT CPACK_PACKAGING_INSTALL_PREFIX) set(CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") endif() diff --git a/toolchain-linux.cmake b/toolchain-linux.cmake index 4f06a3915..2298fef63 100644 --- a/toolchain-linux.cmake +++ b/toolchain-linux.cmake @@ -2,12 +2,20 @@ #set(CMAKE_GENERATOR "Ninja") # Ninja doesn't support platform #set(CMAKE_GENERATOR_PLATFORM x64) +if (NOT python) + set(python "python3") # default for linux +endif() -if (DEFINED ENV{ROCM_PATH}) - set(rocm_bin "$ENV{ROCM_PATH}/bin") -else() - set(ROCM_PATH "/opt/rocm" CACHE PATH "Path to the ROCm installation.") - set(rocm_bin "/opt/rocm/bin") + +if (NOT DEFINED ENV{ROCM_PATH}) + set(ENV{ROCM_PATH} "/opt/rocm" CACHE PATH "Path to the ROCm installation.") +endif() + +message("$ENV{ROCM_PATH}") + +set(rocm_bin "$ENV{ROCM_PATH}/bin") +if (NOT DEFINED CMAKE_PREFIX_PATH) + list( APPEND CMAKE_PREFIX_PATH $ENV{ROCM_PATH}/llvm $ENV{ROCM_PATH}) endif() if (NOT DEFINED ENV{CXX}) From ac5f069d0fdab03ce1fa2db6ffb206b1f6990b49 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Wed, 20 Nov 2024 14:01:45 -0700 Subject: [PATCH 03/14] implemented default paramaters --- CMakeLists.txt | 62 +++++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c9616ba2c..cde9942f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,22 +29,51 @@ option(BUILD_FORTRAN_WRAPPER "Build Fortran wrapper" OFF) option(BUILD_TEST "Build tests (requires googletest)" OFF) option(BUILD_BENCHMARK "Build benchmarks" OFF) cmake_dependent_option(BUILD_BENCHMARK_TUNING - "Build extra benchmarks for kernel configuration tuning" OFF "BUILD_BENCHMARK" OFF) +"Build extra benchmarks for kernel configuration tuning" OFF "BUILD_BENCHMARK" OFF) option(BUILD_ADDRESS_SANITIZER "Build with address sanitizer enabled" OFF) option(CODE_COVERAGE "Build with code coverage flags (clang only)" OFF) option(DEPENDENCIES_FORCE_DOWNLOAD "Don't search the system for dependencies, always download them" OFF) cmake_dependent_option(RUN_SLOW_TESTS "Run extra tests with CTest. These cover niche functionality and take long time" OFF "BUILD_TEST" OFF) + +if (NOT DEFINED ENV{ROCM_PATH}) + set(ENV{ROCM_PATH} "/opt/rocm" CACHE PATH "Path to the ROCm installation.") + endif() + + if (NOT DEFINED ENV{CXX}) + set(ENV{CXX} "$ENV{ROCM_PATH}/bin/amdclang++" CACHE PATH "Path to the C++ compiler.") +endif() + # Install prefix -set(CMAKE_INSTALL_PREFIX "/opt/rocm" CACHE PATH "Install path prefix, prepended onto install directories") +set(CMAKE_INSTALL_PREFIX "$ENV{ROCM_PATH}" CACHE PATH "Install path prefix, prepended onto install directories") + +if(WIN32) + set(CPACK_SOURCE_GENERATOR "ZIP") + set(CPACK_GENERATOR "ZIP") + set(CMAKE_INSTALL_PREFIX "C:/hipSDK" CACHE PATH "Install path") + set(INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") + set(CPACK_SET_DESTDIR OFF) + set(CPACK_PACKAGE_INSTALL_DIRECTORY "${CMAKE_INSTALL_PREFIX}") + set(CPACK_PACKAGING_INSTALL_PREFIX "") + set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY OFF) +else() + set(CMAKE_INSTALL_PREFIX "$ENV{ROCM_PATH}" CACHE PATH "Install path prefix, prepended onto install directories") + #Adding CMAKE_PREFIX_PATH + if(NOT DEFINED CMAKE_PREFIX_PATH) + list( APPEND CMAKE_PREFIX_PATH $ENV{ROCM_PATH}/llvm $ENV{ROCM_PATH}) + endif() + if(NOT CPACK_PACKAGING_INSTALL_PREFIX) + set(CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") + endif() +endif() # CMake modules list(APPEND CMAKE_MODULE_PATH - ${CMAKE_CURRENT_SOURCE_DIR}/cmake - ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules - $ENV{ROCM_PATH}/lib/cmake/hip - ${HIP_PATH}/cmake $ENV{ROCM_PATH}/hip/cmake # FindHIP.cmake - $ENV{ROCM_PATH}/llvm +${CMAKE_CURRENT_SOURCE_DIR}/cmake +${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules +$ENV{ROCM_PATH}/lib/cmake/hip +${HIP_PATH}/cmake $ENV{ROCM_PATH}/hip/cmake # FindHIP.cmake +$ENV{ROCM_PATH}/llvm ) # @@ -192,25 +221,6 @@ endif() # Package (make package) set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.txt") set(CPACK_RPM_PACKAGE_LICENSE "MIT") -if(WIN32) - set(CPACK_SOURCE_GENERATOR "ZIP") - set(CPACK_GENERATOR "ZIP") - set(CMAKE_INSTALL_PREFIX "C:/hipSDK" CACHE PATH "Install path") - set(INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") - set(CPACK_SET_DESTDIR OFF) - set(CPACK_PACKAGE_INSTALL_DIRECTORY "${CMAKE_INSTALL_PREFIX}") - set(CPACK_PACKAGING_INSTALL_PREFIX "") - set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY OFF) -else() - set(CMAKE_INSTALL_PREFIX "$ENV{ROCM_PATH}" CACHE PATH "Install path prefix, prepended onto install directories") - #Adding CMAKE_PREFIX_PATH - if(NOT DEFINED CMAKE_PREFIX_PATH) - list( APPEND CMAKE_PREFIX_PATH $ENV{ROCM_PATH}/llvm $ENV{ROCM_PATH}) - endif() - if(NOT CPACK_PACKAGING_INSTALL_PREFIX) - set(CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") - endif() -endif() if( HIP_RUNTIME_LOWER STREQUAL "rocclr" ) if(BUILD_ADDRESS_SANITIZER) From be0d34a4d328bf435316afc5cfce851511c6e157 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Wed, 20 Nov 2024 14:16:08 -0700 Subject: [PATCH 04/14] removed CXX default --- CMakeLists.txt | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cde9942f9..47187067b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,13 +37,10 @@ cmake_dependent_option(RUN_SLOW_TESTS "Run extra tests with CTest. These cover n if (NOT DEFINED ENV{ROCM_PATH}) - set(ENV{ROCM_PATH} "/opt/rocm" CACHE PATH "Path to the ROCm installation.") - endif() - - if (NOT DEFINED ENV{CXX}) - set(ENV{CXX} "$ENV{ROCM_PATH}/bin/amdclang++" CACHE PATH "Path to the C++ compiler.") +#Path to ROCm installation + set(ENV{ROCM_PATH} "/opt/rocm") endif() - + # Install prefix set(CMAKE_INSTALL_PREFIX "$ENV{ROCM_PATH}" CACHE PATH "Install path prefix, prepended onto install directories") From 8a0a0749718177dd492f998300940bdc73ce3589 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Wed, 20 Nov 2024 14:17:44 -0700 Subject: [PATCH 05/14] updated changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94ead2369..3b5077468 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ Documentation for rocRAND is available at * `--test|-t` is no longer a required flag for `rtest.py`. Instead, the user can use either `--emulation|-e` or `--test|-t`, but not both. * Removed TBB dependency for multi-core processing of host-side generation. +## Fixes + +* Fixed an issue where `CMAKE_PREFIX_PATH` was not defined properly in `CMAKELists.txt` and `toolchain-linux.cmake`. + ## rocRAND 3.2.0 for ROCm 6.3.0 ### Added From 1c32790227d73ea4071621fa98b859a0d18a9729 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Wed, 20 Nov 2024 14:25:28 -0700 Subject: [PATCH 06/14] updated changelog to include rmake fix --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b5077468..673f49542 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Documentation for rocRAND is available at ## Fixes * Fixed an issue where `CMAKE_PREFIX_PATH` was not defined properly in `CMAKELists.txt` and `toolchain-linux.cmake`. +* Fixed an issue where `cmake_platform_opts` in `rmake.py` was sometimes a string instead of a list. ## rocRAND 3.2.0 for ROCm 6.3.0 From 6b1466bf8e9439341d28383c6e6fd1db0581ea47 Mon Sep 17 00:00:00 2001 From: Di Nguyen Date: Wed, 20 Nov 2024 14:32:42 -0700 Subject: [PATCH 07/14] Update CHANGELOG.md Co-authored-by: Jeffrey Novotny --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 673f49542..7343ace50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ Documentation for rocRAND is available at * `--test|-t` is no longer a required flag for `rtest.py`. Instead, the user can use either `--emulation|-e` or `--test|-t`, but not both. * Removed TBB dependency for multi-core processing of host-side generation. -## Fixes +## Resolved issues * Fixed an issue where `CMAKE_PREFIX_PATH` was not defined properly in `CMAKELists.txt` and `toolchain-linux.cmake`. * Fixed an issue where `cmake_platform_opts` in `rmake.py` was sometimes a string instead of a list. From a65d0ccec471805d2fb4b75c4ad70b98589f40e0 Mon Sep 17 00:00:00 2001 From: Di Nguyen Date: Wed, 20 Nov 2024 14:32:58 -0700 Subject: [PATCH 08/14] Update CHANGELOG.md Co-authored-by: Jeffrey Novotny --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7343ace50..67c4cdf03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ Documentation for rocRAND is available at ## Resolved issues * Fixed an issue where `CMAKE_PREFIX_PATH` was not defined properly in `CMAKELists.txt` and `toolchain-linux.cmake`. -* Fixed an issue where `cmake_platform_opts` in `rmake.py` was sometimes a string instead of a list. +* Fixed an issue in `rmake.py` where `cmake_platform_opts` was sometimes a string instead of a list. ## rocRAND 3.2.0 for ROCm 6.3.0 From 1fe583e2026340485323d953111e5b20c6855286 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Thu, 21 Nov 2024 09:46:26 -0700 Subject: [PATCH 09/14] got rid of extra message call --- toolchain-linux.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/toolchain-linux.cmake b/toolchain-linux.cmake index 2298fef63..ce435596f 100644 --- a/toolchain-linux.cmake +++ b/toolchain-linux.cmake @@ -11,8 +11,6 @@ if (NOT DEFINED ENV{ROCM_PATH}) set(ENV{ROCM_PATH} "/opt/rocm" CACHE PATH "Path to the ROCm installation.") endif() -message("$ENV{ROCM_PATH}") - set(rocm_bin "$ENV{ROCM_PATH}/bin") if (NOT DEFINED CMAKE_PREFIX_PATH) list( APPEND CMAKE_PREFIX_PATH $ENV{ROCM_PATH}/llvm $ENV{ROCM_PATH}) From 6b8917f1f13c3a8e5b8bc08148ceb8efcb12ad74 Mon Sep 17 00:00:00 2001 From: Nguyen Date: Thu, 21 Nov 2024 14:14:22 -0700 Subject: [PATCH 10/14] removed part of code where the CXX_COMPILER was forced changed to g++ or cl if it was not gnu --- cmake/Dependencies.cmake | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index 09b7286ad..23aa41088 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -94,14 +94,6 @@ if(BUILD_BENCHMARK) message(FATAL_ERROR "DownloadProject.cmake doesn't support multi-configuration generators.") endif() set(GOOGLEBENCHMARK_ROOT ${CMAKE_CURRENT_BINARY_DIR}/deps/googlebenchmark CACHE PATH "") - if(NOT (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")) - # hip-clang cannot compile googlebenchmark for some reason - if(WIN32) - set(COMPILER_OVERRIDE "-DCMAKE_CXX_COMPILER=cl") - else() - set(COMPILER_OVERRIDE "-DCMAKE_CXX_COMPILER=g++") - endif() - endif() download_project( PROJ googlebenchmark From 19e6aa28c18a9b656182e924f1344e98233e50de Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Thu, 21 Nov 2024 14:31:16 -0700 Subject: [PATCH 11/14] updated changelog to include new Dependencies.cmake changes --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a1299952..8e1e763dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Documentation for rocRAND is available at ### Changed +* Removed a section in `cmake/Dependencies.cmake` that was forcing `DCMAKE_CXX_COMPILER` to be either `cl` or `g++` if the compiler was not `GNU`. * `--test|-t` is no longer a required flag for `rtest.py`. Instead, the user can use either `--emulation|-e` or `--test|-t`, but not both. * Removed TBB dependency for multi-core processing of host-side generation. From 182a23f12db10baad76370ed3bb8327db807b033 Mon Sep 17 00:00:00 2001 From: Di Nguyen Date: Thu, 21 Nov 2024 14:53:05 -0700 Subject: [PATCH 12/14] Update CHANGELOG.md Co-authored-by: Jeffrey Novotny --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e1e763dc..08ff59a48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ Documentation for rocRAND is available at ### Changed -* Removed a section in `cmake/Dependencies.cmake` that was forcing `DCMAKE_CXX_COMPILER` to be either `cl` or `g++` if the compiler was not `GNU`. +* Removed a section in `cmake/Dependencies.cmake` that was forcing `DCMAKE_CXX_COMPILER` to be set to either `cl` or `g++` if the compiler was not `GNU`. * `--test|-t` is no longer a required flag for `rtest.py`. Instead, the user can use either `--emulation|-e` or `--test|-t`, but not both. * Removed TBB dependency for multi-core processing of host-side generation. From 54098389b6083cbf87beca776a8f0eeb51a4b6cd Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Fri, 29 Nov 2024 14:48:48 -0700 Subject: [PATCH 13/14] updated to use google benchmark version 1.8 and cmake version 3.16-3.25 --- CMakeLists.txt | 1 + cmake/Dependencies.cmake | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c7e98f97..ff8bab44a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,7 @@ # SOFTWARE. cmake_minimum_required(VERSION 3.16 FATAL_ERROR) +cmake_policy(VERSION 3.16...3.25) # Build options include(CMakeDependentOption) diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index 23aa41088..36c604440 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -80,7 +80,6 @@ if(BUILD_TEST) endif() endif() - # Benchmark dependencies if(BUILD_BENCHMARK) if(NOT DEPENDENCIES_FORCE_DOWNLOAD) @@ -94,11 +93,12 @@ if(BUILD_BENCHMARK) message(FATAL_ERROR "DownloadProject.cmake doesn't support multi-configuration generators.") endif() set(GOOGLEBENCHMARK_ROOT ${CMAKE_CURRENT_BINARY_DIR}/deps/googlebenchmark CACHE PATH "") - + option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." OFF) + option(BENCHMARK_ENABLE_INSTALL "Enable installation of benchmark." OFF) download_project( PROJ googlebenchmark GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG v1.6.1 + GIT_TAG v1.8.0 INSTALL_DIR ${GOOGLEBENCHMARK_ROOT} CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF -DBENCHMARK_ENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX= -DCMAKE_CXX_STANDARD=14 ${COMPILER_OVERRIDE} LOG_DOWNLOAD TRUE @@ -108,6 +108,8 @@ if(BUILD_BENCHMARK) BUILD_PROJECT TRUE UPDATE_DISCONNECTED TRUE ) + set(HAVE_STD_REGEX ON) + set(RUN_HAVE_STD_REGEX 1) endif() find_package(benchmark REQUIRED CONFIG PATHS ${GOOGLEBENCHMARK_ROOT} NO_DEFAULT_PATH) endif() From db115db376e31e1bf05c3f586cbe7e9218f0ea3e Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Fri, 29 Nov 2024 14:49:26 -0700 Subject: [PATCH 14/14] got rid of run.error_occurred as its no longer supported with google benchmark 1.8 --- benchmark/custom_csv_formater.hpp | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/benchmark/custom_csv_formater.hpp b/benchmark/custom_csv_formater.hpp index 75d05fa6d..204512026 100644 --- a/benchmark/custom_csv_formater.hpp +++ b/benchmark/custom_csv_formater.hpp @@ -152,20 +152,13 @@ inline void customCSVReporter::ReportRuns(const std::vector& reports) inline void customCSVReporter::PrintRunData(const Run& run) { std::ostream& Out = GetOutputStream(); - std::ostream& Err = GetErrorStream(); //get the name of the engine and distribution: - std::string temp = run.benchmark_name(); - std::string deviceName = std::string(temp.begin(), temp.begin() + temp.find("<")); - temp.erase(0, temp.find("<") + 1); - std::string engineName = std::string(temp.begin(), temp.begin() + temp.find(",")); - temp.erase(0, engineName.size() + 1); - std::string mode = "default"; if(deviceName != "device_kernel") @@ -174,7 +167,6 @@ inline void customCSVReporter::PrintRunData(const Run& run) temp.erase(0, temp.find(",") + 1); } std::string disName = std::string(temp.begin(), temp.begin() + temp.find(">")); - std::string lambda = ""; size_t ePos = disName.find("="); @@ -186,13 +178,6 @@ inline void customCSVReporter::PrintRunData(const Run& run) Out << engineName << "," << disName << "," << mode << ","; Out << CsvEscape(run.benchmark_name()) << ","; - if(run.error_occurred) - { - Err << std::string(elements.size() - 3, ','); - Err << "true,"; - Err << CsvEscape(run.error_message) << "\n"; - return; - } // Do not print iteration on bigO and RMS report if(!run.report_big_o && !run.report_rms)