Skip to content

Commit

Permalink
Add run by function name in CLI (#682)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ladisgin
ladisgin authored Jun 6, 2024
1 parent e768496 commit b9eb026
Show file tree
Hide file tree
Showing 9 changed files with 200 additions and 120 deletions.
1 change: 1 addition & 0 deletions server/proto/testgen.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,7 @@ message TestFilter {
string testFilePath = 1;
string testName = 2;
string testSuite = 3;
string functionName = 4;
}

message CoverageAndResultsRequest {
Expand Down
27 changes: 25 additions & 2 deletions server/src/commands/Commands.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -443,6 +443,7 @@ Commands::RunTestsCommands::RunTestsCommands(Commands::MainCommands &commands) {
runCommand = commands.getRunTestsCommand();

runTestCommand = runCommand->add_subcommand("test", "Run specified test");
runFunctionCommand = runCommand->add_subcommand("function", "Run specified function tests");
runFileCommand = runCommand->add_subcommand("file", "Run all tests in specified file");
runProjectCommand = runCommand->add_subcommand("project", "Run all tests for this project");
}
Expand All @@ -459,10 +460,18 @@ CLI::App *Commands::RunTestsCommands::getRunTestCommand() {
return runTestCommand;
}

CLI::App *Commands::RunTestsCommands::getRunFunctionCommand() {
return runFunctionCommand;
}

bool Commands::RunTestsCommands::gotRunTestCommand() {
return runCommand->got_subcommand(runTestCommand);
}

bool Commands::RunTestsCommands::gotRunFunctionCommand() {
return runCommand->got_subcommand(runFunctionCommand);
}

bool Commands::RunTestsCommands::gotRunFileCommand() {
return runCommand->got_subcommand(runFileCommand);
}
Expand All @@ -477,13 +486,23 @@ Commands::RunTestsCommandOptions::RunTestsCommandOptions(Commands::RunTestsComma
commands.getRunTestCommand()
->add_option("--file-path", filePath, "Path to test file")
->required();
commands.getRunTestCommand()->add_flag("--no-coverage", noCoverage,
"Flag that controls coverage generation.");


commands.getRunFunctionCommand()
->add_option("--file-path", filePath, "Path to test file")
->required();
commands.getRunFunctionCommand()->add_option("--function-name", functionName, "Test name")->required();
commands.getRunFunctionCommand()->add_flag("--no-coverage", noCoverage,
"Flag that controls coverage generation.");

commands.getRunFileCommand()
->add_option("--file-path", filePath, "Path to test file")
->required();
commands.getRunTestCommand()->add_flag("--no-coverage", noCoverage,
"Flag that controls coverage generation.");
commands.getRunFileCommand()->add_flag("--no-coverage", noCoverage,
"Flag that controls coverage generation.");

commands.getRunProjectCommand()->add_flag("--no-coverage", noCoverage,
"Flag that controls coverage generation.");
}
Expand All @@ -500,6 +519,10 @@ std::string Commands::RunTestsCommandOptions::getTestName() {
return testName;
}

std::string Commands::RunTestsCommandOptions::getFunctionName() {
return functionName;
}

bool Commands::RunTestsCommandOptions::withCoverage() const {
return !noCoverage;
}
Expand Down
8 changes: 8 additions & 0 deletions server/src/commands/Commands.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,12 +177,16 @@ namespace Commands {

CLI::App *getRunTestCommand();

CLI::App *getRunFunctionCommand();

CLI::App *getRunFileCommand();

CLI::App *getRunProjectCommand();

bool gotRunTestCommand();

bool gotRunFunctionCommand();

bool gotRunFileCommand();

bool gotRunProjectCommand();
Expand All @@ -191,6 +195,7 @@ namespace Commands {
CLI::App *runCommand;

CLI::App *runTestCommand;
CLI::App *runFunctionCommand;
CLI::App *runFileCommand;
CLI::App *runProjectCommand;
};
Expand All @@ -204,12 +209,15 @@ namespace Commands {

std::string getTestName();

std::string getFunctionName();

[[nodiscard]] bool withCoverage() const;

private:
fs::path filePath;
std::string testSuite;
std::string testName;
std::string functionName;

bool noCoverage = false;
};
Expand Down
1 change: 1 addition & 0 deletions server/src/coverage/CoverageAndResultsGenerator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ CoverageAndResultsGenerator::CoverageAndResultsGenerator(
coverageAndResultsRequest->testfilter().testfilepath(),
coverageAndResultsRequest->testfilter().testsuite(),
coverageAndResultsRequest->testfilter().testname(),
coverageAndResultsRequest->testfilter().functionname(),
coverageAndResultsWriter),
coverageAndResultsWriter(coverageAndResultsWriter) {
}
Expand Down
164 changes: 93 additions & 71 deletions server/src/coverage/TestRunner.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#include <utils/stats/TestsExecutionStats.h>
#include <utils/KleeUtils.h>
#include "TestRunner.h"

#include "printers/DefaultMakefilePrinter.h"
Expand All @@ -18,33 +19,37 @@ TestRunner::TestRunner(utbot::ProjectContext projectContext,
std::string testFilePath,
std::string testSuite,
std::string testName,
std::string functionName,
ProgressWriter const *progressWriter)
: projectContext(std::move(projectContext)),
testFilePath(testFilePath.empty() ? std::nullopt : std::make_optional(testFilePath)),
testSuite(std::move(testSuite)), testName(std::move(testName)),
progressWriter(progressWriter) {
: projectContext(std::move(projectContext)),
testFilePath(testFilePath.empty() ? std::nullopt : std::make_optional(testFilePath)),
testSuite(std::move(testSuite)), testName(std::move(testName)), functionName(std::move(functionName)),
progressWriter(progressWriter) {
}

TestRunner::TestRunner(
const testsgen::CoverageAndResultsRequest *coverageAndResultsRequest,
grpc::ServerWriter<testsgen::CoverageAndResultsResponse> *coverageAndResultsWriter,
std::string testFilename,
std::string testSuite,
std::string testName)
: TestRunner(utbot::ProjectContext(coverageAndResultsRequest->projectcontext()),
std::move(testFilename),
std::move(testSuite),
std::move(testName),
&writer) {
const testsgen::CoverageAndResultsRequest *coverageAndResultsRequest,
grpc::ServerWriter<testsgen::CoverageAndResultsResponse> *coverageAndResultsWriter,
std::string testFilename,
std::string testSuite,
std::string testName,
std::string functionName)
: TestRunner(utbot::ProjectContext(coverageAndResultsRequest->projectcontext()),
std::move(testFilename),
std::move(testSuite),
std::move(testName),
std::move(functionName),
&writer) {
writer = ServerCoverageAndResultsWriter(coverageAndResultsWriter);
}

std::vector<UnitTest> TestRunner::getTestsFromMakefile(const fs::path &makefile,
const fs::path &testFilePath) {
const fs::path &testFilePath,
const std::string &filter) {
auto cmdGetAllTests = MakefileUtils::MakefileCommand(projectContext, makefile,
printer::DefaultMakefilePrinter::TARGET_RUN,
"--gtest_list_tests", {"GTEST_FILTER=*"});
auto[out, status, _] = cmdGetAllTests.run(projectContext.getBuildDirAbsPath(), false);
"--gtest_list_tests", {"GTEST_FILTER=" + filter});
auto [out, status, _] = cmdGetAllTests.run(projectContext.getBuildDirAbsPath(), false);
if (status != 0) {
auto [err, _, logFilePath] = cmdGetAllTests.run(projectContext.getBuildDirAbsPath(), true);
progressWriter->writeProgress(StringUtils::stringFormat("command %s failed.\n"
Expand All @@ -55,12 +60,14 @@ std::vector<UnitTest> TestRunner::getTestsFromMakefile(const fs::path &makefile,
throw ExecutionProcessException(err, logFilePath.value());
}
if (out.empty()) {
LOG_S(WARNING) << "Running gtest with flag --gtest_list_tests returns empty output. Does file contain main function?";
LOG_S(WARNING)
<< "Running gtest with flag --gtest_list_tests returns empty output. Does file contain main function?";
return {};
}
std::vector<std::string> gtestListTestsOutput = StringUtils::split(out, '\n');
gtestListTestsOutput.erase(gtestListTestsOutput.begin()); //GTEST prints "Running main() from /opt/gtest/googletest/src/gtest_main.cc"
for (std::string &s : gtestListTestsOutput) {
gtestListTestsOutput.erase(
gtestListTestsOutput.begin()); //GTEST prints "Running main() from /opt/gtest/googletest/src/gtest_main.cc"
for (std::string &s: gtestListTestsOutput) {
StringUtils::trim(s);
}
std::string testSuite;
Expand All @@ -84,50 +91,65 @@ std::vector<UnitTest> TestRunner::getTestsToLaunch() {
if (fs::exists(projectContext.getTestDirAbsPath())) {
FileSystemUtils::RecursiveDirectoryIterator directoryIterator(projectContext.getTestDirAbsPath());
ExecUtils::doWorkWithProgress(
directoryIterator, progressWriter, "Building tests",
[this, &result](fs::directory_entry const &directoryEntry) {
if (!directoryEntry.is_regular_file()) {
return;
}
const auto &testFilePath = directoryEntry.path();
if (testFilePath.extension() == Paths::CXX_EXTENSION &&
StringUtils::endsWith(testFilePath.stem().c_str(), Paths::TEST_SUFFIX)) {
fs::path sourcePath = Paths::testPathToSourcePath(projectContext, testFilePath);
fs::path makefile =
Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);
if (fs::exists(makefile)) {
try {
auto tests = getTestsFromMakefile(makefile, testFilePath);
CollectionUtils::extend(result, tests);
} catch (ExecutionProcessException const &e) {
exceptions.push_back(e);
directoryIterator, progressWriter, "Building tests",
[this, &result](fs::directory_entry const &directoryEntry) {
if (!directoryEntry.is_regular_file()) {
return;
}
const auto &testFilePath = directoryEntry.path();
if (testFilePath.extension() == Paths::CXX_EXTENSION &&
StringUtils::endsWith(testFilePath.stem().c_str(), Paths::TEST_SUFFIX)) {
fs::path sourcePath = Paths::testPathToSourcePath(projectContext, testFilePath);
fs::path makefile =
Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);
if (fs::exists(makefile)) {
try {
auto tests = getTestsFromMakefile(makefile, testFilePath);
CollectionUtils::extend(result, tests);
} catch (ExecutionProcessException const &e) {
exceptions.push_back(e);
}
} else {
LOG_S(WARNING) << StringUtils::stringFormat(
"Makefile for %s not found, candidate: %s", testFilePath, makefile);
}
} else {
LOG_S(WARNING) << StringUtils::stringFormat(
"Makefile for %s not found, candidate: %s", testFilePath, makefile);
}
} else {
if (!StringUtils::endsWith(testFilePath.stem().c_str(), Paths::TEST_SUFFIX) &&
!StringUtils::endsWith(testFilePath.stem().c_str(), Paths::STUB_SUFFIX) &&
!StringUtils::endsWith(testFilePath.stem().c_str(), Paths::MAKE_WRAPPER_SUFFIX) &&
!StringUtils::endsWith(testFilePath.c_str(), Paths::MAKEFILE_EXTENSION)) {
LOG_S(WARNING) << "Found extra file in test directory: " << testFilePath;
if (!StringUtils::endsWith(testFilePath.stem().c_str(), Paths::TEST_SUFFIX) &&
!StringUtils::endsWith(testFilePath.stem().c_str(), Paths::STUB_SUFFIX) &&
!StringUtils::endsWith(testFilePath.stem().c_str(), Paths::MAKE_WRAPPER_SUFFIX) &&
!StringUtils::endsWith(testFilePath.c_str(), Paths::MAKEFILE_EXTENSION)) {
LOG_S(WARNING) << "Found extra file in test directory: " << testFilePath;
}
}
}
});
});
} else {
LOG_S(WARNING) << "Test folder doesn't exist: " << projectContext.getTestDirAbsPath();
}
return result;
}
if (testName.empty()) {

if (testName.empty() && functionName.empty()) {
//for file
fs::path sourcePath = Paths::testPathToSourcePath(projectContext, testFilePath.value());
fs::path makefile = Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);
return getTestsFromMakefile(makefile, testFilePath.value());
}

if (testName.empty()) {
//for function
fs::path sourcePath = Paths::testPathToSourcePath(projectContext, testFilePath.value());
fs::path makefile = Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);


std::string renamedMethodDescription = KleeUtils::getRenamedOperator(functionName);
StringUtils::replaceColon(renamedMethodDescription);

std::string filter = "*." + renamedMethodDescription + Paths::TEST_SUFFIX + "*";

return getTestsFromMakefile(makefile, testFilePath.value(), filter);
}
//for single test
return { UnitTest{ testFilePath.value(), testSuite, testName } };
return {UnitTest{testFilePath.value(), testSuite, testName}};
}

grpc::Status TestRunner::runTests(bool withCoverage, const std::optional<std::chrono::seconds> &testTimeout) {
Expand All @@ -136,22 +158,22 @@ grpc::Status TestRunner::runTests(bool withCoverage, const std::optional<std::ch

const auto buildRunCommands = coverageTool->getBuildRunCommands(testsToLaunch, withCoverage);
ExecUtils::doWorkWithProgress(buildRunCommands, progressWriter, "Running tests",
[this, testTimeout] (BuildRunCommand const &buildRunCommand) {
auto const &[unitTest, buildCommand, runCommand] =
buildRunCommand;
try {
auto status = runTest(buildRunCommand, testTimeout);
testResultMap[unitTest.testFilePath][unitTest.testname] = status;
ExecUtils::throwIfCancelled();
} catch (ExecutionProcessException const &e) {
testsgen::TestResultObject testRes;
testRes.set_testfilepath(unitTest.testFilePath);
testRes.set_testname(unitTest.testname);
testRes.set_status(testsgen::TEST_FAILED);
testResultMap[unitTest.testFilePath][unitTest.testname] = testRes;
exceptions.emplace_back(e);
}
});
[this, testTimeout](BuildRunCommand const &buildRunCommand) {
auto const &[unitTest, buildCommand, runCommand] =
buildRunCommand;
try {
auto status = runTest(buildRunCommand, testTimeout);
testResultMap[unitTest.testFilePath][unitTest.testname] = status;
ExecUtils::throwIfCancelled();
} catch (ExecutionProcessException const &e) {
testsgen::TestResultObject testRes;
testRes.set_testfilepath(unitTest.testFilePath);
testRes.set_testname(unitTest.testname);
testRes.set_status(testsgen::TEST_FAILED);
testResultMap[unitTest.testFilePath][unitTest.testname] = testRes;
exceptions.emplace_back(e);
}
});
LOG_S(DEBUG) << "All run commands were executed";
return Status::OK;
}
Expand All @@ -169,14 +191,14 @@ void TestRunner::init(bool withCoverage) {
}
}

bool TestRunner::buildTest(const utbot::ProjectContext& projectContext, const fs::path& sourcePath) {
bool TestRunner::buildTest(const utbot::ProjectContext &projectContext, const fs::path &sourcePath) {
ExecUtils::throwIfCancelled();
fs::path makefile = Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);
if (fs::exists(makefile)) {
auto command = MakefileUtils::MakefileCommand(projectContext, makefile,
printer::DefaultMakefilePrinter::TARGET_BUILD, "", {});
LOG_S(DEBUG) << "Try compile tests for: " << sourcePath.string();
auto[out, status, logFilePath] = command.run(projectContext.getBuildDirAbsPath(), true);
auto [out, status, logFilePath] = command.run(projectContext.getBuildDirAbsPath(), true);
if (status != 0) {
return false;
}
Expand All @@ -185,18 +207,18 @@ bool TestRunner::buildTest(const utbot::ProjectContext& projectContext, const fs
return false;
}

size_t TestRunner::buildTests(const utbot::ProjectContext& projectContext, const tests::TestsMap& tests) {
size_t TestRunner::buildTests(const utbot::ProjectContext &projectContext, const tests::TestsMap &tests) {
size_t fail_count = 0;
for (const auto &[file, _]: tests) {
if(!TestRunner::buildTest(projectContext, file)) {
if (!TestRunner::buildTest(projectContext, file)) {
fail_count++;
}
}
return fail_count;
}

testsgen::TestResultObject TestRunner::runTest(const BuildRunCommand &command,
const std::optional <std::chrono::seconds> &testTimeout) {
const std::optional<std::chrono::seconds> &testTimeout) {
fs::remove(Paths::getGTestResultsJsonPath(projectContext));
auto res = command.runCommand.run(projectContext.getBuildDirAbsPath(), true, true, testTimeout);
GTestLogger::log(res.output);
Expand Down
Loading

0 comments on commit b9eb026

Please sign in to comment.