Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add CLI option for run function by name #682

Merged
merged 1 commit into from
Jun 6, 2024
Merged

Add CLI option for run function by name #682

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions server/proto/testgen.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,7 @@ message TestFilter {
string testFilePath = 1;
string testName = 2;
string testSuite = 3;
string functionName = 4;
}

message CoverageAndResultsRequest {
Expand Down
27 changes: 25 additions & 2 deletions server/src/commands/Commands.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -443,6 +443,7 @@ Commands::RunTestsCommands::RunTestsCommands(Commands::MainCommands &commands) {
runCommand = commands.getRunTestsCommand();

runTestCommand = runCommand->add_subcommand("test", "Run specified test");
runFunctionCommand = runCommand->add_subcommand("function", "Run specified function tests");
runFileCommand = runCommand->add_subcommand("file", "Run all tests in specified file");
runProjectCommand = runCommand->add_subcommand("project", "Run all tests for this project");
}
Expand All @@ -459,10 +460,18 @@ CLI::App *Commands::RunTestsCommands::getRunTestCommand() {
return runTestCommand;
}

CLI::App *Commands::RunTestsCommands::getRunFunctionCommand() {
return runFunctionCommand;
}

bool Commands::RunTestsCommands::gotRunTestCommand() {
return runCommand->got_subcommand(runTestCommand);
}

bool Commands::RunTestsCommands::gotRunFunctionCommand() {
return runCommand->got_subcommand(runFunctionCommand);
}

bool Commands::RunTestsCommands::gotRunFileCommand() {
return runCommand->got_subcommand(runFileCommand);
}
Expand All @@ -477,13 +486,23 @@ Commands::RunTestsCommandOptions::RunTestsCommandOptions(Commands::RunTestsComma
commands.getRunTestCommand()
->add_option("--file-path", filePath, "Path to test file")
->required();
commands.getRunTestCommand()->add_flag("--no-coverage", noCoverage,
"Flag that controls coverage generation.");


commands.getRunFunctionCommand()
->add_option("--file-path", filePath, "Path to test file")
->required();
commands.getRunFunctionCommand()->add_option("--function-name", functionName, "Test name")->required();
commands.getRunFunctionCommand()->add_flag("--no-coverage", noCoverage,
"Flag that controls coverage generation.");

commands.getRunFileCommand()
->add_option("--file-path", filePath, "Path to test file")
->required();
commands.getRunTestCommand()->add_flag("--no-coverage", noCoverage,
"Flag that controls coverage generation.");
commands.getRunFileCommand()->add_flag("--no-coverage", noCoverage,
"Flag that controls coverage generation.");

commands.getRunProjectCommand()->add_flag("--no-coverage", noCoverage,
"Flag that controls coverage generation.");
}
Expand All @@ -500,6 +519,10 @@ std::string Commands::RunTestsCommandOptions::getTestName() {
return testName;
}

std::string Commands::RunTestsCommandOptions::getFunctionName() {
return functionName;
}

bool Commands::RunTestsCommandOptions::withCoverage() const {
return !noCoverage;
}
Expand Down
8 changes: 8 additions & 0 deletions server/src/commands/Commands.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,12 +177,16 @@ namespace Commands {

CLI::App *getRunTestCommand();

CLI::App *getRunFunctionCommand();

CLI::App *getRunFileCommand();

CLI::App *getRunProjectCommand();

bool gotRunTestCommand();

bool gotRunFunctionCommand();

bool gotRunFileCommand();

bool gotRunProjectCommand();
Expand All @@ -191,6 +195,7 @@ namespace Commands {
CLI::App *runCommand;

CLI::App *runTestCommand;
CLI::App *runFunctionCommand;
CLI::App *runFileCommand;
CLI::App *runProjectCommand;
};
Expand All @@ -204,12 +209,15 @@ namespace Commands {

std::string getTestName();

std::string getFunctionName();

[[nodiscard]] bool withCoverage() const;

private:
fs::path filePath;
std::string testSuite;
std::string testName;
std::string functionName;

bool noCoverage = false;
};
Expand Down
1 change: 1 addition & 0 deletions server/src/coverage/CoverageAndResultsGenerator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ CoverageAndResultsGenerator::CoverageAndResultsGenerator(
coverageAndResultsRequest->testfilter().testfilepath(),
coverageAndResultsRequest->testfilter().testsuite(),
coverageAndResultsRequest->testfilter().testname(),
coverageAndResultsRequest->testfilter().functionname(),
coverageAndResultsWriter),
coverageAndResultsWriter(coverageAndResultsWriter) {
}
Expand Down
164 changes: 93 additions & 71 deletions server/src/coverage/TestRunner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#include <utils/stats/TestsExecutionStats.h>
#include <utils/KleeUtils.h>
#include "TestRunner.h"

#include "printers/DefaultMakefilePrinter.h"
Expand All @@ -18,33 +19,37 @@ TestRunner::TestRunner(utbot::ProjectContext projectContext,
std::string testFilePath,
std::string testSuite,
std::string testName,
std::string functionName,
ProgressWriter const *progressWriter)
: projectContext(std::move(projectContext)),
testFilePath(testFilePath.empty() ? std::nullopt : std::make_optional(testFilePath)),
testSuite(std::move(testSuite)), testName(std::move(testName)),
progressWriter(progressWriter) {
: projectContext(std::move(projectContext)),
testFilePath(testFilePath.empty() ? std::nullopt : std::make_optional(testFilePath)),
testSuite(std::move(testSuite)), testName(std::move(testName)), functionName(std::move(functionName)),
progressWriter(progressWriter) {
}

TestRunner::TestRunner(
const testsgen::CoverageAndResultsRequest *coverageAndResultsRequest,
grpc::ServerWriter<testsgen::CoverageAndResultsResponse> *coverageAndResultsWriter,
std::string testFilename,
std::string testSuite,
std::string testName)
: TestRunner(utbot::ProjectContext(coverageAndResultsRequest->projectcontext()),
std::move(testFilename),
std::move(testSuite),
std::move(testName),
&writer) {
const testsgen::CoverageAndResultsRequest *coverageAndResultsRequest,
grpc::ServerWriter<testsgen::CoverageAndResultsResponse> *coverageAndResultsWriter,
std::string testFilename,
std::string testSuite,
std::string testName,
std::string functionName)
: TestRunner(utbot::ProjectContext(coverageAndResultsRequest->projectcontext()),
std::move(testFilename),
std::move(testSuite),
std::move(testName),
std::move(functionName),
&writer) {
writer = ServerCoverageAndResultsWriter(coverageAndResultsWriter);
}

std::vector<UnitTest> TestRunner::getTestsFromMakefile(const fs::path &makefile,
const fs::path &testFilePath) {
const fs::path &testFilePath,
const std::string &filter) {
auto cmdGetAllTests = MakefileUtils::MakefileCommand(projectContext, makefile,
printer::DefaultMakefilePrinter::TARGET_RUN,
"--gtest_list_tests", {"GTEST_FILTER=*"});
auto[out, status, _] = cmdGetAllTests.run(projectContext.getBuildDirAbsPath(), false);
"--gtest_list_tests", {"GTEST_FILTER=" + filter});
auto [out, status, _] = cmdGetAllTests.run(projectContext.getBuildDirAbsPath(), false);
if (status != 0) {
auto [err, _, logFilePath] = cmdGetAllTests.run(projectContext.getBuildDirAbsPath(), true);
progressWriter->writeProgress(StringUtils::stringFormat("command %s failed.\n"
Expand All @@ -55,12 +60,14 @@ std::vector<UnitTest> TestRunner::getTestsFromMakefile(const fs::path &makefile,
throw ExecutionProcessException(err, logFilePath.value());
}
if (out.empty()) {
LOG_S(WARNING) << "Running gtest with flag --gtest_list_tests returns empty output. Does file contain main function?";
LOG_S(WARNING)
<< "Running gtest with flag --gtest_list_tests returns empty output. Does file contain main function?";
return {};
}
std::vector<std::string> gtestListTestsOutput = StringUtils::split(out, '\n');
gtestListTestsOutput.erase(gtestListTestsOutput.begin()); //GTEST prints "Running main() from /opt/gtest/googletest/src/gtest_main.cc"
for (std::string &s : gtestListTestsOutput) {
gtestListTestsOutput.erase(
gtestListTestsOutput.begin()); //GTEST prints "Running main() from /opt/gtest/googletest/src/gtest_main.cc"
for (std::string &s: gtestListTestsOutput) {
StringUtils::trim(s);
}
std::string testSuite;
Expand All @@ -84,50 +91,65 @@ std::vector<UnitTest> TestRunner::getTestsToLaunch() {
if (fs::exists(projectContext.getTestDirAbsPath())) {
FileSystemUtils::RecursiveDirectoryIterator directoryIterator(projectContext.getTestDirAbsPath());
ExecUtils::doWorkWithProgress(
directoryIterator, progressWriter, "Building tests",
[this, &result](fs::directory_entry const &directoryEntry) {
if (!directoryEntry.is_regular_file()) {
return;
}
const auto &testFilePath = directoryEntry.path();
if (testFilePath.extension() == Paths::CXX_EXTENSION &&
StringUtils::endsWith(testFilePath.stem().c_str(), Paths::TEST_SUFFIX)) {
fs::path sourcePath = Paths::testPathToSourcePath(projectContext, testFilePath);
fs::path makefile =
Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);
if (fs::exists(makefile)) {
try {
auto tests = getTestsFromMakefile(makefile, testFilePath);
CollectionUtils::extend(result, tests);
} catch (ExecutionProcessException const &e) {
exceptions.push_back(e);
directoryIterator, progressWriter, "Building tests",
[this, &result](fs::directory_entry const &directoryEntry) {
if (!directoryEntry.is_regular_file()) {
return;
}
const auto &testFilePath = directoryEntry.path();
if (testFilePath.extension() == Paths::CXX_EXTENSION &&
StringUtils::endsWith(testFilePath.stem().c_str(), Paths::TEST_SUFFIX)) {
fs::path sourcePath = Paths::testPathToSourcePath(projectContext, testFilePath);
fs::path makefile =
Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);
if (fs::exists(makefile)) {
try {
auto tests = getTestsFromMakefile(makefile, testFilePath);
CollectionUtils::extend(result, tests);
} catch (ExecutionProcessException const &e) {
exceptions.push_back(e);
}
} else {
LOG_S(WARNING) << StringUtils::stringFormat(
"Makefile for %s not found, candidate: %s", testFilePath, makefile);
}
} else {
LOG_S(WARNING) << StringUtils::stringFormat(
"Makefile for %s not found, candidate: %s", testFilePath, makefile);
}
} else {
if (!StringUtils::endsWith(testFilePath.stem().c_str(), Paths::TEST_SUFFIX) &&
!StringUtils::endsWith(testFilePath.stem().c_str(), Paths::STUB_SUFFIX) &&
!StringUtils::endsWith(testFilePath.stem().c_str(), Paths::MAKE_WRAPPER_SUFFIX) &&
!StringUtils::endsWith(testFilePath.c_str(), Paths::MAKEFILE_EXTENSION)) {
LOG_S(WARNING) << "Found extra file in test directory: " << testFilePath;
if (!StringUtils::endsWith(testFilePath.stem().c_str(), Paths::TEST_SUFFIX) &&
!StringUtils::endsWith(testFilePath.stem().c_str(), Paths::STUB_SUFFIX) &&
!StringUtils::endsWith(testFilePath.stem().c_str(), Paths::MAKE_WRAPPER_SUFFIX) &&
!StringUtils::endsWith(testFilePath.c_str(), Paths::MAKEFILE_EXTENSION)) {
LOG_S(WARNING) << "Found extra file in test directory: " << testFilePath;
}
}
}
});
});
} else {
LOG_S(WARNING) << "Test folder doesn't exist: " << projectContext.getTestDirAbsPath();
}
return result;
}
if (testName.empty()) {

if (testName.empty() && functionName.empty()) {
//for file
fs::path sourcePath = Paths::testPathToSourcePath(projectContext, testFilePath.value());
fs::path makefile = Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);
return getTestsFromMakefile(makefile, testFilePath.value());
}

if (testName.empty()) {
//for function
fs::path sourcePath = Paths::testPathToSourcePath(projectContext, testFilePath.value());
fs::path makefile = Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);


std::string renamedMethodDescription = KleeUtils::getRenamedOperator(functionName);
StringUtils::replaceColon(renamedMethodDescription);

std::string filter = "*." + renamedMethodDescription + Paths::TEST_SUFFIX + "*";

return getTestsFromMakefile(makefile, testFilePath.value(), filter);
}
//for single test
return { UnitTest{ testFilePath.value(), testSuite, testName } };
return {UnitTest{testFilePath.value(), testSuite, testName}};
}

grpc::Status TestRunner::runTests(bool withCoverage, const std::optional<std::chrono::seconds> &testTimeout) {
Expand All @@ -136,22 +158,22 @@ grpc::Status TestRunner::runTests(bool withCoverage, const std::optional<std::ch

const auto buildRunCommands = coverageTool->getBuildRunCommands(testsToLaunch, withCoverage);
ExecUtils::doWorkWithProgress(buildRunCommands, progressWriter, "Running tests",
[this, testTimeout] (BuildRunCommand const &buildRunCommand) {
auto const &[unitTest, buildCommand, runCommand] =
buildRunCommand;
try {
auto status = runTest(buildRunCommand, testTimeout);
testResultMap[unitTest.testFilePath][unitTest.testname] = status;
ExecUtils::throwIfCancelled();
} catch (ExecutionProcessException const &e) {
testsgen::TestResultObject testRes;
testRes.set_testfilepath(unitTest.testFilePath);
testRes.set_testname(unitTest.testname);
testRes.set_status(testsgen::TEST_FAILED);
testResultMap[unitTest.testFilePath][unitTest.testname] = testRes;
exceptions.emplace_back(e);
}
});
[this, testTimeout](BuildRunCommand const &buildRunCommand) {
auto const &[unitTest, buildCommand, runCommand] =
buildRunCommand;
try {
auto status = runTest(buildRunCommand, testTimeout);
testResultMap[unitTest.testFilePath][unitTest.testname] = status;
ExecUtils::throwIfCancelled();
} catch (ExecutionProcessException const &e) {
testsgen::TestResultObject testRes;
testRes.set_testfilepath(unitTest.testFilePath);
testRes.set_testname(unitTest.testname);
testRes.set_status(testsgen::TEST_FAILED);
testResultMap[unitTest.testFilePath][unitTest.testname] = testRes;
exceptions.emplace_back(e);
}
});
LOG_S(DEBUG) << "All run commands were executed";
return Status::OK;
}
Expand All @@ -169,14 +191,14 @@ void TestRunner::init(bool withCoverage) {
}
}

bool TestRunner::buildTest(const utbot::ProjectContext& projectContext, const fs::path& sourcePath) {
bool TestRunner::buildTest(const utbot::ProjectContext &projectContext, const fs::path &sourcePath) {
ExecUtils::throwIfCancelled();
fs::path makefile = Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);
if (fs::exists(makefile)) {
auto command = MakefileUtils::MakefileCommand(projectContext, makefile,
printer::DefaultMakefilePrinter::TARGET_BUILD, "", {});
LOG_S(DEBUG) << "Try compile tests for: " << sourcePath.string();
auto[out, status, logFilePath] = command.run(projectContext.getBuildDirAbsPath(), true);
auto [out, status, logFilePath] = command.run(projectContext.getBuildDirAbsPath(), true);
if (status != 0) {
return false;
}
Expand All @@ -185,18 +207,18 @@ bool TestRunner::buildTest(const utbot::ProjectContext& projectContext, const fs
return false;
}

size_t TestRunner::buildTests(const utbot::ProjectContext& projectContext, const tests::TestsMap& tests) {
size_t TestRunner::buildTests(const utbot::ProjectContext &projectContext, const tests::TestsMap &tests) {
size_t fail_count = 0;
for (const auto &[file, _]: tests) {
if(!TestRunner::buildTest(projectContext, file)) {
if (!TestRunner::buildTest(projectContext, file)) {
fail_count++;
}
}
return fail_count;
}

testsgen::TestResultObject TestRunner::runTest(const BuildRunCommand &command,
const std::optional <std::chrono::seconds> &testTimeout) {
const std::optional<std::chrono::seconds> &testTimeout) {
fs::remove(Paths::getGTestResultsJsonPath(projectContext));
auto res = command.runCommand.run(projectContext.getBuildDirAbsPath(), true, true, testTimeout);
GTestLogger::log(res.output);
Expand Down
Loading
Loading