Merge pull request #7 from ReCodEx/new-task-types

Add dumpdir and truncate tasks

CMakeLists.txt

@@ -165,6 +165,10 @@ set(SOURCE_FILES
 	${TASKS_DIR}/internal/extract_task.cpp
 	${TASKS_DIR}/internal/fetch_task.h
 	${TASKS_DIR}/internal/fetch_task.cpp
+	${TASKS_DIR}/internal/dump_dir_task.h
+	${TASKS_DIR}/internal/dump_dir_task.cpp
+	${TASKS_DIR}/internal/truncate_task.h
+	${TASKS_DIR}/internal/truncate_task.cpp
 
 	${HELPERS_DIR}/filesystem.h
 	${HELPERS_DIR}/filesystem.cpp

src/tasks/internal/cp_task.h

@@ -12,7 +12,7 @@ class cp_task : public task_base
 public:
 	/**
 	 * Constructor with initialization.
-	 * @param id Unique identificator of load order of tasks.
+	 * @param id Unique identifier of load order of tasks.
 	 * @param task_meta Variable containing further info about task. It's required that
 	 * @a cmd_args entry has just 2 arguments -
 	 * http://www.boost.org/doc/libs/1_59_0_b1/libs/filesystem/doc/reference.html#copy.

src/tasks/internal/dump_dir_task.cpp

@@ -0,0 +1,81 @@
+#include <fstream>
+#include "dump_dir_task.h"
+
+dump_dir_task::dump_dir_task(size_t id, std::shared_ptr<task_metadata> task_meta) : task_base(id, task_meta) {
+	if (task_meta->cmd_args.size() < 2) {
+		throw task_exception(
+			"Wrong number of arguments. Required: 2 (1 optional), Actual: "
+			+ std::to_string(task_meta_->cmd_args.size()));
+	}
+}
+
+dump_dir_task::~dump_dir_task() {
+
+}
+
+std::shared_ptr<task_results> dump_dir_task::run() {
+	auto results = std::make_shared<task_results>();
+	fs::path src_root(task_meta_->cmd_args[0]);
+	fs::path dest_root(task_meta_->cmd_args[1]);
+
+	auto limit = read_task_arg<size_t>(task_meta_->cmd_args, 2, 128);
+	limit *= 1024; // The argument is in kilobytes
+
+	fs::recursive_directory_iterator directory_iterator(src_root), directory_iterator_end;
+	std::vector<fs::path> paths(directory_iterator, directory_iterator_end);
+
+	std::sort(paths.begin(), paths.end(), [] (const fs::path &a, const fs::path &b) {
+		if (a == b) {
+			return false;
+		}
+
+		if (fs::is_directory(a) && !fs::is_directory(b)) {
+			return true;
+		}
+
+		if (fs::is_directory(b) && !fs::is_directory(a)) {
+			return false;
+		}
+
+		if (fs::is_directory(a) && fs::is_directory(b)) {
+			return a < b;
+		}
+
+		return fs::file_size(a) < fs::file_size(b);
+	});
+
+	for (auto &path: paths) {
+		auto relative_path = fs::path(path.string().substr(src_root.string().size()));
+		auto dest_path = dest_root / relative_path;
+
+		if (fs::is_directory(path)) {
+			fs::create_directories(dest_path);
+		} else {
+			size_t size = fs::file_size(path);
+			if (size <= limit) {
+				bool return_value = copy_file(path, dest_path);
+
+				if (!return_value) {
+					results->status = task_status::FAILED;
+					results->error_message = "Copying failed: " + path.string();
+				}
+
+				limit = size > limit ? 0 : limit - size;
+			} else {
+				std::ofstream placeholder(dest_path.string() + ".skipped", std::ios::out | std::ios::app);
+				placeholder << " ";
+				placeholder.close();
+			}
+		}
+	}
+
+	return results;
+}
+
+bool dump_dir_task::copy_file(const fs::path &src, const fs::path &dest) {
+	boost::system::error_code error_code;
+	fs::copy(src, dest, error_code);
+
+	return error_code.value() == boost::system::errc::success;
+}
+

src/tasks/internal/dump_dir_task.h

@@ -0,0 +1,41 @@
+#ifndef RECODEX_WORKER_INTERNAL_CP_DIR_TASK_H
+#define RECODEX_WORKER_INTERNAL_CP_DIR_TASK_H
+
+#include <boost/filesystem.hpp>
+#include "../task_base.h"
+
+namespace fs = boost::filesystem;
+
+/**
+ * Copy a directory recursively, with a limit on the total size of the output.
+ * The files are taken in the order of size (ascending) and files that exceed the size limit are replaced with empty
+ * files whose names are suffixed with ".skipped".
+ */
+class dump_dir_task: public task_base {
+public:
+	/**
+	 * Constructor with initialization.
+	 * @param id Unique identifier of load order of tasks.
+	 * @param task_meta Variable containing further info about task. It's required that
+	 * @a cmd_args entry has 2 or 3 arguments - the source, destination, and optionally a limit
+	 * @throws task_exception on invalid number of arguments.
+	 */
+	dump_dir_task(size_t id, std::shared_ptr<task_metadata> task_meta);
+
+	/**
+	 * Destructor.
+	 */
+	virtual ~dump_dir_task();
+
+	/**
+	 * Run the action.
+	 * @return Evaluation results to be pushed back to frontend.
+	 */
+	virtual std::shared_ptr<task_results> run();
+
+private:
+	bool copy_file(const fs::path &src, const fs::path &dest);
+};
+
+
+#endif //RECODEX_WORKER_INTERNAL_CP_DIR_TASK_H

src/tasks/internal/truncate_task.cpp

@@ -0,0 +1,37 @@
+#include <boost/filesystem.hpp>
+#include "truncate_task.h"
+
+namespace fs = boost::filesystem;
+
+truncate_task::truncate_task(size_t id, std::shared_ptr<task_metadata> task_meta) : task_base(id, task_meta)
+{
+	if (task_meta->cmd_args.size() < 2) {
+		throw task_exception(
+			"Wrong number of arguments. Required: 2, Actual: "
+			+ std::to_string(task_meta_->cmd_args.size()));
+	}
+}
+
+truncate_task::~truncate_task()
+{
+}
+
+std::shared_ptr<task_results> truncate_task::run()
+{
+	auto results = std::make_shared<task_results>();
+
+	fs::path file(task_meta_->cmd_args[0]);
+	auto limit = read_task_arg<size_t>(task_meta_->cmd_args, 1, 128);
+	limit *= 1024;
+
+	if (fs::file_size(file) > limit) {
+		boost::system::error_code error_code;
+		fs::resize_file(file, limit, error_code);
+
+		if (error_code.value() != boost::system::errc::success) {
+			results->status = task_status::FAILED;
+		}
+	}
+
+	return results;
+}

src/tasks/internal/truncate_task.h

@@ -0,0 +1,25 @@
+#ifndef RECODEX_WORKER_TRUNCATE_TASK_H
+#define RECODEX_WORKER_TRUNCATE_TASK_H
+
+#include "../task_base.h"
+
+
+class truncate_task: public task_base {
+public:
+	/**
+	 * Constructor with initialization.
+	 * @param id Unique identifier of load order of tasks.
+	 * @param task_meta Variable containing further info about task. It's required that
+	 * @a cmd_args entry has just 2 arguments - the name of the file to be truncated and the desired size
+	 * @throws task_exception on invalid number of arguments.
+	 */
+	truncate_task(size_t id, std::shared_ptr<task_metadata> task_meta);
+
+	virtual ~truncate_task();
+
+	virtual std::shared_ptr<task_results> run();
+
+};
+
+
+#endif //RECODEX_WORKER_TRUNCATE_TASK_H

src/tasks/task_base.h

@@ -5,6 +5,7 @@
 #include <string>
 #include <cstdlib>
 #include <memory>
+#include <sstream>
 #include "../config/task_results.h"
 #include "../config/task_metadata.h"
 
@@ -136,7 +137,7 @@ class task_base
 
 	/** Weak pointers to parents of task. */
 	std::vector<std::weak_ptr<task_base>> parents_;
-	/** Pointers to task childrens. */
+	/** Pointers to task children. */
 	std::vector<std::shared_ptr<task_base>> children_;
 };
 
@@ -180,6 +181,18 @@ class task_exception : public std::exception
 	std::string what_;
 };
 
+template <typename T> T read_task_arg(const std::vector<std::string> &args, const size_t index, const T &default_value = T())
+{
+	if (index >= args.size()) {
+		return default_value;
+	}
+
+	T value;
+	std::stringstream ss(args.at(index));
+	ss >> value;
+
+	return value;
+}
 
 /**
  * Comparator which is used for topological sorting of tasks

src/tasks/task_factory.cpp

@@ -19,6 +19,8 @@ std::shared_ptr<task_base> task_factory::create_internal_task(size_t id, std::sh
 
 	if (task_meta->binary == "cp") {
 		task = std::make_shared<cp_task>(id, task_meta);
+	} else if (task_meta->binary == "dumpdir") {
+		task = std::make_shared<dump_dir_task>(id, task_meta);
 	} else if (task_meta->binary == "mkdir") {
 		task = std::make_shared<mkdir_task>(id, task_meta);
 	} else if (task_meta->binary == "rename") {
@@ -31,6 +33,8 @@ std::shared_ptr<task_base> task_factory::create_internal_task(size_t id, std::sh
 		task = std::make_shared<extract_task>(id, task_meta);
 	} else if (task_meta->binary == "fetch") {
 		task = std::make_shared<fetch_task>(id, task_meta, fileman_);
+	} else if (task_meta->binary == "truncate") {
+		task = std::make_shared<truncate_task>(id, task_meta);
 	} else {
 		task = nullptr;
 	}

src/tasks/task_factory.h

@@ -7,6 +7,8 @@
 #include "root_task.h"
 #include "internal/archivate_task.h"
 #include "internal/cp_task.h"
+#include "internal/dump_dir_task.h"
+#include "internal/truncate_task.h"
 #include "internal/extract_task.h"
 #include "internal/fetch_task.h"
 #include "internal/mkdir_task.h"

tests/CMakeLists.txt

@@ -115,12 +115,14 @@ add_test_suite(tasks
 	${TASKS_DIR}/root_task.cpp
 	${TASKS_DIR}/external_task.cpp
 	${TASKS_DIR}/internal/cp_task.cpp
+	${TASKS_DIR}/internal/dump_dir_task.cpp
 	${TASKS_DIR}/internal/mkdir_task.cpp
 	${TASKS_DIR}/internal/rename_task.cpp
 	${TASKS_DIR}/internal/rm_task.cpp
 	${TASKS_DIR}/internal/archivate_task.cpp
 	${TASKS_DIR}/internal/extract_task.cpp
 	${TASKS_DIR}/internal/fetch_task.cpp
+	${TASKS_DIR}/internal/truncate_task.cpp
 	${SRC_DIR}/archives/archivator.cpp
 	${SANDBOX_DIR}/isolate_sandbox.cpp
 	${HELPERS_DIR}/logger.cpp
@@ -156,6 +158,18 @@ add_test_suite(string_utils
 	string_utils.cpp
 )
 
+add_test_suite(dump_dir_task
+	${TASKS_DIR}/task_base.cpp
+	${TASKS_DIR}/internal/dump_dir_task.cpp
+	dump_dir_task.cpp
+)
+
+add_test_suite(truncate_task
+	${TASKS_DIR}/task_base.cpp
+	${TASKS_DIR}/internal/truncate_task.cpp
+	truncate_task.cpp
+)
+
 # Tests that depend on external resources
 if(UNIX)
 	set(LIBS

tests/dump_dir_task.cpp

@@ -0,0 +1,89 @@
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+#include <boost/filesystem.hpp>
+#include <fstream>
+#include <memory>
+#include "../src/tasks/internal/dump_dir_task.h"
+
+namespace fs = boost::filesystem;
+
+class dump_dir_task_test : public ::testing::Test {
+protected:
+	fs::path root;
+	fs::path target;
+	std::shared_ptr<dump_dir_task> task;
+	std::shared_ptr<task_metadata> task_meta;
+
+	virtual void SetUp()
+	{
+		target = fs::temp_directory_path() / fs::unique_path();
+		fs::create_directory(target);
+
+		root = fs::temp_directory_path() / fs::unique_path();
+		fs::create_directory(root);
+		fs::create_directory(root / "subdir");
+
+		create_file(root / "file_a", 2048);
+		create_file(root / "file_b", 4096);
+		create_file(root / "subdir" / "file_c", 1536);
+
+		task_meta = std::make_shared<task_metadata>();
+		task_meta->cmd_args = {root.string(), target.string(), "16384"};
+		task = std::make_shared<dump_dir_task>(1, task_meta);
+	}
+
+	virtual void TearDown()
+	{
+		fs::remove_all(root);
+		fs::remove(root);
+		fs::remove_all(target);
+		fs::remove(target);
+	}
+
+	void create_file(const fs::path &path, size_t size)
+	{
+		std::ofstream f(path.string());
+		for (size_t i = 0; i < size; i++) {
+			f << "a";
+		}
+		f.close();
+	}
+};
+
+TEST_F(dump_dir_task_test, everything_fits)
+{
+	auto results = task->run();
+	ASSERT_EQ(task_status::OK, results->status);
+
+	ASSERT_TRUE(fs::exists(target / "file_a"));
+	ASSERT_TRUE(fs::exists(target / "file_b"));
+	ASSERT_TRUE(fs::exists(target / "subdir" / "file_c"));
+}
+
+TEST_F(dump_dir_task_test, everything_skipped)
+{
+	task_meta->cmd_args[2] = "1";
+
+	auto results = task->run();
+	ASSERT_EQ(task_status::OK, results->status);
+
+	ASSERT_FALSE(fs::exists(target / "file_a"));
+	ASSERT_TRUE(fs::exists(target / "file_a.skipped"));
+	ASSERT_FALSE(fs::exists(target / "file_b"));
+	ASSERT_TRUE(fs::exists(target / "file_b.skipped"));
+	ASSERT_FALSE(fs::exists(target / "subdir" / "file_c"));
+	ASSERT_TRUE(fs::exists(target / "subdir" / "file_c.skipped"));
+}
+
+TEST_F(dump_dir_task_test, largest_skipped)
+{
+	task_meta->cmd_args[2] = "4";
+
+	auto results = task->run();
+	ASSERT_EQ(task_status::OK, results->status);
+
+	ASSERT_TRUE(fs::exists(target / "file_a"));
+	ASSERT_FALSE(fs::exists(target / "file_b"));
+	ASSERT_TRUE(fs::exists(target / "file_b.skipped"));
+	ASSERT_TRUE(fs::exists(target / "subdir" / "file_c"));
+}