From 366d738dac3a90575f0748227a6de54d032c3923 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Tue, 2 Nov 2021 17:27:51 +0100 Subject: [PATCH] Add a selfoss CLI tool This introduces two commands for dumping and loading the database contents: - `php bin/selfoss db:export path.json` - `php bin/selfoss db:import path.json` Those are useful for testing, as well as for moving data from one database backend to another. --- .php-cs-fixer.php | 3 +- bin/selfoss | 20 ++ composer.json | 2 + composer.lock | 306 ++++++++++++------------ src/Commands/Database/ExportCommand.php | 49 ++++ src/Commands/Database/ImportCommand.php | 52 ++++ src/daos/CommonSqlDatabase.php | 20 ++ src/daos/DatabaseInterface.php | 9 + src/daos/ItemsInterface.php | 14 ++ src/daos/SourcesInterface.php | 14 ++ src/daos/TagsInterface.php | 14 ++ src/daos/mysql/Items.php | 27 +++ src/daos/mysql/Sources.php | 22 ++ src/daos/mysql/Tags.php | 16 ++ 14 files changed, 414 insertions(+), 154 deletions(-) create mode 100755 bin/selfoss create mode 100644 src/Commands/Database/ExportCommand.php create mode 100644 src/Commands/Database/ImportCommand.php diff --git a/.php-cs-fixer.php b/.php-cs-fixer.php index 39974f827a..0a02c452d6 100644 --- a/.php-cs-fixer.php +++ b/.php-cs-fixer.php @@ -4,7 +4,8 @@ ->exclude('assets') ->exclude('utils') ->in(__DIR__) - ->name('*.phtml'); + ->name('*.phtml') + ->name('selfoss'); $rules = [ '@Symfony' => true, diff --git a/bin/selfoss b/bin/selfoss new file mode 100755 index 0000000000..aec9f7b940 --- /dev/null +++ b/bin/selfoss @@ -0,0 +1,20 @@ +#!/usr/bin/env php +popHandler(); +$handler = new StreamHandler('php://stderr'); +$log->pushHandler($handler); + +$application->add($dice->create(DatabaseExportCommand::class)); +$application->add($dice->create(DatabaseImportCommand::class)); + +$application->run(); diff --git a/composer.json b/composer.json index 5e8b66e638..93cbdfd5d7 100644 --- a/composer.json +++ b/composer.json @@ -22,6 +22,7 @@ "php-http/guzzle6-adapter": "^1.0", "simplepie/simplepie": "^1.3", "smottt/wideimage": "^1.1", + "symfony/console": "^3.4", "violet/streaming-json-encoder": "^1.1", "willwashburn/phpamo": "^1.0" }, @@ -39,6 +40,7 @@ ], "autoload": { "psr-4": { + "Commands\\": "src/Commands/", "controllers\\": "src/controllers/", "daos\\": "src/daos/", "helpers\\": "src/helpers/", diff --git a/composer.lock b/composer.lock index 484bc1a6af..a7b98ed84f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "8b4e7b3be490c10b0bd0c4d63df28d61", + "content-hash": "4a5de52ba0c4702029fecf2d24acd6f2", "packages": [ { "name": "bcosca/fatfree-core", @@ -2209,6 +2209,158 @@ }, "time": "2021-01-09T15:26:41+00:00" }, + { + "name": "symfony/console", + "version": "v3.4.47", + "source": { + "type": "git", + "url": "https://github.com/symfony/console.git", + "reference": "a10b1da6fc93080c180bba7219b5ff5b7518fe81" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/console/zipball/a10b1da6fc93080c180bba7219b5ff5b7518fe81", + "reference": "a10b1da6fc93080c180bba7219b5ff5b7518fe81", + "shasum": "" + }, + "require": { + "php": "^5.5.9|>=7.0.8", + "symfony/debug": "~2.8|~3.0|~4.0", + "symfony/polyfill-mbstring": "~1.0" + }, + "conflict": { + "symfony/dependency-injection": "<3.4", + "symfony/process": "<3.3" + }, + "provide": { + "psr/log-implementation": "1.0" + }, + "require-dev": { + "psr/log": "~1.0", + "symfony/config": "~3.3|~4.0", + "symfony/dependency-injection": "~3.4|~4.0", + "symfony/event-dispatcher": "~2.8|~3.0|~4.0", + "symfony/lock": "~3.4|~4.0", + "symfony/process": "~3.3|~4.0" + }, + "suggest": { + "psr/log": "For using the console logger", + "symfony/event-dispatcher": "", + "symfony/lock": "", + "symfony/process": "" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\Console\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony Console Component", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/console/tree/v3.4.47" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2020-10-24T10:57:07+00:00" + }, + { + "name": "symfony/debug", + "version": "v3.4.47", + "source": { + "type": "git", + "url": "https://github.com/symfony/debug.git", + "reference": "ab42889de57fdfcfcc0759ab102e2fd4ea72dcae" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/debug/zipball/ab42889de57fdfcfcc0759ab102e2fd4ea72dcae", + "reference": "ab42889de57fdfcfcc0759ab102e2fd4ea72dcae", + "shasum": "" + }, + "require": { + "php": "^5.5.9|>=7.0.8", + "psr/log": "~1.0" + }, + "conflict": { + "symfony/http-kernel": ">=2.3,<2.3.24|~2.4.0|>=2.5,<2.5.9|>=2.6,<2.6.2" + }, + "require-dev": { + "symfony/http-kernel": "~2.8|~3.0|~4.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\Debug\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony Debug Component", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/debug/tree/v3.4.47" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2020-10-24T10:57:07+00:00" + }, { "name": "symfony/finder", "version": "v3.4.47", @@ -4717,158 +4869,6 @@ }, "time": "2016-10-03T07:35:21+00:00" }, - { - "name": "symfony/console", - "version": "v3.4.47", - "source": { - "type": "git", - "url": "https://github.com/symfony/console.git", - "reference": "a10b1da6fc93080c180bba7219b5ff5b7518fe81" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/a10b1da6fc93080c180bba7219b5ff5b7518fe81", - "reference": "a10b1da6fc93080c180bba7219b5ff5b7518fe81", - "shasum": "" - }, - "require": { - "php": "^5.5.9|>=7.0.8", - "symfony/debug": "~2.8|~3.0|~4.0", - "symfony/polyfill-mbstring": "~1.0" - }, - "conflict": { - "symfony/dependency-injection": "<3.4", - "symfony/process": "<3.3" - }, - "provide": { - "psr/log-implementation": "1.0" - }, - "require-dev": { - "psr/log": "~1.0", - "symfony/config": "~3.3|~4.0", - "symfony/dependency-injection": "~3.4|~4.0", - "symfony/event-dispatcher": "~2.8|~3.0|~4.0", - "symfony/lock": "~3.4|~4.0", - "symfony/process": "~3.3|~4.0" - }, - "suggest": { - "psr/log": "For using the console logger", - "symfony/event-dispatcher": "", - "symfony/lock": "", - "symfony/process": "" - }, - "type": "library", - "autoload": { - "psr-4": { - "Symfony\\Component\\Console\\": "" - }, - "exclude-from-classmap": [ - "/Tests/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Fabien Potencier", - "email": "fabien@symfony.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Symfony Console Component", - "homepage": "https://symfony.com", - "support": { - "source": "https://github.com/symfony/console/tree/v3.4.47" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2020-10-24T10:57:07+00:00" - }, - { - "name": "symfony/debug", - "version": "v3.4.47", - "source": { - "type": "git", - "url": "https://github.com/symfony/debug.git", - "reference": "ab42889de57fdfcfcc0759ab102e2fd4ea72dcae" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/debug/zipball/ab42889de57fdfcfcc0759ab102e2fd4ea72dcae", - "reference": "ab42889de57fdfcfcc0759ab102e2fd4ea72dcae", - "shasum": "" - }, - "require": { - "php": "^5.5.9|>=7.0.8", - "psr/log": "~1.0" - }, - "conflict": { - "symfony/http-kernel": ">=2.3,<2.3.24|~2.4.0|>=2.5,<2.5.9|>=2.6,<2.6.2" - }, - "require-dev": { - "symfony/http-kernel": "~2.8|~3.0|~4.0" - }, - "type": "library", - "autoload": { - "psr-4": { - "Symfony\\Component\\Debug\\": "" - }, - "exclude-from-classmap": [ - "/Tests/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Fabien Potencier", - "email": "fabien@symfony.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Symfony Debug Component", - "homepage": "https://symfony.com", - "support": { - "source": "https://github.com/symfony/debug/tree/v3.4.47" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2020-10-24T10:57:07+00:00" - }, { "name": "symfony/event-dispatcher", "version": "v3.4.47", diff --git a/src/Commands/Database/ExportCommand.php b/src/Commands/Database/ExportCommand.php new file mode 100644 index 0000000000..e6a7caf424 --- /dev/null +++ b/src/Commands/Database/ExportCommand.php @@ -0,0 +1,49 @@ +database = $database; + $this->items = $items; + $this->sources = $sources; + $this->tags = $tags; + + parent::__construct(); + } + + protected function configure() { + $this->addArgument('path', InputArgument::REQUIRED, 'Path to file to export the database to.'); + } + + protected function execute(InputInterface $input, OutputInterface $output) { + $data = [ + 'schemaVersion' => $this->database->getSchemaVersion(), + 'tags' => $this->tags->getRaw(), + 'sources' => $this->sources->getRaw(), + 'items' => $this->items->getRaw(), + ]; + + file_put_contents($input->getArgument('path'), json_encode($data)); + } +} diff --git a/src/Commands/Database/ImportCommand.php b/src/Commands/Database/ImportCommand.php new file mode 100644 index 0000000000..7c9ae3a4cd --- /dev/null +++ b/src/Commands/Database/ImportCommand.php @@ -0,0 +1,52 @@ +database = $database; + $this->items = $items; + $this->sources = $sources; + $this->tags = $tags; + + parent::__construct(); + } + + protected function configure() { + $this->addArgument('path', InputArgument::REQUIRED, 'Path to file to import the database from.'); + } + + protected function execute(InputInterface $input, OutputInterface $output) { + $data = json_decode(file_get_contents($input->getArgument('path')), true); + + $currentDbSchemaVersion = $this->database->getSchemaVersion(); + + if ($data['schemaVersion'] !== $currentDbSchemaVersion) { + $output->writeln('The database schema version of the current selfoss installation does not match the installation the data was exported from, which may cause issues. For best compatibility, upgrade ' . ($data['schemaVersion'] < $currentDbSchemaVersion ? 'the source selfoss' : 'this selfoss') . ' installation before importing.'); + } + + $this->tags->insertRaw($data['tags']); + $this->sources->insertRaw($data['sources']); + $this->items->insertRaw($data['items']); + } +} diff --git a/src/daos/CommonSqlDatabase.php b/src/daos/CommonSqlDatabase.php index c34583723a..6f8d3ef162 100644 --- a/src/daos/CommonSqlDatabase.php +++ b/src/daos/CommonSqlDatabase.php @@ -43,4 +43,24 @@ public function getSchemaVersion() { return (int) $version[0]['version']; } + + /** + * Insert raw table data into given table. + * + * @param string $table target database table + * @param string[] $fields column names + * @param array[] $data rows to insert + */ + public function insertRaw($table, array $fields, array $data) { + $fieldsSql = implode(', ', $fields); + $valuesSql = implode(', ', array_map(function($field) { + return ":$field"; + }, $fields)); + $values = []; + foreach ($fields as $field) { + $values[":$field"] = $data[$field]; + } + + $this->exec("insert into $table($fieldsSql) values($valuesSql)", $values); + } } diff --git a/src/daos/DatabaseInterface.php b/src/daos/DatabaseInterface.php index 61c6815aee..d7bedc8b65 100644 --- a/src/daos/DatabaseInterface.php +++ b/src/daos/DatabaseInterface.php @@ -31,6 +31,15 @@ public function exec($cmds, $args = null); */ public function insert($query, array $params); + /** + * Insert raw table data into given table. + * + * @param string $table target database table + * @param string[] $fields column names + * @param array[] $data rows to insert + */ + public function insertRaw($table, array $fields, array $data); + /** * Quote string * diff --git a/src/daos/ItemsInterface.php b/src/daos/ItemsInterface.php index 54889c0dae..27c6ee7322 100644 --- a/src/daos/ItemsInterface.php +++ b/src/daos/ItemsInterface.php @@ -214,4 +214,18 @@ public function statuses(DateTime $since); * @return void */ public function bulkStatusUpdate(array $statuses); + + /** + * returns raw items table contents + * + * @return array[] of all items + */ + public function getRaw(); + + /** + * inserts raw data into items table + * + * @param array[] $items + */ + public function insertRaw(array $items); } diff --git a/src/daos/SourcesInterface.php b/src/daos/SourcesInterface.php index e9a65f74aa..9ea32d897f 100644 --- a/src/daos/SourcesInterface.php +++ b/src/daos/SourcesInterface.php @@ -130,4 +130,18 @@ public function getTags($id); * @return int id if any record is found */ public function checkIfExists($title, $spout, array $params); + + /** + * returns raw sources table contents + * + * @return array[] of all sources + */ + public function getRaw(); + + /** + * inserts raw data into sources table + * + * @param array[] $sources + */ + public function insertRaw(array $sources); } diff --git a/src/daos/TagsInterface.php b/src/daos/TagsInterface.php index 5a2c13c0ac..748bf0673e 100644 --- a/src/daos/TagsInterface.php +++ b/src/daos/TagsInterface.php @@ -65,4 +65,18 @@ public function hasTag($tag); * @return void */ public function delete($tag); + + /** + * returns raw tags table contents + * + * @return array[] of all tags + */ + public function getRaw(); + + /** + * inserts raw data into tags table + * + * @param array[] $tags + */ + public function insertRaw(array $tags); } diff --git a/src/daos/mysql/Items.php b/src/daos/mysql/Items.php index 9e1795e960..893318fbff 100644 --- a/src/daos/mysql/Items.php +++ b/src/daos/mysql/Items.php @@ -738,4 +738,31 @@ public function bulkStatusUpdate(array $statuses) { $this->database->commit(); } } + + /** + * {@inheritdoc} + */ + public function getRaw() { + $stmt = static::$stmt; + $items = $this->database->exec('select * from items'); + + return $stmt::ensureRowTypes($items, [ + 'id' => \daos\PARAM_INT, + 'datetime' => \daos\PARAM_DATETIME, + 'unread' => \daos\PARAM_BOOL, + 'starred' => \daos\PARAM_BOOL, + 'source' => \daos\PARAM_INT, + 'tags' => \daos\PARAM_CSV, + 'updatetime' => \daos\PARAM_DATETIME, + ]); + } + + /** + * {@inheritdoc} + */ + public function insertRaw(array $items) { + foreach ($items as $item) { + $this->database->insertRaw('items', ['id', 'datetime', 'title', 'content', 'thumbnail', 'icon', 'unread', 'starred', 'source', 'uid', 'link', 'updatetime', 'author', 'shared', 'lastseen'], $item); + } + } } diff --git a/src/daos/mysql/Sources.php b/src/daos/mysql/Sources.php index 46a115b342..ba3e137084 100644 --- a/src/daos/mysql/Sources.php +++ b/src/daos/mysql/Sources.php @@ -304,4 +304,26 @@ public function checkIfExists($title, $spout, array $params) { return 0; } + + /** + * {@inheritdoc} + */ + public function getRaw() { + $stmt = static::$stmt; + $sources = $this->database->exec('select * from sources'); + + return $stmt::ensureRowTypes($sources, [ + 'id' => \daos\PARAM_INT, + 'tags' => \daos\PARAM_CSV, + ]); + } + + /** + * {@inheritdoc} + */ + public function insertRaw(array $sources) { + foreach ($sources as $source) { + $this->database->insertRaw('sources', ['id', 'title', 'tags', 'filter', 'spout', 'params', 'error', 'lastupdate', 'lastentry'], $source); + } + } } diff --git a/src/daos/mysql/Tags.php b/src/daos/mysql/Tags.php index d99344732e..d2c82119f6 100644 --- a/src/daos/mysql/Tags.php +++ b/src/daos/mysql/Tags.php @@ -165,4 +165,20 @@ public function hasTag($tag) { public function delete($tag) { $this->database->exec('DELETE FROM ' . $this->configuration->dbPrefix . 'tags WHERE tag=:tag', [':tag' => $tag]); } + + /** + * {@inheritdoc} + */ + public function getRaw() { + return $this->database->exec('select * from tags'); + } + + /** + * {@inheritdoc} + */ + public function insertRaw(array $tags) { + foreach ($tags as $tag) { + $this->database->insertRaw('tags', ['tag', 'color'], $tag); + } + } }