From 091f3a56f82ff377dfa4b3f7b5f1169c6e165949 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Fri, 19 Jul 2024 02:15:43 -0400 Subject: [PATCH] chore(refactor): Move sorted_versions function into caller --- docs/news.rst | 11 ++++ scrapyd/eggstorage.py | 9 ++- scrapyd/utils.py | 8 --- tests/test_eggstorage.py | 14 +++- tests/test_utils.py | 137 --------------------------------------- 5 files changed, 32 insertions(+), 147 deletions(-) delete mode 100644 tests/test_utils.py diff --git a/docs/news.rst b/docs/news.rst index b4b5ee09..77702c43 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -57,6 +57,17 @@ CLI - Remove all ``twistd`` subcommands (FTP servers, etc.). Run ``twistd``, if needed. - Run the ``scrapyd.__main__`` module, instead of the ``scrapyd.scripts.scrapyd_run`` module. +Utils +^^^^^ + +Move functions from ``scrapyd.utils`` into their callers: + +- ``sorted_versions`` to ``scrapyd.eggstorage`` +- ``get_crawl_args`` to ``scrapyd.launcher`` +- ``JsonResource``, ``get_spider_list`` and ``UtilsCache`` to ``scrapyd.webservice`` + +Move ``activate_egg`` from ``scrapyd.eggutils`` to ``scrapyd.runner`` + Fixed ~~~~~ diff --git a/scrapyd/eggstorage.py b/scrapyd/eggstorage.py index ad1aceeb..eb735bb6 100644 --- a/scrapyd/eggstorage.py +++ b/scrapyd/eggstorage.py @@ -3,11 +3,18 @@ import shutil from glob import escape, glob +from packaging.version import InvalidVersion, Version from zope.interface import implementer from scrapyd.exceptions import DirectoryTraversalError, EggNotFoundError, ProjectNotFoundError from scrapyd.interfaces import IEggStorage -from scrapyd.utils import sorted_versions + + +def sorted_versions(versions): + try: + return sorted(versions, key=Version) + except InvalidVersion: + return sorted(versions) @implementer(IEggStorage) diff --git a/scrapyd/utils.py b/scrapyd/utils.py index df1dbe26..aaa6219f 100644 --- a/scrapyd/utils.py +++ b/scrapyd/utils.py @@ -5,7 +5,6 @@ from typing import ClassVar from urllib.parse import urlsplit -from packaging.version import InvalidVersion, Version from scrapy.utils.misc import load_object from twisted.web import resource @@ -174,10 +173,3 @@ def _to_native_str(text, encoding="utf-8", errors="strict"): if isinstance(text, str): return text return text.decode(encoding, errors) - - -def sorted_versions(versions): - try: - return sorted(versions, key=Version) - except InvalidVersion: - return sorted(versions) diff --git a/tests/test_eggstorage.py b/tests/test_eggstorage.py index d231b3c5..f5318e00 100644 --- a/tests/test_eggstorage.py +++ b/tests/test_eggstorage.py @@ -8,11 +8,23 @@ from scrapyd.app import application from scrapyd.config import Config -from scrapyd.eggstorage import FilesystemEggStorage +from scrapyd.eggstorage import FilesystemEggStorage, sorted_versions from scrapyd.exceptions import DirectoryTraversalError from scrapyd.interfaces import IEggStorage +@pytest.mark.parametrize( + ("versions", "expected"), + [ + (["zzz", "b", "ddd", "a", "x"], ["a", "b", "ddd", "x", "zzz"]), + (["10", "1", "9"], ["1", "9", "10"]), + (["2.11", "2.01", "2.9"], ["2.01", "2.9", "2.11"]), + ], +) +def test_sorted_versions(versions, expected): + assert sorted_versions(versions) == expected + + @implementer(IEggStorage) class SomeFakeEggStorage: def __init__(self, config): diff --git a/tests/test_utils.py b/tests/test_utils.py deleted file mode 100644 index 634c1548..00000000 --- a/tests/test_utils.py +++ /dev/null @@ -1,137 +0,0 @@ -import os -from io import BytesIO -from pkgutil import get_data -from subprocess import Popen -from unittest import mock - -import pytest -from scrapy.utils.test import get_pythonpath -from twisted.trial import unittest - -from scrapyd import get_application -from scrapyd.exceptions import RunnerError -from scrapyd.interfaces import IEggStorage -from scrapyd.utils import UtilsCache, get_crawl_args, get_spider_list, sorted_versions - - -def get_pythonpath_scrapyd(): - scrapyd_path = __import__("scrapyd").__path__[0] - return os.path.join(os.path.dirname(scrapyd_path), get_pythonpath(), os.environ.get("PYTHONPATH", "")) - - -class UtilsTest(unittest.TestCase): - def test_get_crawl_args(self): - msg = {"_project": "lolo", "_spider": "lala"} - - self.assertEqual(get_crawl_args(msg), ["lala"]) - - msg = {"_project": "lolo", "_spider": "lala", "arg1": "val1"} - cargs = get_crawl_args(msg) - - self.assertEqual(cargs, ["lala", "-a", "arg1=val1"]) - self.assertTrue(all(isinstance(x, str) for x in cargs), cargs) - - def test_get_crawl_args_with_settings(self): - msg = {"_project": "lolo", "_spider": "lala", "arg1": "val1", "settings": {"ONE": "two"}} - cargs = get_crawl_args(msg) - - self.assertEqual(cargs, ["lala", "-a", "arg1=val1", "-s", "ONE=two"]) - self.assertTrue(all(isinstance(x, str) for x in cargs), cargs) - - -class GetSpiderListTest(unittest.TestCase): - def setUp(self): - path = os.path.abspath(self.mktemp()) - j = os.path.join - eggs_dir = j(path, "eggs") - os.makedirs(eggs_dir) - dbs_dir = j(path, "dbs") - os.makedirs(dbs_dir) - logs_dir = j(path, "logs") - os.makedirs(logs_dir) - os.chdir(path) - with open("scrapyd.conf", "w") as f: - f.write("[scrapyd]\n") - f.write(f"eggs_dir = {eggs_dir}\n") - f.write(f"dbs_dir = {dbs_dir}\n") - f.write(f"logs_dir = {logs_dir}\n") - self.app = get_application() - - def add_test_version(self, file, project, version): - eggstorage = self.app.getComponent(IEggStorage) - eggfile = BytesIO(get_data("tests", file)) - eggstorage.put(eggfile, project, version) - - def test_get_spider_list_log_stdout(self): - self.add_test_version("logstdout.egg", "logstdout", "logstdout") - spiders = get_spider_list("logstdout", pythonpath=get_pythonpath_scrapyd()) - # If LOG_STDOUT were respected, the output would be []. - self.assertEqual(sorted(spiders), ["spider1", "spider2"]) - - def test_get_spider_list(self): - # mybot.egg has two spiders, spider1 and spider2 - self.add_test_version("mybot.egg", "mybot", "r1") - spiders = get_spider_list("mybot", pythonpath=get_pythonpath_scrapyd()) - self.assertEqual(sorted(spiders), ["spider1", "spider2"]) - - # mybot2.egg has three spiders, spider1, spider2 and spider3... - # BUT you won't see it here because it's cached. - # Effectivelly it's like if version was never added - self.add_test_version("mybot2.egg", "mybot", "r2") - spiders = get_spider_list("mybot", pythonpath=get_pythonpath_scrapyd()) - self.assertEqual(sorted(spiders), ["spider1", "spider2"]) - - # Let's invalidate the cache for this project... - UtilsCache.invalid_cache("mybot") - - # Now you get the updated list - spiders = get_spider_list("mybot", pythonpath=get_pythonpath_scrapyd()) - self.assertEqual(sorted(spiders), ["spider1", "spider2", "spider3"]) - - # Let's re-deploy mybot.egg and clear cache. It now sees 2 spiders - self.add_test_version("mybot.egg", "mybot", "r3") - UtilsCache.invalid_cache("mybot") - spiders = get_spider_list("mybot", pythonpath=get_pythonpath_scrapyd()) - self.assertEqual(sorted(spiders), ["spider1", "spider2"]) - - # And re-deploying the one with three (mybot2.egg) with a version that - # isn't the higher, won't change what get_spider_list() returns. - self.add_test_version("mybot2.egg", "mybot", "r1a") - UtilsCache.invalid_cache("mybot") - spiders = get_spider_list("mybot", pythonpath=get_pythonpath_scrapyd()) - self.assertEqual(sorted(spiders), ["spider1", "spider2"]) - - @pytest.mark.skipif(os.name == "nt", reason="get_spider_list() unicode fails on windows") - def test_get_spider_list_unicode(self): - # mybotunicode.egg has two spiders, araña1 and araña2 - self.add_test_version("mybotunicode.egg", "mybotunicode", "r1") - spiders = get_spider_list("mybotunicode", pythonpath=get_pythonpath_scrapyd()) - - self.assertEqual(sorted(spiders), ["araña1", "araña2"]) - - def test_failed_spider_list(self): - self.add_test_version("mybot3.egg", "mybot3", "r1") - pypath = get_pythonpath_scrapyd() - # Workaround missing support for context manager in twisted < 15 - - # Add -W ignore to sub-python to prevent warnings & tb mixup in stderr - def popen_wrapper(*args, **kwargs): - cmd, args = args[0], args[1:] - cmd = [cmd[0], "-W", "ignore"] + cmd[1:] - return Popen(cmd, *args, **kwargs) - - with mock.patch("scrapyd.utils.Popen", wraps=popen_wrapper): - exc = self.assertRaises(RunnerError, get_spider_list, "mybot3", pythonpath=pypath) - self.assertRegex(str(exc).rstrip(), r"Exception: This should break the `scrapy list` command$") - - -@pytest.mark.parametrize( - ("versions", "expected"), - [ - (["zzz", "b", "ddd", "a", "x"], ["a", "b", "ddd", "x", "zzz"]), - (["10", "1", "9"], ["1", "9", "10"]), - (["2.11", "2.01", "2.9"], ["2.01", "2.9", "2.11"]), - ], -) -def test_sorted_versions(versions, expected): - assert sorted_versions(versions) == expected