Skip to content

Commit

Permalink
test(webservice): Delete eggs_dir for Windows. Remove mock and python…
Browse files Browse the repository at this point in the history
…path.
  • Loading branch information
jpmckinney committed Jul 20, 2024
1 parent 03d5fc2 commit aaa02e1
Show file tree
Hide file tree
Showing 3 changed files with 31 additions and 42 deletions.
11 changes: 11 additions & 0 deletions tests/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import io
import os.path
import pkgutil
import shutil


def get_egg_data(basename):
Expand All @@ -8,3 +10,12 @@ def get_egg_data(basename):

def root_add_version(root, project, version, basename):
root.eggstorage.put(io.BytesIO(get_egg_data(basename)), project, version)


def clean(config, setting):
directory = os.path.realpath(config.get(setting))
basedir = os.path.realpath(os.path.dirname(os.path.dirname(__file__)))
# Avoid accidentally deleting directories outside the project.
assert os.path.commonprefix((directory, basedir)) == basedir
if os.path.exists(directory):
shutil.rmtree(directory)
12 changes: 2 additions & 10 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
import os.path
import shutil

import pytest
from twisted.web import http
from twisted.web.http import Request
Expand All @@ -9,7 +6,7 @@
from scrapyd import Config
from scrapyd.app import application
from scrapyd.website import Root
from tests import root_add_version
from tests import clean, root_add_version


@pytest.fixture()
Expand All @@ -30,12 +27,7 @@ def root(request):
yield Root(config, app)

for setting in ("dbs_dir", "eggs_dir"):
directory = os.path.realpath(config.get(setting))
basedir = os.path.realpath(os.path.dirname(os.path.dirname(__file__)))
# Avoid accidentally deleting directories outside the project.
assert os.path.commonprefix((directory, basedir)) == basedir
if os.path.exists(directory):
shutil.rmtree(directory)
clean(config, setting)


@pytest.fixture()
Expand Down
50 changes: 18 additions & 32 deletions tests/test_webservice.py
Original file line number Diff line number Diff line change
@@ -1,40 +1,24 @@
import io
import os
import re
from unittest import mock

import pytest
from scrapy.utils.test import get_pythonpath
from twisted.web import error

from scrapyd import get_application
from scrapyd.config import Config
from scrapyd.exceptions import DirectoryTraversalError, RunnerError
from scrapyd.interfaces import IEggStorage
from scrapyd.interfaces import IEggStorage, IJobStorage
from scrapyd.jobstorage import Job
from scrapyd.txapp import application
from scrapyd.webservice import UtilsCache, get_spider_list
from tests import get_egg_data, root_add_version


def fake_list_jobs(*args, **kwargs):
yield Job("proj1", "spider-a", "id1234")


def fake_list_spiders(*args, **kwargs):
return []


def fake_list_spiders_other(*args, **kwargs):
return ["quotesbot", "toscrape-css"]


def get_pythonpath_scrapyd():
scrapyd_path = __import__("scrapyd").__path__[0]
return os.path.join(os.path.dirname(scrapyd_path), get_pythonpath(), os.environ.get("PYTHONPATH", ""))
from tests import clean, get_egg_data, root_add_version


@pytest.fixture()
def app():
return get_application()
yield application

clean(Config(), "eggs_dir")


def add_test_version(app, project, version, basename):
Expand All @@ -43,7 +27,7 @@ def add_test_version(app, project, version, basename):

def test_get_spider_list_log_stdout(app):
add_test_version(app, "logstdout", "logstdout", "logstdout")
spiders = get_spider_list("logstdout", pythonpath=get_pythonpath_scrapyd())
spiders = get_spider_list("logstdout")

# If LOG_STDOUT were respected, the output would be [].
assert sorted(spiders) == ["spider1", "spider2"]
Expand All @@ -52,50 +36,50 @@ def test_get_spider_list_log_stdout(app):
def test_get_spider_list(app):
# mybot.egg has two spiders, spider1 and spider2
add_test_version(app, "mybot", "r1", "mybot")
spiders = get_spider_list("mybot", pythonpath=get_pythonpath_scrapyd())
spiders = get_spider_list("mybot")
assert sorted(spiders) == ["spider1", "spider2"]

# mybot2.egg has three spiders, spider1, spider2 and spider3...
# BUT you won't see it here because it's cached.
# Effectivelly it's like if version was never added
add_test_version(app, "mybot", "r2", "mybot2")
spiders = get_spider_list("mybot", pythonpath=get_pythonpath_scrapyd())
spiders = get_spider_list("mybot")
assert sorted(spiders) == ["spider1", "spider2"]

# Let's invalidate the cache for this project...
UtilsCache.invalid_cache("mybot")

# Now you get the updated list
spiders = get_spider_list("mybot", pythonpath=get_pythonpath_scrapyd())
spiders = get_spider_list("mybot")
assert sorted(spiders) == ["spider1", "spider2", "spider3"]

# Let's re-deploy mybot.egg and clear cache. It now sees 2 spiders
add_test_version(app, "mybot", "r3", "mybot")
UtilsCache.invalid_cache("mybot")
spiders = get_spider_list("mybot", pythonpath=get_pythonpath_scrapyd())
spiders = get_spider_list("mybot")
assert sorted(spiders) == ["spider1", "spider2"]

# And re-deploying the one with three (mybot2.egg) with a version that
# isn't the higher, won't change what get_spider_list() returns.
add_test_version(app, "mybot", "r1a", "mybot2")
UtilsCache.invalid_cache("mybot")
spiders = get_spider_list("mybot", pythonpath=get_pythonpath_scrapyd())
spiders = get_spider_list("mybot")
assert sorted(spiders) == ["spider1", "spider2"]


@pytest.mark.skipif(os.name == "nt", reason="get_spider_list() unicode fails on windows")
def test_get_spider_list_unicode(app):
# mybotunicode.egg has two spiders, araña1 and araña2
add_test_version(app, "mybotunicode", "r1", "mybotunicode")
spiders = get_spider_list("mybotunicode", pythonpath=get_pythonpath_scrapyd())
spiders = get_spider_list("mybotunicode")

assert sorted(spiders) == ["araña1", "araña2"]


def test_failed_spider_list(app):
add_test_version(app, "mybot3", "r1", "mybot3")
with pytest.raises(RunnerError) as exc:
get_spider_list("mybot3", pythonpath=get_pythonpath_scrapyd())
get_spider_list("mybot3")

assert re.search(f"Exception: This should break the `scrapy list` command{os.linesep}$", str(exc.value))

Expand Down Expand Up @@ -195,8 +179,10 @@ def test_list_jobs(txrequest, root_with_egg):
assert set(content) == {"node_name", "status", "pending", "running", "finished"}


@mock.patch("scrapyd.jobstorage.MemoryJobStorage.__iter__", new=fake_list_jobs)
def test_list_jobs_finished(txrequest, root_with_egg):
jobstorage = root_with_egg.app.getComponent(IJobStorage)
jobstorage.add(Job("proj1", "spider-a", "id1234"))

txrequest.args = {}
endpoint = b"listjobs.json"
content = root_with_egg.children[endpoint].render_GET(txrequest)
Expand Down

0 comments on commit aaa02e1

Please sign in to comment.