Change test suite web server (gunicorn) launch to a proper pytest fixture

tests/conftest.py

@@ -1,8 +1,30 @@
 """pytest setup and fixtures common for all tests, regardless of suite"""
 
 import os
+import platform
+import subprocess
 
 import pytest
+import requests
+from requests.adapters import HTTPAdapter, Retry
+
+
+def pytest_configure(config):
+    # Bootstrap Django config
+    from nav.bootstrap import bootstrap_django
+
+    bootstrap_django('pytest')
+
+    # Setup test environment for Django
+    from django.test.utils import setup_test_environment
+
+    setup_test_environment()
+
+    if platform.system() == 'Linux':
+        # Install custom reactor for Twisted tests
+        from nav.ipdevpoll.epollreactor2 import install
+
+        install()
 
 
 @pytest.fixture(scope='session')
@@ -13,3 +35,36 @@ def admin_username():
 @pytest.fixture(scope='session')
 def admin_password():
     return os.environ.get('ADMINPASSWORD', 'admin')
+
+
+@pytest.fixture(scope='session')
+def gunicorn():
+    """Sets up NAV to be served by a gunicorn instance.
+
+    Useful for tests that need to make external HTTP requests to NAV.
+
+    Returns the (expected) base URL of the gunicorn instance.
+    """
+    workspace = os.path.join(os.environ.get('WORKSPACE', ''), 'reports')
+    errorlog = os.path.join(workspace, 'gunicorn-error.log')
+    accesslog = os.path.join(workspace, 'gunicorn-access.log')
+    gunicorn = subprocess.Popen(
+        [
+            'gunicorn',
+            '--error-logfile',
+            errorlog,
+            '--access-logfile',
+            accesslog,
+            'navtest_wsgi:application',
+        ]
+    )
+    # Allow for gunicorn to become ready to serve requests before handing off to a test
+    session = requests.Session()
+    retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504])
+    base_url = "http://localhost:8000"
+    session.mount(base_url, HTTPAdapter(max_retries=retries))
+    session.get(base_url)
+
+    yield base_url
+
+    gunicorn.terminate()

tests/functional/conftest.py

@@ -7,7 +7,6 @@
 from selenium.webdriver.support import expected_conditions as EC
 
 USERNAME = 'admin'
-gunicorn = None
 
 ########################################################################
 #                                                                      #
@@ -27,31 +26,6 @@
 
 def pytest_configure(config):
     subprocess.check_call([SCRIPT_CREATE_DB])
-    start_gunicorn()
-
-    # Bootstrap Django config
-    from nav.bootstrap import bootstrap_django
-
-    bootstrap_django('pytest')
-
-
-def pytest_unconfigure(config):
-    stop_gunicorn()
-
-
-def start_gunicorn():
-    global gunicorn
-    gunicorn_log = open("reports/gunicorn.log", "ab")
-    gunicorn = subprocess.Popen(
-        ['gunicorn', 'navtest_wsgi:application'],
-        stdout=gunicorn_log,
-        stderr=subprocess.STDOUT,
-    )
-
-
-def stop_gunicorn():
-    if gunicorn:
-        gunicorn.terminate()
 
 
 ############
@@ -98,8 +72,11 @@ def __call__(self, driver):
 
 
 @pytest.fixture(scope="session")
-def base_url():
-    return os.environ.get('TARGETURL', 'http://localhost:8000')
+def base_url(gunicorn):
+    """Used as shorthand for the base URL of the test web server, but really just
+    invokes the gunicorn fixture.
+    """
+    yield gunicorn
 
 
 @pytest.fixture

tests/integration/conftest.py

@@ -1,7 +1,6 @@
 import os
 import importlib.util
 import io
-import platform
 import re
 import shlex
 from itertools import cycle
@@ -13,7 +12,6 @@
 import pytest
 from django.test import Client
 
-gunicorn = None
 
 ########################################################################
 #                                                                      #
@@ -33,50 +31,6 @@
 
 def pytest_configure(config):
     subprocess.check_call([SCRIPT_CREATE_DB])
-    os.environ['TARGETURL'] = "http://localhost:8000/"
-    start_gunicorn()
-
-    # Bootstrap Django config
-    from nav.bootstrap import bootstrap_django
-
-    bootstrap_django('pytest')
-
-    if platform.system() == 'Linux':
-        # Install custom reactor for Twisted tests
-        from nav.ipdevpoll.epollreactor2 import install
-
-        install()
-
-    # Setup test environment for Django
-    from django.test.utils import setup_test_environment
-
-    setup_test_environment()
-
-
-def pytest_unconfigure(config):
-    stop_gunicorn()
-
-
-def start_gunicorn():
-    global gunicorn
-    workspace = os.path.join(os.environ.get('WORKSPACE', ''), 'reports')
-    errorlog = os.path.join(workspace, 'gunicorn-error.log')
-    accesslog = os.path.join(workspace, 'gunicorn-access.log')
-    gunicorn = subprocess.Popen(
-        [
-            'gunicorn',
-            '--error-logfile',
-            errorlog,
-            '--access-logfile',
-            accesslog,
-            'navtest_wsgi:application',
-        ]
-    )
-
-
-def stop_gunicorn():
-    if gunicorn:
-        gunicorn.terminate()
 
 
 ########################################################################

tests/integration/web/crawler_test.py

@@ -14,32 +14,28 @@
 from collections import namedtuple
 from http.client import BadStatusLine
 from lxml.html import fromstring
-import os
 
 import pytest
 
 from tidylib import tidy_document
 from urllib.request import (
-    urlopen,
-    build_opener,
-    install_opener,
     HTTPCookieProcessor,
     Request,
+    build_opener,
+    install_opener,
+    urlopen,
 )
 from urllib.error import HTTPError, URLError
 from urllib.parse import (
-    urlsplit,
+    quote,
     urlencode,
-    urlunparse,
+    urljoin,
     urlparse,
-    quote,
+    urlsplit,
+    urlunparse,
 )
-from mock import Mock
 
 
-HOST_URL = os.environ.get('TARGETURL', None)
-USERNAME = os.environ.get('ADMINUSERNAME', 'admin')
-PASSWORD = os.environ.get('ADMINPASSWORD', 'admin')
 TIMEOUT = 90  # seconds?
 
 TIDY_OPTIONS = {
@@ -73,14 +69,6 @@
 
 Page = namedtuple('Page', 'url response content_type content')
 
-if not HOST_URL:
-    pytest.skip(
-        msg="Missing environment variable TARGETURL "
-        "(ADMINUSERNAME, ADMINPASSWORD) , skipping crawler "
-        "tests!",
-        allow_module_level=True,
-    )
-
 
 def normalize_path(url):
     url = urlsplit(url).path.rstrip('/')
@@ -177,7 +165,7 @@ def _queue_links_from(self, content, base_url):
                 self.queue.append('%s://%s%s' % (url.scheme, url.netloc, url.path))
 
     def login(self):
-        login_url = '%sindex/login/' % self.base_url
+        login_url = urljoin(self.base_url, '/index/login/')
         opener = build_opener(HTTPCookieProcessor())
         data = urlencode({'username': self.username, 'password': self.password})
         opener.open(login_url, data.encode('utf-8'), TIMEOUT)
@@ -226,8 +214,8 @@ def _quote_url(url):
 
 
 @pytest.fixture(scope="session")
-def webcrawler():
-    crawler = WebCrawler(HOST_URL, USERNAME, PASSWORD)
+def webcrawler(gunicorn, admin_username, admin_password):
+    crawler = WebCrawler(gunicorn, admin_username, admin_password)
     yield crawler