request for upstream PR

.github/workflows/tests.yml

@@ -6,10 +6,10 @@ on:
 
 jobs:
 
-  centos-8:
-    name: CentOS 8
+  alma-8:
+    name: AlmaLinux 8
     runs-on: ubuntu-latest
-    container: centos:8
+    container: almalinux:8
     steps:
       - run: |
           yum -y install \
@@ -54,58 +54,58 @@ jobs:
       - name: Python 3 integration tests
         run: PYTHONPATH=. python3 ./tests/integration_tests.py
 
-  centos-6:
-    name: CentOS 6
+  fedora-34:
+    name: Fedora 34
     runs-on: ubuntu-latest
-    container: centos:6
+    container: fedora:34
     steps:
-      - run: yum -y install epel-release
       - run: |
           yum -y install \
-            python34-pytest \
-            python34-yaml
-      - uses: actions/checkout@v1
+            python3 \
+            python3-pytest \
+            python3-pyyaml
+      - uses: actions/checkout@v2
 
       - name: Python 3 unit tests
-        run: PYTHONPATH=. py.test-3
+        run: PYTHONPATH=. pytest-3
       - name: Python 3 integration tests
         run: PYTHONPATH=. python3 ./tests/integration_tests.py
 
-  fedora-31:
-    name: Fedora 31
+  fedora-35:
+    name: Fedora 35
     runs-on: ubuntu-latest
-    container: fedora:31
+    container: fedora:35
     steps:
       - run: |
           yum -y install \
-            python2-pytest \
-            python2-pyyaml \
             python3 \
             python3-pytest \
             python3-pyyaml
-      - uses: actions/checkout@v1
-
-      - name: Python 2 unit tests
-        run: PYTHONPATH=. pytest-2
-      - name: Python 2 integration tests
-        run: PYTHONPATH=. python2 ./tests/integration_tests.py
+      - uses: actions/checkout@v2
 
       - name: Python 3 unit tests
         run: PYTHONPATH=. pytest-3
       - name: Python 3 integration tests
         run: PYTHONPATH=. python3 ./tests/integration_tests.py
 
-  fedora-32:
-    name: Fedora 32
+  ubuntu-2004:
+    name: Ubuntu 20.04
     runs-on: ubuntu-latest
-    container: fedora:32
+    container: ubuntu:20.04
     steps:
+      - run: apt update
       - run: |
-          yum -y install \
-            python3 \
+          apt -y install \
+            python-pytest \
+            python-yaml \
             python3-pytest \
-            python3-pyyaml
-      - uses: actions/checkout@v2
+            python3-yaml
+      - uses: actions/checkout@v1
+
+      - name: Python 2 unit tests
+        run: PYTHONPATH=. pytest
+      - name: Python 2 integration tests
+        run: PYTHONPATH=. python2 ./tests/integration_tests.py
 
       - name: Python 3 unit tests
         run: PYTHONPATH=. pytest-3
@@ -213,10 +213,8 @@ jobs:
     runs-on: macos-latest
     steps:
       - run: brew install python
-      - run: pip install PyYAML
-      - run: pip install pytest
+      - run: pip3 install PyYAML
+      - run: pip3 install pytest
       - uses: actions/checkout@v1
       - run: PYTHONPATH=. pytest
-      - run: PYTHONPATH=. python ./tests/integration_tests.py
-
-
+      - run: PYTHONPATH=. python3 ./tests/integration_tests.py

CHANGELOG.md

@@ -1,5 +1,31 @@
 # Change Log
 
+## unreleased
+- Be consistent about warning about old index. The index won't be automatically
+  updated now in some cases and not in others. Instead opt to never
+  auto-update: https://redmine.openinfosecfoundation.org/issues/3249
+- Better flowbit resolution logging in verbose mode
+  https://redmine.openinfosecfoundation.org/issues/3205
+- Hide advanced command line options from help output:
+  https://redmine.openinfosecfoundation.org/issues/3974
+- Fix multiple modifications to a rule:
+  https://redmine.openinfosecfoundation.org/issues/4259
+- Allow spaces in custom HTTP headers. Redmine issue
+  https://redmine.openinfosecfoundation.org/issues/4362
+- Fix "no-test" when set in configuration file:
+  https://redmine.openinfosecfoundation.org/issues/4493
+- Allow more custom characters in custom http header to allow for more
+  of the base64 character set:
+  https://redmine.openinfosecfoundation.org/issues/4701
+- Send custom HTTP headers with check for remote checksum file:
+  https://redmine.openinfosecfoundation.org/issues/4001
+- Fix "check-versions" where the running Suricata is newer than what the index
+  knows about: https://redmine.openinfosecfoundation.org/issues/4373
+
+## 1.2.1 - 2021-02-23
+- Fix --no-merge. Redmine issue
+  https://redmine.openinfosecfoundation.org/issues/4324.
+
 ## 1.2.0 - 2020-10-05
 - Documentation updates.
 

doc/add-source.rst

@@ -24,6 +24,16 @@ Options
    as a custom API key. Example::
 
      add-source --http-header "X-API-Key: 1234"
+
+   HTTP basic authentication can be achieved by setting the HTTP Basic
+   Authentication header with ``base64(user1:password1)``. Example::
+
+      add-source --http-header "Authorization: Basic dXNlcjE6cGFzc3dvcmQx"
+
+   HTTP Bearer authentication can be used by setting the HTTP Bearer Authentication header 
+   with a OAuth2 token containing printable ASCII characters. Example::
+
+      add-source --http-header "Auhorization: Bearer NjA2MTUOTAx?D+wOm4U/vpXQy0xhl!hSaR7#ENVpK59"
 
 .. option:: --no-checksum
 

doc/common-options.rst

@@ -44,3 +44,8 @@
 .. option:: --user-agent <string>
 
    Set a custom user agent string for HTTP requests.
+
+.. option:: -s, --show-advanced
+
+   Show advanced options.
+

doc/quickstart.rst

@@ -122,7 +122,8 @@ This command will:
 
 .. note:: Suricata-Update is also capable of triggering a rule reload,
           but doing so requires some extra configuration that will be
-          covered later.
+          covered later. See the documentation of
+          :command:`--reload-command=<command>` for more details.
 
 Configure Suricata to Load Suricata-Update Managed Rules
 ========================================================

doc/update.rst

@@ -30,11 +30,6 @@ Options
    be due to just recently downloaded, or the remote checksum matching
    the cached copy.
 
-.. option:: --merged=<filename>
-
-   Write a single file containing all rules. This can be used in
-   addition to ``--output`` or instead of ``--output``.
-
 .. option:: --no-merge
 
    Do not merge the rules into a single rule file.
@@ -158,12 +153,26 @@ Options
 .. option:: --reload-command=<command>
 
    A command to run after the rules have been updated; will not run if
-   no change to the output files was made.  For example::
+   no change to the output files was made. For example::
 
-     --reload-command='sudo kill -USR2 $(cat /var/run/suricata.pid)'
+     --reload-command='sudo kill -USR2 $(pidof suricata)'
 
    will tell Suricata to reload its rules.
 
+   Furthermore the reload can be triggered using the Unix socket of Suricata.
+
+   Blocking reload (with Suricata waiting for the reload to finish)::
+
+     --reload-command='sudo suricatasc -c reload-rules'
+
+   Non blocking reload (without restarting Suricata)::
+
+     --reload-command='sudo suricatasc -c ruleset-reload-nonblocking'
+
+   See the Suricata documentation on `Rule Reloads
+   <https://suricata.readthedocs.io/en/latest/rule-management/rule-reload.html>`_
+   for more information.
+
 .. option:: --no-reload
 
    Disable Suricata rule reload.

suricata/update/commands/checkversions.py

@@ -21,6 +21,26 @@
 logger = logging.getLogger()
 
 
+def is_gt(v1, v2):
+    if v1.full == v2.full:
+        return False
+
+    if v1.major < v2.major:
+        return False
+    elif v1.major > v2.major:
+        return True
+
+    if v1.minor < v2.minor:
+        return False
+    elif v1.minor > v2.minor:
+        return True
+
+    if v1.patch < v2.patch:
+        return False
+
+    return True
+
+
 def register(parser):
     parser.set_defaults(func=check_version)
 
@@ -42,7 +62,7 @@ def check_version(suricata_version):
         logger.error("Recommended version was not parsed properly")
         sys.exit(1)
     # In case index is out of date
-    if float(suricata_version.short) > float(recommended.short):
+    if is_gt(suricata_version, recommended):
         return
     # Evaluate if the installed version is present in index
     upgrade_version = version["suricata"].get(suricata_version.short)

suricata/update/commands/disablesource.py

@@ -36,5 +36,5 @@ def disable_source():
         logger.warning("Source %s is not enabled.", name)
         return 0
     logger.debug("Renaming %s to %s.disabled.", filename, filename)
-    os.rename(filename, "%s.disabled" % (filename))
+    shutil.move(filename, "%s.disabled" % (filename))
     logger.info("Source %s has been disabled", name)

suricata/update/commands/enablesource.py

@@ -53,7 +53,7 @@ def enable_source():
     disabled_source_filename = sources.get_disabled_source_filename(name)
     if os.path.exists(disabled_source_filename):
         logger.info("Re-enabling previously disabled source for %s.", name)
-        os.rename(disabled_source_filename, enabled_source_filename)
+        shutil.move(disabled_source_filename, enabled_source_filename)
         update_params = True
 
     if not os.path.exists(sources.get_index_filename()):

suricata/update/commands/listsources.py

@@ -21,7 +21,6 @@
 from suricata.update import config
 from suricata.update import sources
 from suricata.update import util
-from suricata.update.commands.updatesources import update_sources
 from suricata.update import exceptions
 
 logger = logging.getLogger()
@@ -72,11 +71,9 @@ def list_sources():
 
     free_only = config.args().free
     if not sources.source_index_exists(config):
-        logger.info("No source index found, running update-sources")
-        try:
-            update_sources()
-        except exceptions.ApplicationError as err:
-            logger.warning("%s: will use bundled index.", err)
+        logger.warning("Source index does not exist, will use bundled one.")
+        logger.warning("Please run suricata-update update-sources.")
+
     index = sources.load_source_index(config)
     for name, source in index.get_sources().items():
         is_not_free = source.get("subscribe-url")

suricata/update/configs/update.yaml

@@ -36,6 +36,8 @@ ignore:
 
 # Provide a command to reload the Suricata rules.
 # May be overrided by the --reload-command command line option.
+# See the documentation of --reload-command for the different options
+# to reload Suricata rules.
 #reload-command: sudo systemctl reload suricata
 
 # Remote rule sources. Simply a list of URLs.

suricata/update/data/update.py

@@ -15,6 +15,7 @@
 # 02110-1301, USA.
 
 import os.path
+import sys
 
 try:
     from urllib2 import urlopen
@@ -23,11 +24,16 @@
 
 import yaml
 
+DEFAULT_URL = "https://raw.githubusercontent.com/oisf/suricata-intel-index/master/index.yaml"
+
 def embed_index():
     """Embed a copy of the index as a Python source file. We can't use a
     datafile yet as there is no easy way to do with distutils."""
+    if len(sys.argv) > 1:
+        url = sys.argv[1]
+    else:
+        url = DEFAULT_URL
     dist_filename = os.path.join(os.path.dirname(__file__), "index.py")
-    url = "https://raw.githubusercontent.com/oisf/suricata-intel-index/master/index.yaml"
     response = urlopen(url)
     index = yaml.safe_load(response.read())