From a7da753f79c7c1fe6f05de01c01b9a25eba397fd Mon Sep 17 00:00:00 2001 From: Henri DF Date: Sun, 8 Nov 2020 21:43:15 +0100 Subject: [PATCH 01/42] Don't use tempfile.NamedTemporaryFile() It has problematic semantics on Windows. --- suricata/update/main.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/suricata/update/main.py b/suricata/update/main.py index 41cf0cb6..19d25a0f 100644 --- a/suricata/update/main.py +++ b/suricata/update/main.py @@ -182,13 +182,15 @@ def fetch(self, url): os.makedirs(config.get_cache_dir(), mode=0o770) logger.info("Fetching %s." % (url)) try: - tmp_fileobj = tempfile.NamedTemporaryFile() + tmp_dlobj = tempfile.TemporaryFile() net.get( net_arg, - tmp_fileobj, + tmp_dlobj, progress_hook=self.progress_hook) - shutil.copyfile(tmp_fileobj.name, tmp_filename) + tmp_fileobj=open(tmp_filename, 'w+') + shutil.copyfileobj(tmp_dlobj, tmp_fileobj) tmp_fileobj.close() + tmp_dlobj.close() except URLError as err: if os.path.exists(tmp_filename): logger.warning( From 820c2826b316fe086ceb81da5c1eb305db13d756 Mon Sep 17 00:00:00 2001 From: Henri DF Date: Mon, 9 Nov 2020 17:41:52 +0100 Subject: [PATCH 02/42] use shutil.move instead of os.rename (os.rename fails on Windows if dst file exists). --- suricata/update/commands/disablesource.py | 2 +- suricata/update/commands/enablesource.py | 2 +- suricata/update/main.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/suricata/update/commands/disablesource.py b/suricata/update/commands/disablesource.py index 6a64a7bd..33a43e2a 100644 --- a/suricata/update/commands/disablesource.py +++ b/suricata/update/commands/disablesource.py @@ -36,5 +36,5 @@ def disable_source(): logger.warning("Source %s is not enabled.", name) return 0 logger.debug("Renaming %s to %s.disabled.", filename, filename) - os.rename(filename, "%s.disabled" % (filename)) + shutil.move(filename, "%s.disabled" % (filename)) logger.info("Source %s has been disabled", name) diff --git a/suricata/update/commands/enablesource.py b/suricata/update/commands/enablesource.py index 53bb68a3..e9396f24 100644 --- a/suricata/update/commands/enablesource.py +++ b/suricata/update/commands/enablesource.py @@ -53,7 +53,7 @@ def enable_source(): disabled_source_filename = sources.get_disabled_source_filename(name) if os.path.exists(disabled_source_filename): logger.info("Re-enabling previously disabled source for %s.", name) - os.rename(disabled_source_filename, enabled_source_filename) + shutil.move(disabled_source_filename, enabled_source_filename) update_params = True if not os.path.exists(sources.get_index_filename()): diff --git a/suricata/update/main.py b/suricata/update/main.py index 19d25a0f..1bd3d85f 100644 --- a/suricata/update/main.py +++ b/suricata/update/main.py @@ -498,7 +498,7 @@ def write_merged(filename, rulemap, dep_files): else: handle_filehash_files(rule, dep_files, kw) print(rule.format(), file=fileobj) - os.rename(tmp_filename, filename) + shutil.move(tmp_filename, filename) def write_to_directory(directory, files, rulemap, dep_files): # List of rule IDs that have been added. @@ -558,7 +558,7 @@ def write_to_directory(directory, files, rulemap, dep_files): tmp_filename = ".".join([outpath, "tmp"]) io.open(tmp_filename, encoding="utf-8", mode="w").write( u"\n".join(content)) - os.rename(tmp_filename, outpath) + shutil.move(tmp_filename, outpath) def write_yaml_fragment(filename, files): logger.info( From 98a4c39b9f147ff56f518aca862ff2e536644162 Mon Sep 17 00:00:00 2001 From: Henri DF Date: Tue, 1 Dec 2020 18:18:50 +0100 Subject: [PATCH 03/42] Revert "Don't use tempfile.NamedTemporaryFile()" This reverts commit a7da753f79c7c1fe6f05de01c01b9a25eba397fd. --- suricata/update/main.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/suricata/update/main.py b/suricata/update/main.py index 1bd3d85f..6629f14a 100644 --- a/suricata/update/main.py +++ b/suricata/update/main.py @@ -182,15 +182,13 @@ def fetch(self, url): os.makedirs(config.get_cache_dir(), mode=0o770) logger.info("Fetching %s." % (url)) try: - tmp_dlobj = tempfile.TemporaryFile() + tmp_fileobj = tempfile.NamedTemporaryFile() net.get( net_arg, - tmp_dlobj, + tmp_fileobj, progress_hook=self.progress_hook) - tmp_fileobj=open(tmp_filename, 'w+') - shutil.copyfileobj(tmp_dlobj, tmp_fileobj) + shutil.copyfile(tmp_fileobj.name, tmp_filename) tmp_fileobj.close() - tmp_dlobj.close() except URLError as err: if os.path.exists(tmp_filename): logger.warning( From 0b10a8383fe1d2060589fa10822ee9dd223477b0 Mon Sep 17 00:00:00 2001 From: Henri DF Date: Tue, 1 Dec 2020 18:20:44 +0100 Subject: [PATCH 04/42] Fix file writing for Windows --- suricata/update/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/suricata/update/main.py b/suricata/update/main.py index 6629f14a..d145f36a 100644 --- a/suricata/update/main.py +++ b/suricata/update/main.py @@ -182,13 +182,14 @@ def fetch(self, url): os.makedirs(config.get_cache_dir(), mode=0o770) logger.info("Fetching %s." % (url)) try: - tmp_fileobj = tempfile.NamedTemporaryFile() + tmp_fileobj = tempfile.NamedTemporaryFile(delete=False) net.get( net_arg, tmp_fileobj, progress_hook=self.progress_hook) - shutil.copyfile(tmp_fileobj.name, tmp_filename) tmp_fileobj.close() + shutil.copyfile(tmp_fileobj.name, tmp_filename) + os.unlink(tmp_fileobj.name) except URLError as err: if os.path.exists(tmp_filename): logger.warning( From 1f9cfa524d3f5205346bfdf5e41e1d7dde85cc06 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Wed, 27 Jan 2021 15:29:29 -0600 Subject: [PATCH 05/42] github-ci: remove CentOS 6 build CentOS 6 has been end of lifed. --- .github/workflows/tests.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b8f372bc..410ce7a8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,23 +54,6 @@ jobs: - name: Python 3 integration tests run: PYTHONPATH=. python3 ./tests/integration_tests.py - centos-6: - name: CentOS 6 - runs-on: ubuntu-latest - container: centos:6 - steps: - - run: yum -y install epel-release - - run: | - yum -y install \ - python34-pytest \ - python34-yaml - - uses: actions/checkout@v1 - - - name: Python 3 unit tests - run: PYTHONPATH=. py.test-3 - - name: Python 3 integration tests - run: PYTHONPATH=. python3 ./tests/integration_tests.py - fedora-31: name: Fedora 31 runs-on: ubuntu-latest From 726c121a6765a16651994ed0bae67cddfdffdf3e Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Wed, 27 Jan 2021 15:32:14 -0600 Subject: [PATCH 06/42] github-ci: fix MacOS test Make sure Python 3 is used. --- .github/workflows/tests.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 410ce7a8..fba0ed65 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -196,10 +196,8 @@ jobs: runs-on: macos-latest steps: - run: brew install python - - run: pip install PyYAML - - run: pip install pytest + - run: pip3 install PyYAML + - run: pip3 install pytest - uses: actions/checkout@v1 - run: PYTHONPATH=. pytest - - run: PYTHONPATH=. python ./tests/integration_tests.py - - + - run: PYTHONPATH=. python3 ./tests/integration_tests.py From fa8fb27255c46c92d24a969518f4665c72d07dbd Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 15 Feb 2021 16:35:27 -0600 Subject: [PATCH 07/42] fix: fix --no-merge option The no-merge handling was not updated when the file storage was converted to a list causing it to fail. Also add a --no-merge test to our integration test. Fixes issue: https://redmine.openinfosecfoundation.org/issues/4324 --- suricata/update/main.py | 26 ++++++++++++++++---------- tests/integration_tests.py | 7 +++++++ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/suricata/update/main.py b/suricata/update/main.py index 41cf0cb6..b1f08191 100644 --- a/suricata/update/main.py +++ b/suricata/update/main.py @@ -508,9 +508,9 @@ def write_to_directory(directory, files, rulemap, dep_files): oldset = {} if not args.quiet: - for filename in files: + for file in files: outpath = os.path.join( - directory, os.path.basename(filename)) + directory, os.path.basename(file.filename)) if os.path.exists(outpath): for rule in rule_mod.parse_file(outpath): @@ -533,15 +533,15 @@ def write_to_directory(directory, files, rulemap, dep_files): len(removed), len(modified))) - for filename in sorted(files): + for file in sorted(files): outpath = os.path.join( - directory, os.path.basename(filename)) + directory, os.path.basename(file.filename)) logger.debug("Writing %s." % outpath) - if not filename.endswith(".rules"): - open(outpath, "wb").write(files[filename]) + if not file.filename.endswith(".rules"): + open(outpath, "wb").write(file.content) else: content = [] - for line in io.StringIO(files[filename].decode("utf-8")): + for line in io.StringIO(file.content.decode("utf-8")): rule = rule_mod.parse(line) if not rule: content.append(line.strip()) @@ -552,7 +552,13 @@ def write_to_directory(directory, files, rulemap, dep_files): handle_dataset_files(rule, dep_files) else: handle_filehash_files(rule, dep_files, kw) - content.append(rulemap[rule.id].format()) + if rule.id in rulemap: + content.append(rulemap[rule.id].format()) + else: + # Just pass the input through. Most likey a + # rule from a file that was ignored, but we'll + # still pass it through. + content.append(line.strip()) tmp_filename = ".".join([outpath, "tmp"]) io.open(tmp_filename, encoding="utf-8", mode="w").write( u"\n".join(content)) @@ -1234,10 +1240,10 @@ def _main(): file_tracker.add(output_filename) write_merged(os.path.join(output_filename), rulemap, dep_files) else: - for filename in files: + for file in files: file_tracker.add( os.path.join( - config.get_output_dir(), os.path.basename(filename))) + config.get_output_dir(), os.path.basename(file.filename))) write_to_directory(config.get_output_dir(), files, rulemap, dep_files) manage_classification(suriconf, classification_files) diff --git a/tests/integration_tests.py b/tests/integration_tests.py index 77057533..a421ebff 100755 --- a/tests/integration_tests.py +++ b/tests/integration_tests.py @@ -51,6 +51,13 @@ def delete(path): assert(os.path.exists(os.path.join(DATA_DIR, "update", "cache"))) assert(os.path.exists(os.path.join(DATA_DIR, "rules", "suricata.rules"))) +# Default run with data directory and --no-merge +run(common_args + common_update_args + ["--no-merge"]) +assert(os.path.exists(DATA_DIR)) +assert(os.path.exists(os.path.join(DATA_DIR, "update", "cache"))) +assert(os.path.exists(os.path.join(DATA_DIR, "rules", "emerging-deleted.rules"))) +assert(os.path.exists(os.path.join(DATA_DIR, "rules", "emerging-current_events.rules"))) + # Still a default run, but set --output to an alternate location." run(common_args + common_update_args + ["--output", "./tests/tmp/_rules"]) assert(os.path.exists(os.path.join(DATA_DIR, "_rules"))) From 019400c50e85770377532bc46bbecf64eaa42a80 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Wed, 17 Feb 2021 09:35:32 -0600 Subject: [PATCH 08/42] changelog: update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fc6b285..b0b8dcf4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Change Log +## unreleased +- Fix --no-merge. Redmine issue + https://redmine.openinfosecfoundation.org/issues/4324. + ## 1.2.0 - 2020-10-05 - Documentation updates. From 518b61c9c9db2b1e182c5f0d67067cf734624311 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 22 Feb 2021 22:48:42 -0600 Subject: [PATCH 09/42] doc: remove --merged option This option was never implemented and is an artifact of the tools suricata-update was based on. Redmine issue: https://redmine.openinfosecfoundation.org/issues/4215 --- doc/update.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/doc/update.rst b/doc/update.rst index 625f5a2b..ec692253 100644 --- a/doc/update.rst +++ b/doc/update.rst @@ -30,11 +30,6 @@ Options be due to just recently downloaded, or the remote checksum matching the cached copy. -.. option:: --merged= - - Write a single file containing all rules. This can be used in - addition to ``--output`` or instead of ``--output``. - .. option:: --no-merge Do not merge the rules into a single rule file. From 50e857f75e576e239d8306a6ac55946a1ce252a6 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 22 Feb 2021 22:54:11 -0600 Subject: [PATCH 10/42] version: 1.2.1 --- CHANGELOG.md | 2 +- suricata/update/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0b8dcf4..eac578e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log -## unreleased +## 1.2.1 - 2021-02-23 - Fix --no-merge. Redmine issue https://redmine.openinfosecfoundation.org/issues/4324. diff --git a/suricata/update/version.py b/suricata/update/version.py index 01c4c83a..97e80e5a 100644 --- a/suricata/update/version.py +++ b/suricata/update/version.py @@ -4,4 +4,4 @@ # Alpha: 1.0.0a1 # Development: 1.0.0dev0 # Release candidate: 1.0.0rc1 -version = "1.2.0" +version = "1.2.1" From 884d0c99c5d32cb9420105a1905d35c0c7041d89 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Tue, 13 Apr 2021 13:13:45 -0600 Subject: [PATCH 11/42] version: 1.3.0dev0 --- suricata/update/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/suricata/update/version.py b/suricata/update/version.py index 97e80e5a..1a4c44af 100644 --- a/suricata/update/version.py +++ b/suricata/update/version.py @@ -4,4 +4,4 @@ # Alpha: 1.0.0a1 # Development: 1.0.0dev0 # Release candidate: 1.0.0rc1 -version = "1.2.1" +version = "1.3.0dev0" From 7fafdcaf01361534b1c0c725363d35deb013b328 Mon Sep 17 00:00:00 2001 From: Andreas Dolp Date: Thu, 25 Feb 2021 19:23:09 +0100 Subject: [PATCH 12/42] HTTP BasicAuth support. Allow whitespaces in entire HTTP header value. This enables HTTP Basic Authentication. Redmine issue: https://redmine.openinfosecfoundation.org/issues/4362 --- doc/add-source.rst | 4 ++++ suricata/update/net.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/add-source.rst b/doc/add-source.rst index e5b2a532..17b80e6e 100644 --- a/doc/add-source.rst +++ b/doc/add-source.rst @@ -25,6 +25,10 @@ Options add-source --http-header "X-API-Key: 1234" + HTTP basic authentication can be achieved by setting the HTTP Basic + Authentication header with ``base64(user1:password1)``. Example:: + add-source --http-header "Authorization: Basic dXNlcjE6cGFzc3dvcmQx" + .. option:: --no-checksum Skips downloading the checksum URL for the rule source. diff --git a/suricata/update/net.py b/suricata/update/net.py index 49a58cf5..dddb3254 100644 --- a/suricata/update/net.py +++ b/suricata/update/net.py @@ -91,7 +91,7 @@ def is_header_clean(header): if len(header) != 2: return False name, val = header[0].strip(), header[1].strip() - if re.match( r"^[\w-]+$", name) and re.match(r"^[\w-]+$", val): + if re.match( r"^[\w-]+$", name) and re.match(r"^[\w\s-]+$", val): return True return False From b9b5ef1ca92cc67305296029ab7ac6c0ca7c4da6 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Tue, 13 Apr 2021 13:23:34 -0600 Subject: [PATCH 13/42] tests: add integration test for --http-header --- tests/integration_tests.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/integration_tests.py b/tests/integration_tests.py index a421ebff..878134a5 100755 --- a/tests/integration_tests.py +++ b/tests/integration_tests.py @@ -91,3 +91,16 @@ def delete(path): assert(not os.path.exists( os.path.join( DATA_DIR, "update", "sources", "oisf-trafficid.yaml.disabled"))) + +# Add a source with a custom header. +run(common_args + [ + "add-source", "--http-header", "Header: NoSpaces", + "testing-header-nospaces", + "file:///doesnotexist"]) + +# Add a source with a custom header with spaces in the value +# (https://redmine.openinfosecfoundation.org/issues/4362) +run(common_args + [ + "add-source", + "--http-header", "Authorization: Basic dXNlcjE6cGFzc3dvcmQx", + "testing-header-with-spaces", "file:///doesnotexist"]) From 6ce306a931e7bf92191009418f5188865393f615 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Tue, 13 Apr 2021 13:31:33 -0600 Subject: [PATCH 14/42] travis-ci: remove, appears they haven't been building And GitHub CI has all these tests covered. --- .travis.yml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0b74c709..00000000 --- a/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ -language: python - -python: - - "2.7" - - "3.5" - - "3.6" - -# This is required to test on Python 3.7 at the moment. -matrix: - include: - - python: 3.7 - dist: xenial - sudo: true - -script: - - PYTHONPATH=. pytest - - PYTHONPATH=. python ./tests/integration_tests.py From 07f69af9809f3988187b35e98240d0e9bf0d0dc6 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Tue, 13 Apr 2021 13:32:42 -0600 Subject: [PATCH 15/42] github-ci: add fedora 33, remove fedora 31 (eol) --- .github/workflows/tests.yml | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fba0ed65..98c1f89d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,34 +54,27 @@ jobs: - name: Python 3 integration tests run: PYTHONPATH=. python3 ./tests/integration_tests.py - fedora-31: - name: Fedora 31 + fedora-32: + name: Fedora 32 runs-on: ubuntu-latest - container: fedora:31 + container: fedora:32 steps: - run: | yum -y install \ - python2-pytest \ - python2-pyyaml \ python3 \ python3-pytest \ python3-pyyaml - - uses: actions/checkout@v1 - - - name: Python 2 unit tests - run: PYTHONPATH=. pytest-2 - - name: Python 2 integration tests - run: PYTHONPATH=. python2 ./tests/integration_tests.py + - uses: actions/checkout@v2 - name: Python 3 unit tests run: PYTHONPATH=. pytest-3 - name: Python 3 integration tests run: PYTHONPATH=. python3 ./tests/integration_tests.py - fedora-32: - name: Fedora 32 + fedora-33: + name: Fedora 33 runs-on: ubuntu-latest - container: fedora:32 + container: fedora:33 steps: - run: | yum -y install \ From f1992856e9c77a50475355707d1b17ba5807caa9 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Tue, 13 Apr 2021 13:34:12 -0600 Subject: [PATCH 16/42] github-ci: add Ubuntu 20.04 --- .github/workflows/tests.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 98c1f89d..211fee6e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -88,6 +88,30 @@ jobs: - name: Python 3 integration tests run: PYTHONPATH=. python3 ./tests/integration_tests.py + ubuntu-2004: + name: Ubuntu 20.04 + runs-on: ubuntu-latest + container: ubuntu:20.04 + steps: + - run: apt update + - run: | + apt -y install \ + python-pytest \ + python-yaml \ + python3-pytest \ + python3-yaml + - uses: actions/checkout@v1 + + - name: Python 2 unit tests + run: PYTHONPATH=. pytest + - name: Python 2 integration tests + run: PYTHONPATH=. python2 ./tests/integration_tests.py + + - name: Python 3 unit tests + run: PYTHONPATH=. pytest-3 + - name: Python 3 integration tests + run: PYTHONPATH=. python3 ./tests/integration_tests.py + ubuntu-1804: name: Ubuntu 18.04 runs-on: ubuntu-latest From c68baf18f3ba043c0c29ffd1aa14cb8b7547e345 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Tue, 13 Apr 2021 13:37:44 -0600 Subject: [PATCH 17/42] changelog: update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eac578e8..8d236ae8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Change Log +## unreleased +- Allow spaces in custom HTTP headers. Redmine issue + https://redmine.openinfosecfoundation.org/issues/4362 + ## 1.2.1 - 2021-02-23 - Fix --no-merge. Redmine issue https://redmine.openinfosecfoundation.org/issues/4324. From 7e472b5378e31033a2ca050765327ae2e466863f Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 17 May 2021 23:15:26 -0600 Subject: [PATCH 18/42] conf: allow no-test in configuration file If no-test was set in the configuration file, it was always being overridden by the command, even if not set on the command line. This change allow no-test to be set in the configuration file like: no-test: true Redmine issue: https://redmine.openinfosecfoundation.org/versions/168 --- suricata/update/parsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/suricata/update/parsers.py b/suricata/update/parsers.py index 6d86e455..9f2c9f2a 100644 --- a/suricata/update/parsers.py +++ b/suricata/update/parsers.py @@ -133,7 +133,7 @@ {'metavar': '', 'help': "Command to test Suricata configuration"}), (("--no-test",), - {'action': 'store_true', 'default': False, + {'action': 'store_true', 'default': None, 'help': "Disable testing rules with Suricata"}), (("--no-merge",), {'action': 'store_true', 'default': False, From 9364b5abbc3e5818c46b8f515783c29b5fbbd684 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 17 May 2021 23:54:40 -0600 Subject: [PATCH 19/42] testing: remove python 3.5, add python 3.8 Python 3.5 has been end of lifed. --- tox-integration.ini | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tox-integration.ini b/tox-integration.ini index 8926b1f4..dfa991f3 100644 --- a/tox-integration.ini +++ b/tox-integration.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py27, py35, py36, py37 +envlist = py27, py36, py37, py38 [testenv] commands = python ./tests/integration_tests.py diff --git a/tox.ini b/tox.ini index e599a77a..5ce1245d 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py27, py35, py36, py37 +envlist = py27, py36, py37, py38 [testenv] commands = pytest From 5829e0b3ccc2c97c4e16ffd513960c44d164b93f Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Tue, 18 May 2021 00:01:29 -0600 Subject: [PATCH 20/42] changelog: update --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d236ae8..c29c1f67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## unreleased - Allow spaces in custom HTTP headers. Redmine issue https://redmine.openinfosecfoundation.org/issues/4362 +- Fix "no-test" when set in configuration file: + https://redmine.openinfosecfoundation.org/issues/4493 ## 1.2.1 - 2021-02-23 - Fix --no-merge. Redmine issue From 74c3e6d2396176a0c64af54cf4680496f38749b8 Mon Sep 17 00:00:00 2001 From: Jascha Sticher Date: Fri, 16 Jul 2021 15:15:29 +0200 Subject: [PATCH 21/42] Use HTTP-Headers for checksum request --- suricata/update/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/suricata/update/main.py b/suricata/update/main.py index b1f08191..eed2805b 100644 --- a/suricata/update/main.py +++ b/suricata/update/main.py @@ -106,12 +106,13 @@ def __init__(self): def check_checksum(self, tmp_filename, url): try: - checksum_url = url + ".md5" + checksum_url = url[0] + ".md5" + net_arg=(checksum_url,url[1]) local_checksum = hashlib.md5( open(tmp_filename, "rb").read()).hexdigest().strip() remote_checksum_buf = io.BytesIO() logger.info("Checking %s." % (checksum_url)) - net.get(checksum_url, remote_checksum_buf) + net.get(net_arg, remote_checksum_buf) remote_checksum = remote_checksum_buf.getvalue().decode().strip() logger.debug("Local checksum=|%s|; remote checksum=|%s|" % ( local_checksum, remote_checksum)) @@ -174,7 +175,7 @@ def fetch(self, url): url) return self.extract_files(tmp_filename) if checksum: - if self.check_checksum(tmp_filename, url): + if self.check_checksum(tmp_filename, net_arg): logger.info("Remote checksum has not changed. " "Not fetching.") return self.extract_files(tmp_filename) From 720ec5642f97ccf370e02ce659f915de090513e6 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Thu, 7 Oct 2021 12:34:25 -0600 Subject: [PATCH 22/42] tests: fix checksum test for new argument type --- suricata/update/main.py | 1 + tests/test_main.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/suricata/update/main.py b/suricata/update/main.py index eed2805b..7d3b3664 100644 --- a/suricata/update/main.py +++ b/suricata/update/main.py @@ -105,6 +105,7 @@ def __init__(self): self.istty = os.isatty(sys.stdout.fileno()) def check_checksum(self, tmp_filename, url): + print(url) try: checksum_url = url[0] + ".md5" net_arg=(checksum_url,url[1]) diff --git a/tests/test_main.py b/tests/test_main.py index 1425cd19..86fa486a 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -62,7 +62,12 @@ def test_check_checksum(self): os.path.dirname(os.path.realpath(__file__))) local_file = "%s/emerging.rules.tar.gz" % ( os.path.dirname(os.path.realpath(__file__))) - r = fetch.check_checksum(local_file, url) + + # The URL passed to check_checksum is actually a tuple: + # (url, custom-header, has checksum url) + net_arg = (url, None, True) + + r = fetch.check_checksum(local_file, net_arg) self.assertTrue(r) class ThresholdProcessorTestCase(unittest.TestCase): From a120096f8d943e1abd7542ee1e2804421c1fff41 Mon Sep 17 00:00:00 2001 From: JacobRoed Date: Wed, 22 Sep 2021 09:14:33 +0200 Subject: [PATCH 23/42] Allow more characters in the custom headers Allow more characters in the custom header regular expression to allow for Oauth2 tokens. Feature #4701. [Commit message rewritten by jish@oisf.net with permission] --- doc/add-source.rst | 10 ++++++++++ suricata/update/net.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/add-source.rst b/doc/add-source.rst index 17b80e6e..53ae834d 100644 --- a/doc/add-source.rst +++ b/doc/add-source.rst @@ -24,6 +24,16 @@ Options as a custom API key. Example:: add-source --http-header "X-API-Key: 1234" + + HTTP basic authentication can be achieved by setting the HTTP Basic Authentication header with base64(user1:password1). Example:: + + add-source --http-header "Authorization: Basic dXNlcjE6cGFzc3dvcmQx" + + HTTP Bearer authentication can be used by setting the HTTP Bearer Authentication header + with a OAuth2 token containing printable ASCII characters. Example:: + + add-source --http-header "Auhorization: Bearer NjA2MTUOTAx?D+wOm4U/vpXQy0xhl!hSaR7#ENVpK59" + HTTP basic authentication can be achieved by setting the HTTP Basic Authentication header with ``base64(user1:password1)``. Example:: diff --git a/suricata/update/net.py b/suricata/update/net.py index dddb3254..5f4d08e7 100644 --- a/suricata/update/net.py +++ b/suricata/update/net.py @@ -91,7 +91,7 @@ def is_header_clean(header): if len(header) != 2: return False name, val = header[0].strip(), header[1].strip() - if re.match( r"^[\w-]+$", name) and re.match(r"^[\w\s-]+$", val): + if re.match( r"^[\w-]+$", name) and re.match(r"^[\w\s -~]+$", val): return True return False From eebba20fe754a50fd3a36ba88b615a42a86dbf7c Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Thu, 7 Oct 2021 12:22:00 -0600 Subject: [PATCH 24/42] changelog: update --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c29c1f67..49d5e985 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ https://redmine.openinfosecfoundation.org/issues/4362 - Fix "no-test" when set in configuration file: https://redmine.openinfosecfoundation.org/issues/4493 +- Allow more custom characters in custom http header to allow for more + of the base64 character set: + https://redmine.openinfosecfoundation.org/issues/4701 +- Send custom HTTP headers with check for remote checksum file: + https://redmine.openinfosecfoundation.org/issues/4001 ## 1.2.1 - 2021-02-23 - Fix --no-merge. Redmine issue From e8f77151ea127d149d9872536d939fb5688e9123 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Thu, 7 Oct 2021 13:16:49 -0600 Subject: [PATCH 25/42] doc: fix formatting error --- doc/add-source.rst | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/doc/add-source.rst b/doc/add-source.rst index 53ae834d..c49692c2 100644 --- a/doc/add-source.rst +++ b/doc/add-source.rst @@ -25,7 +25,8 @@ Options add-source --http-header "X-API-Key: 1234" - HTTP basic authentication can be achieved by setting the HTTP Basic Authentication header with base64(user1:password1). Example:: + HTTP basic authentication can be achieved by setting the HTTP Basic + Authentication header with ``base64(user1:password1)``. Example:: add-source --http-header "Authorization: Basic dXNlcjE6cGFzc3dvcmQx" @@ -33,11 +34,6 @@ Options with a OAuth2 token containing printable ASCII characters. Example:: add-source --http-header "Auhorization: Bearer NjA2MTUOTAx?D+wOm4U/vpXQy0xhl!hSaR7#ENVpK59" - - - HTTP basic authentication can be achieved by setting the HTTP Basic - Authentication header with ``base64(user1:password1)``. Example:: - add-source --http-header "Authorization: Basic dXNlcjE6cGFzc3dvcmQx" .. option:: --no-checksum From 947a4ddd58f583ef5c2a0963bd242c411b89f280 Mon Sep 17 00:00:00 2001 From: Andreas Dolp Date: Mon, 26 Jul 2021 21:44:09 +0200 Subject: [PATCH 26/42] Doc: Improve documentation of Suricata reload. Redmine issue: https://redmine.openinfosecfoundation.org/issues/4564 --- doc/quickstart.rst | 3 ++- doc/update.rst | 18 ++++++++++++++++-- suricata/update/configs/update.yaml | 2 ++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/doc/quickstart.rst b/doc/quickstart.rst index dabc1876..0f17a7b1 100644 --- a/doc/quickstart.rst +++ b/doc/quickstart.rst @@ -122,7 +122,8 @@ This command will: .. note:: Suricata-Update is also capable of triggering a rule reload, but doing so requires some extra configuration that will be - covered later. + covered later. See the documentation of + :command:`--reload-command=` for more details. Configure Suricata to Load Suricata-Update Managed Rules ======================================================== diff --git a/doc/update.rst b/doc/update.rst index ec692253..19ab5257 100644 --- a/doc/update.rst +++ b/doc/update.rst @@ -153,12 +153,26 @@ Options .. option:: --reload-command= A command to run after the rules have been updated; will not run if - no change to the output files was made. For example:: + no change to the output files was made. For example:: - --reload-command='sudo kill -USR2 $(cat /var/run/suricata.pid)' + --reload-command='sudo kill -USR2 $(pidof suricata)' will tell Suricata to reload its rules. + Furthermore the reload can be triggered using the Unix socket of Suricata. + + Blocking reload (with Suricata waiting for the reload to finish):: + + --reload-command='sudo suricatasc -c reload-rules' + + Non blocking reload (without restarting Suricata):: + + --reload-command='sudo suricatasc -c ruleset-reload-nonblocking' + + See the Suricata documentation on `Rule Reloads + `_ + for more information. + .. option:: --no-reload Disable Suricata rule reload. diff --git a/suricata/update/configs/update.yaml b/suricata/update/configs/update.yaml index e113f539..358e869d 100644 --- a/suricata/update/configs/update.yaml +++ b/suricata/update/configs/update.yaml @@ -36,6 +36,8 @@ ignore: # Provide a command to reload the Suricata rules. # May be overrided by the --reload-command command line option. +# See the documentation of --reload-command for the different options +# to reload Suricata rules. #reload-command: sudo systemctl reload suricata # Remote rule sources. Simply a list of URLs. From fd3b36d6c9f2289105822901aa044f725ff36802 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Thu, 28 Oct 2021 09:10:11 -0600 Subject: [PATCH 27/42] index: allow custom url for embedded index update --- suricata/update/data/update.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/suricata/update/data/update.py b/suricata/update/data/update.py index 1d05df4f..e57c1469 100644 --- a/suricata/update/data/update.py +++ b/suricata/update/data/update.py @@ -15,6 +15,7 @@ # 02110-1301, USA. import os.path +import sys try: from urllib2 import urlopen @@ -23,11 +24,16 @@ import yaml +DEFAULT_URL = "https://raw.githubusercontent.com/oisf/suricata-intel-index/master/index.yaml" + def embed_index(): """Embed a copy of the index as a Python source file. We can't use a datafile yet as there is no easy way to do with distutils.""" + if len(sys.argv) > 1: + url = sys.argv[1] + else: + url = DEFAULT_URL dist_filename = os.path.join(os.path.dirname(__file__), "index.py") - url = "https://raw.githubusercontent.com/oisf/suricata-intel-index/master/index.yaml" response = urlopen(url) index = yaml.safe_load(response.read()) From 2e08c36d3799ffcffdf21c482a887984c89fbedb Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Wed, 29 Dec 2021 11:09:11 -0600 Subject: [PATCH 28/42] integration tests: format --- tests/integration_tests.py | 80 ++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 34 deletions(-) diff --git a/tests/integration_tests.py b/tests/integration_tests.py index 878134a5..a15b4190 100755 --- a/tests/integration_tests.py +++ b/tests/integration_tests.py @@ -5,22 +5,24 @@ DATA_DIR = "./tests/tmp" + def run(args): subprocess.check_call(args) + def delete(path): if os.path.isdir(path): shutil.rmtree(path) else: os.unlink(path) + print("Python executable: %s" % sys.executable) print("Python version: %s" % str(sys.version)) print("Current directory: %s" % os.getcwd()) # Override the default source index URL to avoid hitting the network. -os.environ["SOURCE_INDEX_URL"] = "file://%s/tests/index.yaml" % ( - os.getcwd()) +os.environ["SOURCE_INDEX_URL"] = "file://%s/tests/index.yaml" % (os.getcwd()) os.environ["ETOPEN_URL"] = "file://%s/tests/emerging.rules.tar.gz" % ( os.getcwd()) @@ -31,76 +33,86 @@ def delete(path): common_args = [ sys.executable, "./bin/suricata-update", - "-D", DATA_DIR, - "-c", "./tests/empty", + "-D", + DATA_DIR, + "-c", + "./tests/empty", ] common_update_args = [ "--no-test", "--no-reload", - "--suricata-conf", "./tests/suricata.yaml", - "--disable-conf", "./tests/disable.conf", - "--enable-conf", "./tests/empty", - "--drop-conf", "./tests/empty", - "--modify-conf", "./tests/empty", + "--suricata-conf", + "./tests/suricata.yaml", + "--disable-conf", + "./tests/disable.conf", + "--enable-conf", + "./tests/empty", + "--drop-conf", + "./tests/empty", + "--modify-conf", + "./tests/empty", ] # Default run with data directory. run(common_args + common_update_args) -assert(os.path.exists(DATA_DIR)) -assert(os.path.exists(os.path.join(DATA_DIR, "update", "cache"))) -assert(os.path.exists(os.path.join(DATA_DIR, "rules", "suricata.rules"))) +assert (os.path.exists(DATA_DIR)) +assert (os.path.exists(os.path.join(DATA_DIR, "update", "cache"))) +assert (os.path.exists(os.path.join(DATA_DIR, "rules", "suricata.rules"))) # Default run with data directory and --no-merge run(common_args + common_update_args + ["--no-merge"]) -assert(os.path.exists(DATA_DIR)) -assert(os.path.exists(os.path.join(DATA_DIR, "update", "cache"))) -assert(os.path.exists(os.path.join(DATA_DIR, "rules", "emerging-deleted.rules"))) -assert(os.path.exists(os.path.join(DATA_DIR, "rules", "emerging-current_events.rules"))) +assert (os.path.exists(DATA_DIR)) +assert (os.path.exists(os.path.join(DATA_DIR, "update", "cache"))) +assert (os.path.exists( + os.path.join(DATA_DIR, "rules", "emerging-deleted.rules"))) +assert (os.path.exists( + os.path.join(DATA_DIR, "rules", "emerging-current_events.rules"))) # Still a default run, but set --output to an alternate location." run(common_args + common_update_args + ["--output", "./tests/tmp/_rules"]) -assert(os.path.exists(os.path.join(DATA_DIR, "_rules"))) +assert (os.path.exists(os.path.join(DATA_DIR, "_rules"))) # Update sources. run(common_args + ["update-sources"]) -assert(os.path.exists(os.path.join(DATA_DIR, "update", "cache", "index.yaml"))) +assert (os.path.exists(os.path.join(DATA_DIR, "update", "cache", + "index.yaml"))) # Now delete the index and run lists-sources to see if it downloads # the index. delete(os.path.join(DATA_DIR, "update", "cache", "index.yaml")) run(common_args + ["list-sources"]) -assert(os.path.exists(os.path.join(DATA_DIR, "update", "cache", "index.yaml"))) +assert (os.path.exists(os.path.join(DATA_DIR, "update", "cache", + "index.yaml"))) # Enable a source. run(common_args + ["enable-source", "oisf/trafficid"]) -assert(os.path.exists( +assert (os.path.exists( os.path.join(DATA_DIR, "update", "sources", "oisf-trafficid.yaml"))) # Disable the source. run(common_args + ["disable-source", "oisf/trafficid"]) -assert(not os.path.exists( - os.path.join( - DATA_DIR, "update", "sources", "oisf-trafficid.yaml"))) -assert(os.path.exists( - os.path.join( - DATA_DIR, "update", "sources", "oisf-trafficid.yaml.disabled"))) +assert (not os.path.exists( + os.path.join(DATA_DIR, "update", "sources", "oisf-trafficid.yaml"))) +assert (os.path.exists( + os.path.join(DATA_DIR, "update", "sources", + "oisf-trafficid.yaml.disabled"))) # Remove the source. run(common_args + ["remove-source", "oisf/trafficid"]) -assert(not os.path.exists( - os.path.join( - DATA_DIR, "update", "sources", "oisf-trafficid.yaml.disabled"))) +assert (not os.path.exists( + os.path.join(DATA_DIR, "update", "sources", + "oisf-trafficid.yaml.disabled"))) # Add a source with a custom header. run(common_args + [ "add-source", "--http-header", "Header: NoSpaces", - "testing-header-nospaces", - "file:///doesnotexist"]) + "testing-header-nospaces", "file:///doesnotexist" +]) # Add a source with a custom header with spaces in the value # (https://redmine.openinfosecfoundation.org/issues/4362) run(common_args + [ - "add-source", - "--http-header", "Authorization: Basic dXNlcjE6cGFzc3dvcmQx", - "testing-header-with-spaces", "file:///doesnotexist"]) + "add-source", "--http-header", "Authorization: Basic dXNlcjE6cGFzc3dvcmQx", + "testing-header-with-spaces", "file:///doesnotexist" +]) From f3d6d1b9b4aae9602c02c334bd78b522f4689084 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Wed, 29 Dec 2021 12:51:23 -0600 Subject: [PATCH 29/42] tests: add integration tests for multiple modifications --- tests/integration_tests.py | 104 +++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/tests/integration_tests.py b/tests/integration_tests.py index a15b4190..b32e7145 100755 --- a/tests/integration_tests.py +++ b/tests/integration_tests.py @@ -2,6 +2,8 @@ import os import subprocess import shutil +import tempfile +import suricata.update.rule DATA_DIR = "./tests/tmp" @@ -116,3 +118,105 @@ def delete(path): "add-source", "--http-header", "Authorization: Basic dXNlcjE6cGFzc3dvcmQx", "testing-header-with-spaces", "file:///doesnotexist" ]) + + +class IntegrationTest: + def __init__(self, configs={}): + self.directory = tempfile.mkdtemp(dir=DATA_DIR) + self.configs = configs + self.args = [] + self.write_configs() + + if not "update.yaml" in self.configs: + self.args += ["-c", "./tests/empty"] + + def write_configs(self): + for config in self.configs: + config_filename = "%s/%s" % (self.directory, config) + with open(config_filename, "w") as of: + of.write(self.configs[config]) + if config == "modify.conf": + self.args += ["--modify-conf", config_filename] + elif config == "drop.conf": + self.args += ["--drop-conf", config_filename] + elif config == "enable.conf": + self.args += ["--enable-conf", config_filename] + elif config == "disable.conf": + self.args += ["--disable-conf", config_filename] + + def run(self): + args = [ + sys.executable, + "./bin/suricata-update", + "-D", + self.directory, + "--no-test", + "--no-reload", + "--suricata-conf", + "./tests/suricata.yaml", + ] + self.args + subprocess.check_call(args) + self.check() + self.clean() + + def clean(self): + if self.directory.startswith(DATA_DIR): + shutil.rmtree(self.directory) + + def check(self): + pass + + def get_rule_by_sid(self, sid): + """ Return all rules where the provided substring is found. """ + with open("%s/rules/suricata.rules" % (self.directory)) as inf: + for line in inf: + rule = suricata.update.rule.parse(line) + if rule.sid == sid: + return rule + return None + + +class MultipleModifyTest(IntegrationTest): + + configs = { + "modify.conf": + """ +modifysid emerging-exploit.rules "^alert" | "drop" +modifysid * "^drop(.*)noalert(.*)" | "alert${1}noalert${2}" + """ + } + + def __init__(self): + IntegrationTest.__init__(self, self.configs) + + def check(self): + # This rule should have been converted to drop. + rule1 = self.get_rule_by_sid(2103461) + assert(rule1.action == "drop") + + # This one should have been converted back to alert. + rule2 = self.get_rule_by_sid(2023184) + assert(rule2.action == "alert") + +class DropAndModifyTest(IntegrationTest): + + configs = { + "drop.conf": """ +2024029 + """, + "modify.conf": """ +2024029 "ET INFO" "TEST INFO" + """ + } + + def __init__(self): + IntegrationTest.__init__(self, self.configs) + + def check(self): + rule1 = self.get_rule_by_sid(2024029) + assert(rule1.action == "drop") + assert(rule1.msg.startswith("TEST INFO")) + + +MultipleModifyTest().run() +DropAndModifyTest().run() From a4694f76d85c8a08c3878650a1f0ed71b272ad1e Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Wed, 29 Dec 2021 10:33:42 -0600 Subject: [PATCH 30/42] Fix multiple modifications not having an affect. In the case where multiple modifications changed a rule (drop included), only the last modification took affect. This is due to the modifications each being done on a clean version of the rule, rather than the previously modified version of the rule. Ticket #4259 --- suricata/update/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/suricata/update/main.py b/suricata/update/main.py index 7d3b3664..49627867 100644 --- a/suricata/update/main.py +++ b/suricata/update/main.py @@ -1193,15 +1193,15 @@ def _main(): for fltr in drop_filters: if fltr.match(rule): - rulemap[rule.id] = fltr.run(rule) + rule = fltr.run(rule) drop_count += 1 for fltr in modify_filters: if fltr.match(rule): - new_rule = fltr.run(rule) - if new_rule: - rulemap[rule.id] = new_rule - modify_count += 1 + rule = fltr.run(rule) + modify_count += 1 + + rulemap[key] = rule # Check if we should disable ja3 rules. try: From ba4dc1109d060c5ae9a1516666470f1e0adbe0f9 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Wed, 29 Dec 2021 11:04:25 -0600 Subject: [PATCH 31/42] Fix counter accuracy. Instead of counting how many modifications took place, just count the number of rules that were modified. --- suricata/update/main.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/suricata/update/main.py b/suricata/update/main.py index 49627867..22b9da14 100644 --- a/suricata/update/main.py +++ b/suricata/update/main.py @@ -1179,6 +1179,12 @@ def _main(): for key, rule in rulemap.items(): + # To avoid duplicate counts when a rule has more than one modification + # to it, we track the actions here then update the counts at the end. + enabled = False + modified = False + dropped = False + for matcher in disable_matchers: if rule.enabled and matcher.match(rule): logger.debug("Disabling: %s" % (rule.brief())) @@ -1189,17 +1195,24 @@ def _main(): if not rule.enabled and matcher.match(rule): logger.debug("Enabling: %s" % (rule.brief())) rule.enabled = True - enable_count += 1 + enabled = True for fltr in drop_filters: if fltr.match(rule): rule = fltr.run(rule) - drop_count += 1 + dropped = True for fltr in modify_filters: if fltr.match(rule): rule = fltr.run(rule) - modify_count += 1 + modified = True + + if enabled: + enable_count += 1 + if modified: + modify_count += 1 + if dropped: + drop_count += 1 rulemap[key] = rule From 6eb0ac2a7b672ec3ec011875c7aab49d7ad1ae6c Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Wed, 5 Jan 2022 11:42:37 -0600 Subject: [PATCH 32/42] changelog: update --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49d5e985..3a1feff2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Change Log ## unreleased +- Fix multiple modifications to a rule: + https://redmine.openinfosecfoundation.org/issues/4259 - Allow spaces in custom HTTP headers. Redmine issue https://redmine.openinfosecfoundation.org/issues/4362 - Fix "no-test" when set in configuration file: From 00d1b7322da9bbf7da6ac442da9356a9a7dbe61a Mon Sep 17 00:00:00 2001 From: Joe Atzberger Date: Thu, 18 Nov 2021 18:30:19 -0500 Subject: [PATCH 33/42] trivial typo in comment --- suricata/update/net.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/suricata/update/net.py b/suricata/update/net.py index 5f4d08e7..eac060e9 100644 --- a/suricata/update/net.py +++ b/suricata/update/net.py @@ -98,7 +98,7 @@ def is_header_clean(header): def get(url, fileobj, progress_hook=None): """ Perform a GET request against a URL writing the contents into - the provideded file like object. + the provided file-like object. :param url: The URL to fetch :param fileobj: The fileobj to write the content to From 1ea732764cafdad1d37f82c8d1a3f9beddaae64a Mon Sep 17 00:00:00 2001 From: Riju <19.riju@gmail.com> Date: Sat, 17 Oct 2020 02:07:13 +0530 Subject: [PATCH 34/42] logging: Improve flowbit logs Add and update debug logs for flowbit requirements to display pass Ticket #3205. --- suricata/update/main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/suricata/update/main.py b/suricata/update/main.py index 22b9da14..4a1c569e 100644 --- a/suricata/update/main.py +++ b/suricata/update/main.py @@ -627,13 +627,15 @@ def dump_sample_configs(): def resolve_flowbits(rulemap, disabled_rules): flowbit_resolver = rule_mod.FlowbitResolver() flowbit_enabled = set() + pass_ = 1 while True: + logger.debug("Checking flowbits for pass %d of rules.", pass_) flowbits = flowbit_resolver.get_required_flowbits(rulemap) logger.debug("Found %d required flowbits.", len(flowbits)) required_rules = flowbit_resolver.get_required_rules(rulemap, flowbits) logger.debug( - "Found %d rules to enable to for flowbit requirements", - len(required_rules)) + "Found %d rules to enable for flowbit requirements (pass %d)", + len(required_rules), pass_) if not required_rules: logger.debug("All required rules enabled.") break @@ -645,6 +647,7 @@ def resolve_flowbits(rulemap, disabled_rules): rule.enabled = True rule.noalert = True flowbit_enabled.add(rule) + pass_ = pass_ + 1 logger.info("Enabled %d rules for flowbit dependencies." % ( len(flowbit_enabled))) From edb88e5a0b306848692820dd95eb959d822f43d5 Mon Sep 17 00:00:00 2001 From: Tharushi Jayasekara Date: Sat, 17 Oct 2020 09:56:00 +0530 Subject: [PATCH 35/42] doc: add --show-advanced to common options Ticket #3974. --- doc/common-options.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/common-options.rst b/doc/common-options.rst index bd08cf0e..d56df99d 100644 --- a/doc/common-options.rst +++ b/doc/common-options.rst @@ -44,3 +44,8 @@ .. option:: --user-agent Set a custom user agent string for HTTP requests. + +.. option:: -s, --show-advanced + + Show advanced options. + \ No newline at end of file From afe925721a31747e59f1ffb6c5f56b37c5f03874 Mon Sep 17 00:00:00 2001 From: Tharushi Jayasekara Date: Thu, 15 Oct 2020 19:04:26 +0530 Subject: [PATCH 36/42] help output: hide advanced options behind a flag Cleaning up the suricata help output and hiding the extra options behind a "--show-advanced" option. Hidden options are: * --user-agent * --no-check-certificate * --yaml-fragment * --url * --local * --sid-msg-map * --sid-msg-map-2 * --ignore * --no-ignore * --threshold-in * --threshold-out * --dump-sample-configs * --etopen * --reload-command * --test-command * --no-merge Ticket #3974. --- suricata/update/parsers.py | 64 ++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 20 deletions(-) diff --git a/suricata/update/parsers.py b/suricata/update/parsers.py index 9f2c9f2a..adda46b2 100644 --- a/suricata/update/parsers.py +++ b/suricata/update/parsers.py @@ -31,6 +31,11 @@ default_update_yaml = config.DEFAULT_UPDATE_YAML_PATH +show_advanced = False + +if "-s" in sys.argv or "--show-advanced" in sys.argv: + show_advanced = True + # Global arguments - command line options for suricata-update global_arg = [ (("-v", "--verbose"), @@ -56,13 +61,18 @@ 'help': "Override Suricata version"}), (("--user-agent",), {'metavar': '', - 'help': "Set custom user-agent string"}), + 'help': "Set custom user-agent string" + if show_advanced else argparse.SUPPRESS}), (("--no-check-certificate",), {'action': 'store_true', 'default': None, - 'help': "Disable server SSL/TLS certificate verification"}), + 'help': "Disable server SSL/TLS certificate verification" + if show_advanced else argparse.SUPPRESS}), (("-V", "--version"), {'action': 'store_true', 'default': False, - 'help': "Display version"}) + 'help': "Display version"}), + (("-s","--show-advanced"), + {'action': 'store_true', + 'help': "Show advanced options"}), ] # Update arguments - command line options for suricata-update @@ -74,22 +84,27 @@ {'action': 'store_true', 'default': False, 'help': "Force operations that might otherwise be skipped"}), (("--yaml-fragment",), - {'metavar': '', - 'help': "Output YAML fragment for rule inclusion"}), + {'metavar': '', + 'help': "Output YAML fragment for rule inclusion" + if show_advanced else argparse.SUPPRESS}), (("--url",), {'metavar': '', 'action': 'append', 'default': [], 'help': "URL to use instead of auto-generating one " - "(can be specified multiple times)"}), + "(can be specified multiple times)" + if show_advanced else argparse.SUPPRESS}), (("--local",), {'metavar': '', 'action': 'append', 'default': [], 'help': "Local rule files or directories " - "(can be specified multiple times)"}), + "(can be specified multiple times)" + if show_advanced else argparse.SUPPRESS}), (("--sid-msg-map",), - {'metavar': '', - 'help': "Generate a sid-msg.map file"}), + {'metavar': '', + 'help': "Generate a sid-msg.map file" + if show_advanced else argparse.SUPPRESS}), (("--sid-msg-map-2",), {'metavar': '', - 'help': "Generate a v2 sid-msg.map file"}), + 'help': "Generate a v2 sid-msg.map file" + if show_advanced else argparse.SUPPRESS}), (("--disable-conf",), {'metavar': '', @@ -107,37 +122,46 @@ (("--ignore",), {'metavar': '', 'action': 'append', 'default': None, 'help': "Filenames to ignore " - "(can be specified multiple times; default: *deleted.rules)"}), + "(can be specified multiple times; default: *deleted.rules)" + if show_advanced else argparse.SUPPRESS}), (("--no-ignore",), - {'action': 'store_true', 'default': False, - 'help': "Disables the ignore option."}), + {'action': 'store_true', 'default': False, + 'help': "Disables the ignore option." + if show_advanced else argparse.SUPPRESS}), (("--threshold-in",), {'metavar': '', - 'help': "Filename of rule thresholding configuration"}), + 'help': "Filename of rule thresholding configuration" + if show_advanced else argparse.SUPPRESS}), (("--threshold-out",), {'metavar': '', - 'help': "Output of processed threshold configuration"}), + 'help': "Output of processed threshold configuration" + if show_advanced else argparse.SUPPRESS}), (("--dump-sample-configs",), {'action': 'store_true', 'default': False, - 'help': "Dump sample config files to current directory"}), + 'help': "Dump sample config files to current directory" + if show_advanced else argparse.SUPPRESS}), (("--etopen",), {'action': 'store_true', - 'help': "Use ET-Open rules (default)"}), + 'help': "Use ET-Open rules (default)" + if show_advanced else argparse.SUPPRESS}), (("--reload-command",), {'metavar': '', - 'help': "Command to run after update if modified"}), + 'help': "Command to run after update if modified" + if show_advanced else argparse.SUPPRESS}), (("--no-reload",), {'action': 'store_true', 'default': False, 'help': "Disable reload"}), (("-T", "--test-command"), {'metavar': '', - 'help': "Command to test Suricata configuration"}), + 'help': "Command to test Suricata configuration" + if show_advanced else argparse.SUPPRESS}), (("--no-test",), {'action': 'store_true', 'default': None, 'help': "Disable testing rules with Suricata"}), (("--no-merge",), {'action': 'store_true', 'default': False, - 'help': "Do not merge the rules into a single file"}), + 'help': "Do not merge the rules into a single file" + if show_advanced else argparse.SUPPRESS}), (("--offline",), {'action': 'store_true', 'help': "Run offline using most recent cached rules"}), From cb2df52d16ab54f9aac0ec872aac1eeeba3325cb Mon Sep 17 00:00:00 2001 From: Tharushi Jayasekara Date: Thu, 8 Oct 2020 22:16:21 +0530 Subject: [PATCH 37/42] Add warning to update sources if no index found Don't automatically update the sources on list-sources if it does not exist. Instead just use the bundled version. Ticket #3249. --- suricata/update/commands/listsources.py | 9 +++------ tests/integration_tests.py | 3 +-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/suricata/update/commands/listsources.py b/suricata/update/commands/listsources.py index 103d77bd..d35c3cd2 100644 --- a/suricata/update/commands/listsources.py +++ b/suricata/update/commands/listsources.py @@ -21,7 +21,6 @@ from suricata.update import config from suricata.update import sources from suricata.update import util -from suricata.update.commands.updatesources import update_sources from suricata.update import exceptions logger = logging.getLogger() @@ -72,11 +71,9 @@ def list_sources(): free_only = config.args().free if not sources.source_index_exists(config): - logger.info("No source index found, running update-sources") - try: - update_sources() - except exceptions.ApplicationError as err: - logger.warning("%s: will use bundled index.", err) + logger.warning("Source index does not exist, will use bundled one.") + logger.warning("Please run suricata-update update-sources.") + index = sources.load_source_index(config) for name, source in index.get_sources().items(): is_not_free = source.get("subscribe-url") diff --git a/tests/integration_tests.py b/tests/integration_tests.py index b32e7145..89705853 100755 --- a/tests/integration_tests.py +++ b/tests/integration_tests.py @@ -84,8 +84,7 @@ def delete(path): # the index. delete(os.path.join(DATA_DIR, "update", "cache", "index.yaml")) run(common_args + ["list-sources"]) -assert (os.path.exists(os.path.join(DATA_DIR, "update", "cache", - "index.yaml"))) +assert(not os.path.exists(os.path.join(DATA_DIR, "update", "cache", "index.yaml"))) # Enable a source. run(common_args + ["enable-source", "oisf/trafficid"]) From 39a2624ab42f2e2c6bfaf3d158402a6540327a9b Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Fri, 7 Jan 2022 12:04:48 -0600 Subject: [PATCH 38/42] changelog: update --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a1feff2..cb2c6e08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ # Change Log ## unreleased +- Be consistent about warning about old index. The index won't be automatically + updated now in some cases and not in others. Instead opt to never + auto-update: https://redmine.openinfosecfoundation.org/issues/3249 +- Better flowbit resolution logging in verbose mode + https://redmine.openinfosecfoundation.org/issues/3205 +- Hide advanced command line options from help output: + https://redmine.openinfosecfoundation.org/issues/3974 - Fix multiple modifications to a rule: https://redmine.openinfosecfoundation.org/issues/4259 - Allow spaces in custom HTTP headers. Redmine issue From 6f6d8dc67c8fc36f080c26a725869bd5c05711c4 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 28 Mar 2022 16:04:32 -0600 Subject: [PATCH 39/42] github-ci: convert centos 8 build to almalinux 8 --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 211fee6e..f4a16509 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,10 +6,10 @@ on: jobs: - centos-8: - name: CentOS 8 + alma-8: + name: AlmaLinux 8 runs-on: ubuntu-latest - container: centos:8 + container: almalinux:8 steps: - run: | yum -y install \ From 8159187f52f6fc7e4f7df1138ab01cfe95054048 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 28 Mar 2022 16:06:58 -0600 Subject: [PATCH 40/42] github-ci: update fedora tests to f34 and f35 --- .github/workflows/tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f4a16509..091da29c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,10 +54,10 @@ jobs: - name: Python 3 integration tests run: PYTHONPATH=. python3 ./tests/integration_tests.py - fedora-32: - name: Fedora 32 + fedora-34: + name: Fedora 34 runs-on: ubuntu-latest - container: fedora:32 + container: fedora:34 steps: - run: | yum -y install \ @@ -71,10 +71,10 @@ jobs: - name: Python 3 integration tests run: PYTHONPATH=. python3 ./tests/integration_tests.py - fedora-33: - name: Fedora 33 + fedora-35: + name: Fedora 35 runs-on: ubuntu-latest - container: fedora:33 + container: fedora:35 steps: - run: | yum -y install \ From 099754bfd910070050a4a9b6e7741e47ba466434 Mon Sep 17 00:00:00 2001 From: Shivani Bhardwaj Date: Wed, 29 Dec 2021 16:23:57 +0530 Subject: [PATCH 41/42] checkversions: fix wrong version checks Ticket 4373 --- suricata/update/commands/checkversions.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/suricata/update/commands/checkversions.py b/suricata/update/commands/checkversions.py index 98a0f385..34923170 100644 --- a/suricata/update/commands/checkversions.py +++ b/suricata/update/commands/checkversions.py @@ -21,6 +21,26 @@ logger = logging.getLogger() +def is_gt(v1, v2): + if v1.full == v2.full: + return False + + if v1.major < v2.major: + return False + elif v1.major > v2.major: + return True + + if v1.minor < v2.minor: + return False + elif v1.minor > v2.minor: + return True + + if v1.patch < v2.patch: + return False + + return True + + def register(parser): parser.set_defaults(func=check_version) @@ -42,7 +62,7 @@ def check_version(suricata_version): logger.error("Recommended version was not parsed properly") sys.exit(1) # In case index is out of date - if float(suricata_version.short) > float(recommended.short): + if is_gt(suricata_version, recommended): return # Evaluate if the installed version is present in index upgrade_version = version["suricata"].get(suricata_version.short) From 353dd227edf1958822821e83c85d785b400fff32 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 28 Mar 2022 16:23:57 -0600 Subject: [PATCH 42/42] changelog: add fix for issue 4373 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb2c6e08..f69c767d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ https://redmine.openinfosecfoundation.org/issues/4701 - Send custom HTTP headers with check for remote checksum file: https://redmine.openinfosecfoundation.org/issues/4001 +- Fix "check-versions" where the running Suricata is newer than what the index + knows about: https://redmine.openinfosecfoundation.org/issues/4373 ## 1.2.1 - 2021-02-23 - Fix --no-merge. Redmine issue