Merge branch 'develop' into THREAT-367/snowflake-streaming-rules

.github/workflows/check-deprecated.yml

@@ -0,0 +1,42 @@
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  check_removed_rules:
+    name: Check Removed Rules
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
+        with:
+          disable-sudo: true
+          egress-policy: block
+          allowed-endpoints: >
+            files.pythonhosted.org:443
+            github.com:443
+            pypi.org:443
+      - name: Checkout panther-analysis
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1
+
+      - name: Fetch Release
+        run: |
+          git fetch --depth=1 origin develop
+
+      - name: Set python version
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 #v5.2.0
+        with:
+          python-version: "3.11"
+
+      - name: Install pipenv
+        run: pip install pipenv
+
+      - name: Setup venv
+        run: make venv
+
+      - name: Check for Removed Rules
+        run: |
+          pipenv run make check-deprecated
+

.github/workflows/check-mitre.yml

@@ -19,7 +19,7 @@ jobs:
             github.com:443
             pypi.org:443
       - name: Checkout panther-analysis
-        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 #v4.2.0
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1
 
       - name: Set python version
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 #v5.2.0

.github/workflows/check-packs.yml

@@ -22,7 +22,7 @@ jobs:
             pypi.org:443
 
       - name: Checkout panther-analysis
-        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 #v4.2.0
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1
 
       - name: Set python version
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 #v5.2.0
@@ -42,7 +42,7 @@ jobs:
           panther_analysis_tool check-packs || echo "errors=`cat errors.txt`" >> $GITHUB_OUTPUT
 
       - name: Comment PR
-        uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6
+        uses: thollander/actions-comment-pull-request@e2c37e53a7d2227b61585343765f73a9ca57eda9
         if: ${{ steps.check-packs.outputs.errors }}
         with:
           mode: upsert
@@ -54,7 +54,7 @@ jobs:
             ```
           comment_tag: check-packs
       - name: Delete comment
-        uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6
+        uses: thollander/actions-comment-pull-request@e2c37e53a7d2227b61585343765f73a9ca57eda9
         if: ${{ !steps.check-packs.outputs.errors }}
         with:
           mode: delete

.github/workflows/docker.yml

@@ -27,11 +27,11 @@ jobs:
             registry-1.docker.io:443
             www.python.org:443
       - name: Checkout panther-analysis
-        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 #v4.2.0
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1
       - uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf #v3.2.0
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db #v3.6.1
+        uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 #v3.7.1
       - name: Build Image
         run: docker buildx build --load -f Dockerfile -t panther-analysis:latest .
       - name: Test Image

.github/workflows/lint.yml

@@ -19,7 +19,7 @@ jobs:
             github.com:443
             pypi.org:443
       - name: Checkout panther-analysis
-        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 #v4.2.0
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1
 
       - name: Set python version
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 #v5.2.0

.github/workflows/release.yml

@@ -18,7 +18,7 @@ jobs:
       - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
         with:
           egress-policy: audit
-      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 #v4.2.0
+      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1
         with:
           fetch-depth: 0
           token: ${{ env.GITHUB_TOKEN }}

.github/workflows/sync-from-upstream.yml

@@ -37,7 +37,7 @@ jobs:
           branch: "sync_upstream_${{steps.set_upstream.outputs.latest-release}}"
       # Checkout this repo into the branch
       - name: Checkout your local repo in PR branch
-        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 #v4.2.0
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1
         with:
           ref: "sync_upstream_${{steps.set_upstream.outputs.latest-release}}"
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -24,7 +24,7 @@ jobs:
           exit 0
           
       - name: Checkout panther-analysis
-        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 #v4.2.0
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1
 
       - name: Set python version
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 #v5.2.0

.github/workflows/upload.yml

@@ -25,7 +25,7 @@ jobs:
           exit 0
 
       - name: Checkout panther-analysis
-        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 #v4.2.0
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1
 
       - name: Set python version
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 #v5.2.0

.github/workflows/validate.yml

@@ -24,7 +24,7 @@ jobs:
           exit 0
 
       - name: Checkout panther-analysis
-        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 #v4.2.0
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1
 
       - name: Set python version
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 #v5.2.0

.scripts/deleted_rules.py

@@ -0,0 +1,102 @@
+""" Checks to see if an Analysis item was removed from the repo, and whether it was added to the
+deprecated.txt file. """
+
+import argparse
+import logging
+import os
+import re
+import subprocess
+
+import panther_analysis_tool.command.bulk_delete as pat_delete
+import panther_analysis_tool.util as pat_util
+
+diff_pattern = re.compile(r'^-(?:RuleID|PolicyID|QueryName):\s*"?([\w.]+)"?')
+
+
+def get_deleted_ids() -> set[str]:
+    # Run git diff, get output
+    result = subprocess.run(["git", "diff", "origin/develop", "HEAD"], capture_output=True)
+    if result.stderr:
+        raise Exception(result.stderr.decode("utf-8"))
+
+    ids = set()
+    for line in result.stdout.decode("utf-8").split("\n"):
+        if m := diff_pattern.match(line):
+            # Add the ID to the list
+            ids.add(m.group(1))
+
+    return ids
+
+
+def get_deprecated_ids() -> set[str]:
+    """Returns all the IDs listed in `deprecated.txt`."""
+    with open("deprecated.txt", "r") as f:
+        return set(f.read().split("\n"))
+
+
+def check(_):
+    if ids := get_deleted_ids() - get_deprecated_ids():
+        print("❌ The following rule IDs may have been deleted:")
+        for id_ in ids:
+            print(f"\t{id_}")
+        exit(1)
+    else:
+        print("✅ No unaccounted deletions found! You're in the clear! 👍")
+
+
+def remove(args):
+    api_token = args.api_token or os.environ.get("PANTHER_API_TOKEN")
+    api_host = args.api_host or os.environ.get("PANTHER_API_HOST")
+
+    if not (api_token and api_host):
+        opts = []
+        if not api_token:
+            print("No API token was found or provided!")
+            opts.append("--api-token")
+        if not api_host:
+            print("No API host was found or provided!")
+            opts.append("--api-host")
+        print(f"You can pass API credentials using {' and '.join(opts)} in your command.")
+        exit(1)
+
+    ids = list(get_deprecated_ids())
+
+    pat_args = argparse.Namespace(
+        analysis_id=ids, query_id=[], confirm_bypass=True, api_token=api_token, api_host=api_host
+    )
+
+    logging.basicConfig(
+        format="[%(levelname)s][%(name)s]: %(message)s",
+        level=logging.INFO,
+    )
+
+    return_code, out = pat_util.func_with_api_backend(pat_delete.run)(pat_args)
+
+    if return_code == 1:
+        if out:
+            logging.error(out)
+    elif return_code == 0:
+        if out:
+            logging.info(out)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers(title="subcommands")
+
+    check_help = "Check if any items have been removed and not added to deprecated.txt"
+    parser_check = subparsers.add_parser("check", help=check_help)
+    parser_check.set_defaults(func=check)
+
+    remove_help = "Delete the entires listed in deprecated.txt"
+    parser_remove = subparsers.add_parser("remove", help=remove_help)
+    parser_remove.add_argument("--api-token", type=str, required=False)
+    parser_remove.add_argument("--api-host", type=str, required=False)
+    parser_remove.set_defaults(func=remove)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

.scripts/mitre_mapping_check.py

@@ -11,28 +11,31 @@
 # All MITRE Tags must match this regex pattern
 MITRE_PATTERN = re.compile("^TA\d+\:T\d+(\.\d+)?$")
 
+
 def main(path: Path) -> bool:
+    # Ignore any schema test files
+    #   Schema tests can't be loaded by panther_analysis_tool because each file contains multiple
+    #   YAML documents.
+    ignore_files = list(path.glob("**/*_tests.y*ml"))
+
     # Load Repo
-    analysis_items = load_analysis_specs([path], ignore_files=[])
+    analysis_items = load_analysis_specs([path], ignore_files=ignore_files)
 
-    items_with_invalid_mappings = [] # Record all items with bad tags
+    items_with_invalid_mappings = []  # Record all items with bad tags
     for analysis_item in analysis_items:
-        rel_path = analysis_item[0] # Relative path to YAML file
-        spec = analysis_item[2] # YAML spec as a dict
+        rel_path = analysis_item[0]  # Relative path to YAML file
+        spec = analysis_item[2]  # YAML spec as a dict
 
-        bad_tags = [] # Record the invalid tags for this analysis item
+        bad_tags = []  # Record the invalid tags for this analysis item
         if reports := spec.get("Reports"):
             if mitre := reports.get("MITRE ATT&CK"):
                 for mapping in mitre:
                     if not MITRE_PATTERN.match(mapping):
                         bad_tags.append(mapping)
-        
+
         if bad_tags:
-            items_with_invalid_mappings.append({
-                "rel_path": rel_path,
-                "bad_tags": bad_tags
-            })
-
+            items_with_invalid_mappings.append({"rel_path": rel_path, "bad_tags": bad_tags})
+
     if items_with_invalid_mappings:
         print("❌ Some items had invalid MITRE mapping formats:")
         print()
@@ -42,16 +45,21 @@ def main(path: Path) -> bool:
                 print("\t" + bad_tag)
             print()
 
-        print(("To ensure that your MITRE mappings are correctly displayed in the Panther "
-               "console, make sure your MITRE mappings are formatted like 'TA0000:T0000'."))
+        print(
+            (
+                "To ensure that your MITRE mappings are correctly displayed in the Panther "
+                "console, make sure your MITRE mappings are formatted like 'TA0000:T0000'."
+            )
+        )
     else:
         print("✅ No invalid MITRE mappings found! You're in the clear! 👍")
-    
+
     return bool(items_with_invalid_mappings)
 
+
 if __name__ == "__main__":
-    path = Path.cwd() # Default to current directory
+    path = Path.cwd()  # Default to current directory
     if len(sys.argv) > 1:
         path = Path(sys.argv[1])
     if main(path):
-        exit(1) # Exit with error if issues were found
+        exit(1)  # Exit with error if issues were found

.vscode/rule_jsonschema.json

@@ -5,6 +5,9 @@
 
   "type": "object",
   "properties": {
+    "AlertTitle": {
+      "$ref": "#/definitions/AlertTitle"
+    },
     "AnalysisType": {
       "$ref": "#/definitions/AnalysisType"
     },
@@ -85,6 +88,12 @@
     }
   ],
   "definitions": {
+    "AlertTitle": {
+      "$comment": "https://docs.panther.com/detections/rules/writing-simple-detections#alerttitle",
+      "description": "Use AlertTitle to dynamically set the title of an alert generated by a match on this detection.",
+      "type": "string",
+      "default": "rule"
+    },
     "AnalysisType": {
       "description": "what kind of detection",
       "type": "string",

Makefile

@@ -62,6 +62,12 @@ install:
 test: global-helpers-unit-test
 	pipenv run panther_analysis_tool test $(TEST_ARGS)
 
+check-deprecated:
+	pipenv run python3 ./.scripts/deleted_rules.py check
+
+remove-deprecated:
+	pipenv run python3 ./.scripts/deleted_rules.py remove
+
 docker-build:
 	docker build -t panther-analysis:latest .