Show prevalence of rules in the output

CHANGELOG.md

@@ -8,6 +8,7 @@
 - binja: add support for forwarded exports #1646 @xusheng6
 - binja: add support for symtab names #1504 @xusheng6
 - add com class/interface features #322 @Aayush-goel-04
+- Show prevalence of rules in the output #520 @Aayush-Goel-04
 
 ### Breaking Changes
 

capa/render/default.py

@@ -7,6 +7,7 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import collections
+from typing import Dict
 
 import tabulate
 
@@ -72,19 +73,30 @@ def rec(match: rd.Match):
 
 def render_capabilities(doc: rd.ResultDocument, ostream: StringIO):
     """
+    render capabilities sorted by:
+      - prevalence (rare to unknown)
+      - namespace (alphabetical)
+
     example::
 
-        +-------------------------------------------------------+-------------------------------------------------+
-        | CAPABILITY                                            | NAMESPACE                                       |
-        |-------------------------------------------------------+-------------------------------------------------|
-        | check for OutputDebugString error (2 matches)         | anti-analysis/anti-debugging/debugger-detection |
-        | read and send data from client to server              | c2/file-transfer                                |
-        | ...                                                   | ...                                             |
-        +-------------------------------------------------------+-------------------------------------------------+
+        +-------------------------------------------------------+-------------------------------------------------+------------+
+        | CAPABILITY                                            | NAMESPACE                                       | PREVALENCE |
+        |-------------------------------------------------------+-------------------------------------------------|------------|
+        | check for OutputDebugString error (2 matches)         | anti-analysis/anti-debugging/debugger-detection | rare       |
+        | ...                                                   | ...                                             | ...        |
+        |-------------------------------------------------------|-------------------------------------------------|------------|
+        | read and send data from client to server              | c2/file-transfer                                | common     |
+        | ...                                                   | ...                                             | ...        |
+        +-------------------------------------------------------+-------------------------------------------------+------------+
     """
     subrule_matches = find_subrule_matches(doc)
 
-    rows = []
+    # seperate rules based on their prevalence
+    common: Dict[str, str] = {"capability": "", "namespace": "", "prevalence": ""}
+    had_common = False
+    rare: Dict[str, str] = {"capability": "", "namespace": "", "prevalence": ""}
+    had_rare = False
+
     for rule in rutils.capability_rules(doc):
         if rule.meta.name in subrule_matches:
             # rules that are also matched by other rules should not get rendered by default.
@@ -97,11 +109,34 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO):
             capability = rutils.bold(rule.meta.name)
         else:
             capability = f"{rutils.bold(rule.meta.name)} ({count} matches)"
-        rows.append((capability, rule.meta.namespace))
+
+        namespace = rule.meta.namespace if rule.meta.namespace is not None else ""
+        prevalence = rutils.bold(rule.meta.prevalence) if rule.meta.prevalence != "unknown" else "unknown"
+
+        if "rare" in prevalence:
+            rare["capability"] += capability + "\n"
+            rare["namespace"] += namespace + "\n"
+            rare["prevalence"] += prevalence + "\n"
+            had_rare = True
+        else:
+            common["capability"] += capability + "\n"
+            common["namespace"] += namespace + "\n"
+            common["prevalence"] += prevalence + "\n"
+            had_common = True
+
+    rows = []
+    if had_rare:
+        rows.append((rare["capability"], rare["namespace"], rare["prevalence"]))
+    if had_common:
+        rows.append((common["capability"], common["namespace"], common["prevalence"]))
 
     if rows:
         ostream.write(
-            tabulate.tabulate(rows, headers=[width("Capability", 50), width("Namespace", 50)], tablefmt="mixed_outline")
+            tabulate.tabulate(
+                rows,
+                headers=[width("Capability", 50), width("Namespace", 50), width("Prevalence", 10)],
+                tablefmt="mixed_grid",
+            )
         )
         ostream.write("\n")
     else:

capa/render/result_document.py

@@ -5,13 +5,17 @@
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+import gzip
+import json
 import datetime
 import collections
 from typing import Dict, List, Tuple, Union, Literal, Optional
 from pathlib import Path
+from functools import lru_cache
 
 from pydantic import Field, BaseModel, ConfigDict
 
+import capa.main
 import capa.rules
 import capa.engine
 import capa.features.common
@@ -501,9 +505,20 @@ class MaecMetadata(FrozenModel):
     model_config = ConfigDict(frozen=True, populate_by_name=True)
 
 
+@lru_cache(maxsize=None)
+def load_rules_prevalence() -> Dict[str, str]:
+    CD = capa.main.get_default_root()
+    file = CD / "assets" / "rules_prevalence_data" / "rules_prevalence.json.gz"
+    if not file.exists():
+        return {}
+    with gzip.open(file, "rb") as gzfile:
+        return json.loads(gzfile.read().decode("utf-8"))
+
+
 class RuleMetadata(FrozenModel):
     name: str
     namespace: Optional[str] = None
+    prevalence: str = "unknown"
     authors: Tuple[str, ...]
     scope: capa.rules.Scope
     attack: Tuple[AttackSpec, ...] = Field(alias="att&ck")
@@ -521,6 +536,7 @@ def from_capa(cls, rule: capa.rules.Rule) -> "RuleMetadata":
         return cls(
             name=rule.meta.get("name"),
             namespace=rule.meta.get("namespace"),
+            prevalence=load_rules_prevalence().get(rule.meta.get("name"), "unknown"),
             authors=rule.meta.get("authors"),
             scope=capa.rules.Scope(rule.meta.get("scope")),
             attack=tuple(map(AttackSpec.from_str, rule.meta.get("att&ck", []))),

capa/render/utils.py

@@ -5,7 +5,6 @@
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-
 import io
 from typing import Union, Iterator
 

capa/render/verbose.py

@@ -139,6 +139,9 @@ def render_rules(ostream, doc: rd.ResultDocument):
 
             rows.append((key, v))
 
+        prevalence = rutils.bold(rule.meta.prevalence) if rule.meta.prevalence != "unknown" else "unknown"
+        rows.insert(1, ("prevalence", prevalence))
+
         if rule.meta.scope != capa.rules.FILE_SCOPE:
             locations = [m[0] for m in doc.rules[rule.meta.name].matches]
             rows.append(("matches", "\n".join(map(format_address, locations))))

capa/render/vverbose.py

@@ -305,6 +305,9 @@ def render_rules(ostream, doc: rd.ResultDocument):
             # library rules should not have a namespace
             rows.append(("namespace", rule.meta.namespace))
 
+        prevalence = rutils.bold(rule.meta.prevalence) if rule.meta.prevalence != "unknown" else "unknown"
+        rows.append(("prevalence", prevalence))
+
         if rule.meta.maec.analysis_conclusion or rule.meta.maec.analysis_conclusion_ov:
             rows.append(
                 (

capa/rules/__init__.py

@@ -16,18 +16,9 @@
 import binascii
 import collections
 from enum import Enum
-from pathlib import Path
-
-from capa.helpers import assert_never
-
-try:
-    from functools import lru_cache
-except ImportError:
-    # need to type ignore this due to mypy bug here (duplicate name):
-    # https://github.com/python/mypy/issues/1153
-    from backports.functools_lru_cache import lru_cache  # type: ignore
-
 from typing import Any, Set, Dict, List, Tuple, Union, Iterator, Optional
+from pathlib import Path
+from functools import lru_cache
 
 import yaml
 import pydantic
@@ -43,6 +34,7 @@
 import capa.features.common
 import capa.features.basicblock
 from capa.engine import Statement, FeatureSet
+from capa.helpers import assert_never
 from capa.features.common import MAX_BYTES_FEATURE_SIZE, Feature
 from capa.features.address import Address