Merge pull request #273 from SigmaHQ:fix-sigmastring

Fix ReplaceStringTransformation and SigmaString plain string conversion

sigma/processing/transformations.py

@@ -762,13 +762,17 @@ def apply(
 class ReplaceStringTransformation(StringValueTransformation):
     """
     Replace string part matched by regular expresssion with replacement string that can reference
-    capture groups. It operates on the plain string parts of the SigmaString value.
+    capture groups. Normally, the replacement operates on the plain string representation of the
+    SigmaString. This allows also to include special characters and placeholders in the replacement.
+    By enabling the skip_special parameter, the replacement is only applied to the plain string
+    parts of a SigmaString and special characters and placeholders are left untouched.
 
-    This is basically an interface to re.sub() and can use all features available there.
+    The replacement is implemented with re.sub() and can use all features available there.
     """
 
     regex: str
     replacement: str
+    skip_special: bool = False
 
     def __post_init__(self):
         super().__post_init__()
@@ -781,9 +785,15 @@ def __post_init__(self):
 
     def apply_string_value(self, field: str, val: SigmaString) -> SigmaString:
         if isinstance(val, SigmaString):
-            return val.map_parts(
-                lambda s: self.re.sub(self.replacement, s), lambda p: isinstance(p, str)
-            )
+            if self.skip_special:
+                return val.map_parts(
+                    lambda s: self.re.sub(self.replacement, s), lambda p: isinstance(p, str)
+                )
+            else:
+                sigma_string_plain = str(val)
+                replaced = self.re.sub(self.replacement, sigma_string_plain)
+                postprocessed_backslashes = re.sub(r"\\(?![*?])", r"\\\\", replaced)
+                return SigmaString(postprocessed_backslashes)
 
 
 @dataclass

sigma/rule.py

@@ -436,9 +436,22 @@ def to_plain(self) -> Union[Dict[str, Union[str, int, None]], List[str]]:
             )
 
         if len(self.original_value) > 1:
-            value = [value.to_plain() for value in self.original_value]
+            value = [
+                (
+                    value.to_plain(True)
+                    if isinstance(value, SigmaString)
+                    and SigmaRegularExpressionModifier in self.modifiers
+                    else value.to_plain()
+                )
+                for value in self.original_value
+            ]
         else:
-            value = self.original_value[0].to_plain()
+            value = (
+                self.original_value[0].to_plain(True)
+                if isinstance(self.original_value[0], SigmaString)
+                and SigmaRegularExpressionModifier in self.modifiers
+                else self.original_value[0].to_plain()
+            )
 
         if (
             self.is_keyword() and len(self.modifiers) == 0

sigma/types.py

@@ -355,10 +355,17 @@ def __eq__(self, other: Union["SigmaString", str]) -> bool:
             )
 
     def __str__(self) -> str:
+        return self.to_plain()
+
+    def to_plain(self, regex: bool = False) -> str:
+        """Generate string representation of SigmaString with or without regex escaping."""
         rs = ""
         for s in self.s:
             if isinstance(s, str):
-                rs += s
+                if regex:
+                    rs += s
+                else:
+                    rs += s.replace("*", "\\*").replace("?", "\\?")
             elif isinstance(s, SpecialChars):
                 rs += special_char_mapping[s]
             elif isinstance(s, Placeholder):
@@ -372,9 +379,9 @@ def __str__(self) -> str:
     def __repr__(self) -> str:
         return str(f"SigmaString({self.s})")
 
-    def to_plain(self):
-        """Return plain string representation of SigmaString, equivalent to converting it with str()."""
-        return str(self)
+    def to_plain_regex(self):
+        """Return plain string representation of SigmaString with reduced escaping."""
+        return self._stringify(True)
 
     def __bytes__(self) -> bytes:
         return str(self).encode()

tests/test_processing_transformations.py

@@ -1344,6 +1344,36 @@ def test_replace_string_specials(dummy_pipeline):
     )
     transformation = ReplaceStringTransformation("^.*\\\\", "/")
     transformation.apply(dummy_pipeline, sigma_rule)
+    assert sigma_rule.detection.detections["test"] == SigmaDetection(
+        [
+            SigmaDetection(
+                [
+                    SigmaDetectionItem("field1", [], [SigmaString("/value")]),
+                    SigmaDetectionItem("field2", [], [SigmaNumber(123)]),
+                ]
+            )
+        ]
+    )
+
+
+def test_replace_string_skip_specials(dummy_pipeline):
+    sigma_rule = SigmaRule.from_dict(
+        {
+            "title": "Test",
+            "logsource": {"category": "test"},
+            "detection": {
+                "test": [
+                    {
+                        "field1": "*\\value",
+                        "field2": 123,
+                    }
+                ],
+                "condition": "test",
+            },
+        }
+    )
+    transformation = ReplaceStringTransformation("^.*\\\\", "/", True)
+    transformation.apply(dummy_pipeline, sigma_rule)
     assert sigma_rule.detection.detections["test"] == SigmaDetection(
         [
             SigmaDetection(
@@ -1356,6 +1386,38 @@ def test_replace_string_specials(dummy_pipeline):
     )
 
 
+def test_replace_string_backslashes(dummy_pipeline):
+    sigma_rule = SigmaRule.from_dict(
+        {
+            "title": "Test",
+            "logsource": {"category": "test"},
+            "detection": {
+                "test": [
+                    {
+                        "field1": r"backslash\\value",
+                        "field2": r"backslash\\\\value",
+                        "field3": r"plainwildcard\*value",
+                    }
+                ],
+                "condition": "test",
+            },
+        }
+    )
+    transformation = ReplaceStringTransformation("value", "test")
+    transformation.apply(dummy_pipeline, sigma_rule)
+    assert sigma_rule.detection.detections["test"] == SigmaDetection(
+        [
+            SigmaDetection(
+                [
+                    SigmaDetectionItem("field1", [], [SigmaString(r"backslash\\test")]),
+                    SigmaDetectionItem("field2", [], [SigmaString(r"backslash\\\\test")]),
+                    SigmaDetectionItem("field3", [], [SigmaString(r"plainwildcard\*test")]),
+                ]
+            )
+        ]
+    )
+
+
 def test_replace_string_invalid():
     with pytest.raises(SigmaRegularExpressionError, match="Regular expression.*invalid"):
         ReplaceStringTransformation("*", "test")

tests/test_types.py

@@ -320,6 +320,20 @@ def test_strings_to_string():
     assert str(SigmaString("test*?")) == "test*?"
 
 
+def test_strings_with_plain_wildcards_to_string():
+    plain_s = "value\*with\?plain*wild?cards"
+    s = SigmaString(plain_s)
+    assert s.s == (
+        "value*with?plain",
+        SpecialChars.WILDCARD_MULTI,
+        "wild",
+        SpecialChars.WILDCARD_SINGLE,
+        "cards",
+    )
+    assert str(s) == plain_s
+    assert SigmaString(str(s)) == s
+
+
 def test_strings_with_placeholder_to_string():
     assert str(SigmaString("te%var%st").insert_placeholders()) == "te%var%st"