feat(css_tag_replace): new editor (#211)

EdJoPaTo · web-flow · commit 23a31b5c09f1 · 2025-04-10T09:50:17.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Generate the JSON Schema for the `website-stalker.yaml` with `website-stalker json-schema`.
+- new editor: `css_tag_replace`
 - Support zstd response body decompression
 
 ## [0.25.1] - 2025-01-30
diff --git a/README.md b/README.md
@@ -347,6 +347,57 @@ editors:
       reverse: true
 ```
 
+#### `css_tag_replace`
+
+Replace [HTML tags](https://developer.mozilla.org/en-US/docs/Glossary/Tag) matching a given [CSS Selector](https://developer.mozilla.org/en-US/docs/Learn/CSS/Building_blocks/Selectors).
+
+For example, the following config will replace all `h3` tags with `h2` tags.
+
+```yaml
+editors:
+  - css_tag_replace:
+      selector: h3
+      replace: h2
+```
+
+```diff
+ <html>
+ <head></head>
+ <body>
+-  <h3 class="green">
++  <h2 class="green">
+     Hello
+-  </h3>
++  </h2>
+   World
+ </body>
+ </html>
+```
+
+This can be helpful to ensure some kind of structure especially when editors like [`html_markdownify`](#html_markdownify) are used.
+Think about a website where only some subsections are of interest and selected via the [`css_select`](#css_select).
+While the `header` contains some `h1` the selected part skips the `h2` headings and continues with `h3` headings.
+Also, `<strong>` are (incorrectly) used as subheadings.
+Parsing this to Markdown results in less optimal structure as `h2` are skipped and `<strong>` don't result in headings.
+Ideal would be a single `h1` and then continous depending on the depth `h2`, `h3` and so on.
+The following can help with that:
+
+```yaml
+editors:
+  # Select the header and some interesting sections
+  - css_select: header, main section.interesting
+  # First migrate the h3 tags to h2 so there is no gap
+  - css_tag_replace:
+      selector: h3
+      replace: h2
+  # Then migrate all strong tags to proper headings
+  - css_tag_replace:
+      selector: strong
+      replace: h3
+  # In the end parse to Markdown
+  - html_markdownify
+```
+
 #### `debug_files`
 
 This editor passes its input through without modifying it.
diff --git a/src/editor/css_tag_replace.rs b/src/editor/css_tag_replace.rs
@@ -0,0 +1,70 @@
+use html5ever::{namespace_url, ns, LocalName, QualName};
+use schemars::JsonSchema;
+use scraper::{Html, Node, Selector};
+use serde::Deserialize;
+
+#[derive(Debug, Clone, Deserialize, JsonSchema)]
+#[serde(deny_unknown_fields)]
+pub struct CssTagReplace {
+    #[schemars(with = "String")]
+    pub selector: Selector,
+
+    #[schemars(with = "String")]
+    pub replace: LocalName,
+}
+
+impl CssTagReplace {
+    pub fn apply(&self, html: &str) -> anyhow::Result<String> {
+        let mut html = Html::parse_document(html);
+        let selected = html
+            .select(&self.selector)
+            .map(|element| element.id())
+            .collect::<Vec<_>>();
+        anyhow::ensure!(!selected.is_empty(), "selected nothing");
+        for node_id in selected {
+            let mut node = html
+                .tree
+                .get_mut(node_id)
+                .expect("Element ID should exist as it was just taken from the given HTML");
+            let Node::Element(element) = node.value() else {
+                unreachable!("Select only selects elements");
+            };
+            element.name = QualName::new(None, ns!(html), self.replace.clone());
+        }
+
+        Ok(html.html())
+    }
+}
+
+#[cfg(test)]
+#[track_caller]
+fn case<TAG: Into<LocalName>>(selectors: &str, replace: TAG, html: &str, expected: &str) {
+    let result = CssTagReplace {
+        selector: Selector::parse(selectors).unwrap(),
+        replace: replace.into(),
+    }
+    .apply(html)
+    .expect("Should select something");
+    assert_eq!(result, expected);
+}
+
+#[test]
+fn only_tag() {
+    let html = "<body><h1>Hello</h1>World<h3>Foo</h3>Bar</body>";
+    let expected = "<html><head></head><body><h1>Hello</h1>World<h2>Foo</h2>Bar</body></html>";
+    case("h3", "h2", html, expected);
+}
+
+#[test]
+fn keeps_attributes() {
+    let html = r#"<body><h2 class="green">Hello</h2>World</body>"#;
+    let expected = r#"<html><head></head><body><h1 class="green">Hello</h1>World</body></html>"#;
+    case("h2", "h1", html, expected);
+}
+
+#[test]
+fn more_specific_selector() {
+    let html = "<body><header><div>Headline</div></header><main><div>Something</div></main></body>";
+    let expected = "<html><head></head><body><header><div>Headline</div></header><main><p>Something</p></main></body></html>";
+    case("main div", "p", html, expected);
+}
diff --git a/src/editor/mod.rs b/src/editor/mod.rs
@@ -9,6 +9,7 @@ pub mod css_flatten;
 pub mod css_remove;
 pub mod css_selector;
 pub mod css_sort;
+pub mod css_tag_replace;
 pub mod debug_files;
 pub mod html_markdown;
 pub mod html_pretty;
@@ -34,6 +35,7 @@ pub enum Editor {
     CssRemove(#[schemars(with = "String")] scraper::Selector),
     CssSelect(#[schemars(with = "String")] scraper::Selector),
     CssSort(css_sort::CssSort),
+    CssTagReplace(css_tag_replace::CssTagReplace),
     DebugFiles(PathBuf),
     HtmlMarkdownify,
     HtmlPrettify,
@@ -52,6 +54,7 @@ impl Editor {
             Self::CssRemove(_) => "css_remove",
             Self::CssSelect(_) => "css_select",
             Self::CssSort(_) => "css_sort",
+            Self::CssTagReplace(_) => "css_tag_replace",
             Self::DebugFiles(_) => "debug_files",
             Self::HtmlMarkdownify => "html_markdownify",
             Self::HtmlPrettify => "html_prettify",
@@ -82,6 +85,10 @@ impl Editor {
                 extension: Some("html"),
                 text: sort.apply(url, &input.text),
             }),
+            Self::CssTagReplace(replace) => Ok(Content {
+                extension: Some("html"),
+                text: replace.apply(&input.text)?,
+            }),
             Self::DebugFiles(path) => debug_files::debug_files(path, input),
             Self::HtmlMarkdownify => Ok(Content {
                 extension: Some("md"),
