templates: support string patterns in template language

CHANGELOG.md

@@ -109,6 +109,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
   output using template expressions, similar to `jj op log`. Also added
   `--no-op-diff` flag to suppress the operation diff.
 
+* A nearly identical string pattern system as revsets is now supported in the
+  template language, and is exposed as `string.match(pattern)`.
+
 ### Fixed bugs
 
 * `jj git clone` now correctly fetches all tags, unless `--fetch-tags` is

cli/src/commit_templater.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//! Template environment for `jj log`, `jj evolog` and similar.
+
 use std::any::Any;
 use std::cmp::Ordering;
 use std::cmp::max;
@@ -116,6 +118,7 @@ pub trait CommitTemplateLanguageExtension {
     fn build_cache_extensions(&self, extensions: &mut ExtensionsMap);
 }
 
+/// Template environment for `jj log` and `jj evolog`.
 pub struct CommitTemplateLanguage<'repo> {
     repo: &'repo dyn Repo,
     path_converter: &'repo RepoPathUiConverter,

cli/src/operation_templater.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//! Template environment for `jj op log`.
+
 use std::any::Any;
 use std::cmp::Ordering;
 use std::collections::HashMap;
@@ -58,6 +60,7 @@ pub trait OperationTemplateEnvironment {
     fn current_op_id(&self) -> Option<&OperationId>;
 }
 
+/// Template environment for `jj op log`.
 pub struct OperationTemplateLanguage {
     repo_loader: RepoLoader,
     current_op_id: Option<OperationId>,

cli/src/template.pest

@@ -33,6 +33,8 @@ string_literal = ${ "\"" ~ (string_content | string_escape)* ~ "\"" }
 raw_string_content = @{ (!"'" ~ ANY)* }
 raw_string_literal = ${ "'" ~ raw_string_content ~ "'" }
 
+any_string_literal = _{ string_literal | raw_string_literal }
+
 integer_literal = @{
   ASCII_NONZERO_DIGIT ~ ASCII_DIGIT*
   | "0"
@@ -56,6 +58,7 @@ div_op = { "/" }
 rem_op = { "%" }
 logical_not_op = { "!" }
 negate_op = { "-" }
+pattern_kind_op = { ":" }
 prefix_ops = _{ logical_not_op | negate_op }
 infix_ops = _{
   logical_or_op
@@ -88,10 +91,21 @@ formal_parameters = {
   | ""
 }
 
+// NOTE: string pattern identifiers additionally allow "-" in them, which
+// results in some oddness with the `-` operator, though does not yet cause
+// ambiguity. This may prove annoying at some future point.
+string_pattern_identifier = @{ (ASCII_ALPHA | "_") ~ (ASCII_ALPHANUMERIC | "_" | "-")* }
+string_pattern = ${
+    // Unlike the revset language, we're not allowing bare words here because
+    // templates are generally not written on-the-fly.
+    string_pattern_identifier ~ pattern_kind_op ~ any_string_literal
+}
+
 primary = _{
   ("(" ~ template ~ ")")
   | function
   | lambda
+  | string_pattern
   | identifier
   | string_literal
   | raw_string_literal

cli/src/template_builder.rs

@@ -70,7 +70,11 @@ use crate::templater::WrapTemplateProperty;
 use crate::text_util;
 use crate::time_util;
 
-/// Callbacks to build language-specific evaluation objects from AST nodes.
+/// Callbacks to build usage-context-specific evaluation objects from AST nodes.
+///
+/// This is used to implement different meanings of `self` or different
+/// globally available functions in the template language depending on the
+/// context in which it is invoked.
 pub trait TemplateLanguage<'a> {
     type Property: CoreTemplatePropertyVar<'a>;
 
@@ -87,6 +91,8 @@ pub trait TemplateLanguage<'a> {
         function: &FunctionCallNode,
     ) -> TemplateParseResult<Self::Property>;
 
+    /// Creates a method call thunk for the given `function` of the given
+    /// `property`.
     fn build_method(
         &self,
         diagnostics: &mut TemplateDiagnostics,
@@ -713,6 +719,7 @@ impl<'a, P: CoreTemplatePropertyVar<'a>> Expression<P> {
     }
 }
 
+/// Environment (locals and self) in a stack frame.
 pub struct BuildContext<'i, P> {
     /// Map of functions to create `L::Property`.
     local_variables: HashMap<&'i str, &'i dyn Fn() -> P>,
@@ -902,6 +909,25 @@ fn builtin_string_methods<'a, L: TemplateLanguage<'a> + ?Sized>()
             Ok(out_property.into_dyn_wrapped())
         },
     );
+    map.insert(
+        "match",
+        |_language, _diagnostics, _build_ctx, self_property, function| {
+            let [needle_node] = function.expect_exact_arguments()?;
+            let needle = template_parser::expect_string_pattern(needle_node)?;
+            let regex = needle.to_regex();
+
+            let out_property = self_property.and_then(move |haystack| {
+                if let Some(m) = regex.find(haystack.as_bytes()) {
+                    Ok(std::str::from_utf8(m.as_bytes())?.to_owned())
+                } else {
+                    // We don't have optional strings, so empty string is the
+                    // right null value.
+                    Ok(String::new())
+                }
+            });
+            Ok(out_property.into_dyn_wrapped())
+        },
+    );
     map.insert(
         "starts_with",
         |language, diagnostics, build_ctx, self_property, function| {
@@ -1939,6 +1965,10 @@ pub fn build_expression<'a, L: TemplateLanguage<'a> + ?Sized>(
             let property = Literal(value.clone()).into_dyn_wrapped();
             Ok(Expression::unlabeled(property))
         }
+        ExpressionKind::StringPattern { .. } => Err(TemplateParseError::expression(
+            "String patterns may not be used as expression values",
+            node.span,
+        )),
         ExpressionKind::Unary(op, arg_node) => {
             let property = build_unary_operation(language, diagnostics, build_ctx, *op, arg_node)?;
             Ok(Expression::unlabeled(property))
@@ -2962,13 +2992,49 @@ mod tests {
             env.render_ok(r#""description 123".contains(description.first_line())"#),
             @"true");
 
+        // String patterns are not stringifiable
+        insta::assert_snapshot!(env.parse_err(r#""fa".starts_with(regex:'[a-f]o+')"#), @r#"
+         --> 1:18
+          |
+        1 | "fa".starts_with(regex:'[a-f]o+')
+          |                  ^-------------^
+          |
+          = String patterns may not be used as expression values
+        "#);
+
         // inner template error should propagate
         insta::assert_snapshot!(env.render_ok(r#""foo".contains(bad_string)"#), @"<Error: Bad>");
         insta::assert_snapshot!(
             env.render_ok(r#""foo".contains("f" ++ bad_string) ++ "bar""#), @"<Error: Bad>bar");
         insta::assert_snapshot!(
             env.render_ok(r#""foo".contains(separate("o", "f", bad_string))"#), @"<Error: Bad>");
 
+        insta::assert_snapshot!(env.render_ok(r#""fooo".match(regex:'[a-f]o+')"#), @"fooo");
+        insta::assert_snapshot!(env.render_ok(r#""fa".match(regex:'[a-f]o+')"#), @"");
+        insta::assert_snapshot!(env.render_ok(r#""hello".match(regex:"h(ell)o")"#), @"hello");
+        insta::assert_snapshot!(env.render_ok(r#""HEllo".match(regex-i:"h(ell)o")"#), @"HEllo");
+        insta::assert_snapshot!(env.render_ok(r#""hEllo".match(glob:"h*o")"#), @"hEllo");
+        insta::assert_snapshot!(env.render_ok(r#""Hello".match(glob:"h*o")"#), @"");
+        insta::assert_snapshot!(env.render_ok(r#""HEllo".match(glob-i:"h*o")"#), @"HEllo");
+        insta::assert_snapshot!(env.render_ok(r#""hello".match("he")"#), @"he");
+        insta::assert_snapshot!(env.render_ok(r#""hello".match(substring:"he")"#), @"he");
+        insta::assert_snapshot!(env.render_ok(r#""hello".match(exact:"he")"#), @"");
+
+        // Evil regexes can cause invalid UTF-8 output, which nothing can
+        // really be done about given we're matching against non-UTF-8 stuff a
+        // lot as well.
+        insta::assert_snapshot!(env.render_ok(r#""🥺".match(regex:'(?-u)^(?:.)')"#), @"<Error: incomplete utf-8 byte sequence from index 0>");
+
+        insta::assert_snapshot!(env.parse_err(r#""🥺".match(not-a-pattern:"abc")"#), @r#"
+         --> 1:11
+          |
+        1 | "🥺".match(not-a-pattern:"abc")
+          |           ^-----------------^
+          |
+          = Bad string pattern
+        Invalid string pattern kind `not-a-pattern:`
+        "#);
+
         insta::assert_snapshot!(env.render_ok(r#""".first_line()"#), @"");
         insta::assert_snapshot!(env.render_ok(r#""foo\nbar".first_line()"#), @"foo");
 

cli/src/template_parser.rs

@@ -33,6 +33,7 @@ use jj_lib::dsl_util::FunctionCallParser;
 use jj_lib::dsl_util::InvalidArguments;
 use jj_lib::dsl_util::StringLiteralParser;
 use jj_lib::dsl_util::collect_similar;
+use jj_lib::str_util::StringPattern;
 use pest::Parser as _;
 use pest::iterators::Pair;
 use pest::iterators::Pairs;
@@ -69,6 +70,7 @@ impl Rule {
             Self::string_literal => None,
             Self::raw_string_content => None,
             Self::raw_string_literal => None,
+            Self::any_string_literal => None,
             Self::integer_literal => None,
             Self::identifier => None,
             Self::concat_op => Some("++"),
@@ -87,6 +89,7 @@ impl Rule {
             Self::rem_op => Some("%"),
             Self::logical_not_op => Some("!"),
             Self::negate_op => Some("-"),
+            Self::pattern_kind_op => Some(":"),
             Self::prefix_ops => None,
             Self::infix_ops => None,
             Self::function => None,
@@ -95,6 +98,8 @@ impl Rule {
             Self::function_arguments => None,
             Self::lambda => None,
             Self::formal_parameters => None,
+            Self::string_pattern_identifier => None,
+            Self::string_pattern => None,
             Self::primary => None,
             Self::term => None,
             Self::expression => None,
@@ -285,6 +290,11 @@ pub enum ExpressionKind<'i> {
     Boolean(bool),
     Integer(i64),
     String(String),
+    /// `<kind>:"<value>"`
+    StringPattern {
+        kind: &'i str,
+        value: String,
+    },
     Unary(UnaryOp, Box<ExpressionNode<'i>>),
     Binary(BinaryOp, Box<ExpressionNode<'i>>, Box<ExpressionNode<'i>>),
     Concat(Vec<ExpressionNode<'i>>),
@@ -302,7 +312,10 @@ impl<'i> FoldableExpression<'i> for ExpressionKind<'i> {
     {
         match self {
             Self::Identifier(name) => folder.fold_identifier(name, span),
-            Self::Boolean(_) | Self::Integer(_) | Self::String(_) => Ok(self),
+            ExpressionKind::Boolean(_)
+            | ExpressionKind::Integer(_)
+            | ExpressionKind::String(_)
+            | ExpressionKind::StringPattern { .. } => Ok(self),
             Self::Unary(op, arg) => {
                 let arg = Box::new(folder.fold_expression(*arg)?);
                 Ok(Self::Unary(op, arg))
@@ -458,6 +471,12 @@ fn parse_lambda_node(pair: Pair<Rule>) -> TemplateParseResult<LambdaNode> {
     })
 }
 
+fn parse_raw_string_literal(pair: Pair<Rule>) -> String {
+    let [content] = pair.into_inner().collect_array().unwrap();
+    assert_eq!(content.as_rule(), Rule::raw_string_content);
+    content.as_str().to_owned()
+}
+
 fn parse_term_node(pair: Pair<Rule>) -> TemplateParseResult<ExpressionNode> {
     assert_eq!(pair.as_rule(), Rule::term);
     let mut inner = pair.into_inner();
@@ -469,9 +488,7 @@ fn parse_term_node(pair: Pair<Rule>) -> TemplateParseResult<ExpressionNode> {
             ExpressionNode::new(ExpressionKind::String(text), span)
         }
         Rule::raw_string_literal => {
-            let [content] = expr.into_inner().collect_array().unwrap();
-            assert_eq!(content.as_rule(), Rule::raw_string_content);
-            let text = content.as_str().to_owned();
+            let text = parse_raw_string_literal(expr);
             ExpressionNode::new(ExpressionKind::String(text), span)
         }
         Rule::integer_literal => {
@@ -480,6 +497,21 @@ fn parse_term_node(pair: Pair<Rule>) -> TemplateParseResult<ExpressionNode> {
             })?;
             ExpressionNode::new(ExpressionKind::Integer(value), span)
         }
+        Rule::string_pattern => {
+            let [kind, op, literal] = expr.into_inner().collect_array().unwrap();
+            assert_eq!(kind.as_rule(), Rule::string_pattern_identifier);
+            assert_eq!(op.as_rule(), Rule::pattern_kind_op);
+            let kind = kind.as_str();
+            let text = match literal.as_rule() {
+                Rule::string_literal => STRING_LITERAL_PARSER.parse(literal.into_inner()),
+                Rule::raw_string_literal => parse_raw_string_literal(literal),
+                other => {
+                    panic!("Unexpected literal rule in string pattern: {other:?}")
+                }
+            };
+            // The actual parsing and construction of the pattern is deferred to later.
+            ExpressionNode::new(ExpressionKind::StringPattern { kind, value: text }, span)
+        }
         Rule::identifier => ExpressionNode::new(parse_identifier_or_literal(expr), span),
         Rule::function => {
             let function = Box::new(FUNCTION_CALL_PARSER.parse(
@@ -663,6 +695,23 @@ pub fn expect_string_literal<'a>(node: &'a ExpressionNode<'_>) -> TemplateParseR
     })
 }
 
+/// Unwraps inner value if the given `node` is a string pattern
+///
+/// This forces it to be static so that it need not be part of the type system.
+pub fn expect_string_pattern(node: &ExpressionNode<'_>) -> TemplateParseResult<StringPattern> {
+    catch_aliases_no_diagnostics(node, |node| match &node.kind {
+        ExpressionKind::StringPattern { kind, value } => StringPattern::from_str_kind(value, kind)
+            .map_err(|err| {
+                TemplateParseError::expression("Bad string pattern", node.span).with_source(err)
+            }),
+        ExpressionKind::String(string) => Ok(StringPattern::Substring(string.clone())),
+        _ => Err(TemplateParseError::expression(
+            "Expected string pattern",
+            node.span,
+        )),
+    })
+}
+
 /// Unwraps inner node if the given `node` is a lambda.
 pub fn expect_lambda<'a, 'i>(
     node: &'a ExpressionNode<'i>,
@@ -835,6 +884,7 @@ mod tests {
             | ExpressionKind::Boolean(_)
             | ExpressionKind::Integer(_)
             | ExpressionKind::String(_) => node.kind,
+            ExpressionKind::StringPattern { .. } => node.kind,
             ExpressionKind::Unary(op, arg) => {
                 let arg = Box::new(normalize_tree(*arg));
                 ExpressionKind::Unary(op, arg)
@@ -1139,6 +1189,51 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_string_pattern() {
+        assert_eq!(
+            parse_into_kind(r#"regex:"meow""#),
+            Ok(ExpressionKind::StringPattern {
+                kind: "regex",
+                value: "meow".to_owned()
+            }),
+        );
+        assert_eq!(
+            parse_into_kind(r#"regex:'\r\n'"#),
+            Ok(ExpressionKind::StringPattern {
+                kind: "regex",
+                value: r#"\r\n"#.to_owned()
+            })
+        );
+        assert_eq!(
+            parse_into_kind(r#"regex-i:'\r\n'"#),
+            Ok(ExpressionKind::StringPattern {
+                kind: "regex-i",
+                value: r#"\r\n"#.to_owned()
+            })
+        );
+        assert_eq!(
+            parse_into_kind("regex:meow"),
+            Err(TemplateParseErrorKind::SyntaxError),
+            "no bare words in string patterns in templates"
+        );
+        assert_eq!(
+            parse_into_kind("regex: 'with spaces'"),
+            Err(TemplateParseErrorKind::SyntaxError),
+            "no spaces after"
+        );
+        assert_eq!(
+            parse_into_kind("regex :'with spaces'"),
+            Err(TemplateParseErrorKind::SyntaxError),
+            "no spaces before either"
+        );
+        assert_eq!(
+            parse_into_kind("regex : 'with spaces'"),
+            Err(TemplateParseErrorKind::SyntaxError),
+            "certainly not both"
+        );
+    }
+
     #[test]
     fn test_integer_literal() {
         assert_eq!(parse_into_kind("0"), Ok(ExpressionKind::Integer(0)));