diff --git a/Cargo.lock b/Cargo.lock index ade3b6b06..8f132822f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1196,6 +1196,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "dyn-clone" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" + [[package]] name = "ecow" version = "0.2.3" @@ -1759,6 +1765,12 @@ dependencies = [ "serde", ] +[[package]] +name = "hifijson" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9958ab3ce3170c061a27679916bd9b969eceeb5e8b120438e6751d0987655c42" + [[package]] name = "hmac" version = "0.12.1" @@ -2301,6 +2313,47 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +[[package]] +name = "jaq-core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d03ee1b714d63f0a5820131180056b592ecd400b32879c848e53e58707f19df0" +dependencies = [ + "dyn-clone", + "once_cell", + "typed-arena", +] + +[[package]] +name = "jaq-json" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8daf2b52304419d7bf5ec32891884c65274a3eedc0b5834b84627099901a1176" +dependencies = [ + "foldhash", + "hifijson", + "indexmap 2.6.0", + "jaq-core", + "jaq-std", + "serde_json", +] + +[[package]] +name = "jaq-std" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80e2c65cceafd4c0019f15a0dac7c0dd659b0fcf5182fc3a10d15b89d89ac6e8" +dependencies = [ + "aho-corasick", + "base64 0.22.1", + "chrono", + "jaq-core", + "libm", + "log", + "regex-lite", + "urlencoding", +] + [[package]] name = "jemalloc-sys" version = "0.3.2" @@ -3499,6 +3552,12 @@ dependencies = [ "regex-syntax 0.8.5", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -3819,6 +3878,9 @@ dependencies = [ "indicatif", "insta", "itertools", + "jaq-core", + "jaq-json", + "jaq-std", "jemallocator-global", "libc", "log", @@ -5078,6 +5140,12 @@ dependencies = [ "utf-8", ] +[[package]] +name = "typed-arena" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" + [[package]] name = "typed-path" version = "0.9.3" diff --git a/Cargo.toml b/Cargo.toml index 1057faa80..e2f1a0dfc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ rustic - fast, encrypted, deduplicated backups powered by Rust """ [features] -default = ["tui", "webdav"] +default = ["tui", "webdav", "rhai", "jq"] release = ["default", "self-update"] # Allocators @@ -37,6 +37,10 @@ webdav = [ ] mount = ["dep:fuse_mt"] +# Filtering +rhai = ["dep:rhai"] +jq = ["dep:jaq-core", "dep:jaq-std", "dep:jaq-json"] + [[bin]] name = "rustic" path = "src/bin/rustic.rs" @@ -87,7 +91,6 @@ serde_with = { version = "3", features = ["base64"] } aho-corasick = "1" chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } comfy-table = "7" -rhai = { version = "1", features = ["sync", "serde", "no_optimize", "no_module", "no_custom_syntax", "only_i64"] } scopeguard = "1" semver = { version = "1", optional = true } simplelog = "0.12" @@ -117,6 +120,12 @@ self_update = { version = "=0.39.0", default-features = false, optional = true, tar = "0.4.43" toml = "0.8" +# filtering +jaq-core = { version = "2", optional = true } +jaq-json = { version = "1", features = ["serde_json"], optional = true } +jaq-std = { version = "2", optional = true } +rhai = { version = "1", features = ["sync", "serde", "no_optimize", "no_module", "no_custom_syntax", "only_i64"], optional = true } + [dev-dependencies] abscissa_core = { version = "0.8.1", default-features = false, features = ["testing"] } assert_cmd = "2.0.16" diff --git a/config/README.md b/config/README.md index 2deef5fb1..9d94e7c3b 100644 --- a/config/README.md +++ b/config/README.md @@ -159,21 +159,22 @@ See [Global Hooks](#global-hooks-globalhooks). ### Snapshot-Filter Options `[snapshot-filter]` -| Attribute | Description | Default Value | Example Value | CLI Option | -| ------------------ | ---------------------------------------------------------------------- | ------------- | ------------------------ | -------------------- | -| filter-hosts | Array of hosts to filter snapshots. | Not set | ["myhost", "host2"] | --filter-host | -| filter-labels | Array of labels to filter snapshots. | Not set | ["mylabal"] | --filter-label | -| filter-paths | Array of pathlists to filter snapshots. | Not set | ["/home,/root"] | --filter-paths | -| filter-paths-exact | Array or string of paths to filter snapshots. Exact match. | Not set | ["path1,path2", "path3"] | --filter-paths-exact | -| filter-tags | Array of taglists to filter snapshots. | Not set | ["tag1,tag2"] | --filter-tags | -| filter-tags-exact | Array or string of tags to filter snapshots. Exact match. | Not set | ["tag1,tag2", "tag3"] | --filter-tags-exact | -| filter-before | Filter snapshots before the given date/time | Not set | "2024-01-01" | --filter-before | -| filter-after | Filter snapshots after the given date/time | Not set | "2023-01-01 11:15:23" | --filter-after | -| filter-size | Filter snapshots for a total size in the size range. | Not set | "1MB..1GB" | --filter-size | -| | If a single value is given, this is taken as lower bound. | | "500 k" | | -| filter-size-added | Filter snapshots for a size added to the repository in the size range. | Not set | "1MB..1GB" | --filter-size-added | -| | If a single value is given, this is taken as lower bound. | | "500 k" | | -| filter-fn | Custom filter function for snapshots. | Not set | | --filter-fn | +| Attribute | Description | Default Value | Example Value | CLI Option | +| ------------------ | ---------------------------------------------------------------------- | ------------- | -------------------------- | -------------------- | +| filter-hosts | Array of hosts to filter snapshots. | Not set | ["myhost", "host2"] | --filter-host | +| filter-labels | Array of labels to filter snapshots. | Not set | ["mylabal"] | --filter-label | +| filter-paths | Array of pathlists to filter snapshots. | Not set | ["/home,/root"] | --filter-paths | +| filter-paths-exact | Array or string of paths to filter snapshots. Exact match. | Not set | ["path1,path2", "path3"] | --filter-paths-exact | +| filter-tags | Array of taglists to filter snapshots. | Not set | ["tag1,tag2"] | --filter-tags | +| filter-tags-exact | Array or string of tags to filter snapshots. Exact match. | Not set | ["tag1,tag2", "tag3"] | --filter-tags-exact | +| filter-before | Filter snapshots before the given date/time | Not set | "2024-01-01" | --filter-before | +| filter-after | Filter snapshots after the given date/time | Not set | "2023-01-01 11:15:23" | --filter-after | +| filter-size | Filter snapshots for a total size in the size range. | Not set | "1MB..1GB" | --filter-size | +| | If a single value is given, this is taken as lower bound. | | "500 k" | | +| filter-size-added | Filter snapshots for a size added to the repository in the size range. | Not set | "1MB..1GB" | --filter-size-added | +| | If a single value is given, this is taken as lower bound. | | "500 k" | | +| filter-fn | Custom filter function for snapshots. | Not set | | --filter-fn | +| filter-jq | Custom filter jq function for snapshots. Should return bool | Not set | ".summary.files_added > 1" | --filter-jq | ### Backup Options `[backup]` diff --git a/config/full.toml b/config/full.toml index 4d3636472..bc93bb083 100644 --- a/config/full.toml +++ b/config/full.toml @@ -95,6 +95,7 @@ filter-before = "2024-02-05 12:15" # Default: not set filter-size = "200MiB" # Default: not set filter-size-added = "1 MB..10MB" # Default: not set filter-fn = '|sn| {sn.host == "host1" || sn.description.contains("test")}' # Default: no filter function +filter-jq = '.description | contains ("test")' # Default: no jq filter function # Backup options: These options are used for all sources when calling the backup command. # They can be overwritten by source-specific options (see below) or command line options. @@ -174,6 +175,7 @@ filter-before = "2024-02-05 12:15" # Default: not set filter-size = "200MiB" # Default: not set filter-size-added = "1 MB..10MB" # Default: not set filter-fn = '|sn| {sn.host == "host1" || sn.description.contains("test")}' # Default: no filter function +filter-jq = '.description | contains ("test")' # Default: no jq filter function # The retention options follow. All of these are not set by default. keep-tags = ["tag1", "tag2,tag3"] # Default: not set keep-ids = [ diff --git a/src/error.rs b/src/error.rs index 06a6d5261..bef628fc5 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,6 +1,7 @@ //! Error types use abscissa_core::error::{BoxError, Context}; +#[cfg(feature = "rhai")] use rhai::EvalAltResult; use std::{ fmt::{self, Display}, @@ -18,6 +19,7 @@ pub(crate) enum ErrorKind { } /// Kinds of [`rhai`] errors +#[cfg(feature = "rhai")] #[derive(Debug, Error)] pub(crate) enum RhaiErrorKinds { #[error(transparent)] diff --git a/src/filtering.rs b/src/filtering.rs index 93327c0ed..eda6414a7 100644 --- a/src/filtering.rs +++ b/src/filtering.rs @@ -1,28 +1,46 @@ +#[cfg(feature = "rhai")] use crate::error::RhaiErrorKinds; -use bytesize::ByteSize; -use derive_more::derive::Display; -use log::warn; -use rustic_core::{repofile::SnapshotFile, StringList}; +#[cfg(feature = "rhai")] +use std::error::Error; use std::{ - error::Error, fmt::{Debug, Display}, str::FromStr, }; +#[cfg(feature = "jq")] +use anyhow::{anyhow, bail}; +use bytesize::ByteSize; +use derive_more::derive::Display; +use log::warn; +use rustic_core::{repofile::SnapshotFile, StringList}; + use cached::proc_macro::cached; use chrono::{DateTime, Local, NaiveTime}; use conflate::Merge; + +#[cfg(feature = "jq")] +use jaq_core::{ + load::{Arena, File, Loader}, + Compiler, Ctx, Filter, Native, RcIter, +}; +#[cfg(feature = "jq")] +use jaq_json::Val; +#[cfg(feature = "rhai")] use rhai::{serde::to_dynamic, Dynamic, Engine, FnPtr, AST}; use serde::{Deserialize, Serialize}; +#[cfg(feature = "jq")] +use serde_json::Value; use serde_with::{serde_as, DisplayFromStr}; /// A function to filter snapshots /// /// The function is called with a [`SnapshotFile`] and must return a boolean. +#[cfg(feature = "rhai")] #[derive(Clone, Debug)] pub(crate) struct SnapshotFn(FnPtr, AST); +#[cfg(feature = "rhai")] impl FromStr for SnapshotFn { type Err = RhaiErrorKinds; fn from_str(s: &str) -> Result { @@ -33,17 +51,7 @@ impl FromStr for SnapshotFn { } } -#[cached(key = "String", convert = r#"{ s.to_string() }"#, size = 1)] -fn string_to_fn(s: &str) -> Option { - match SnapshotFn::from_str(s) { - Ok(filter_fn) => Some(filter_fn), - Err(err) => { - warn!("Error evaluating filter-fn {s}: {err}",); - None - } - } -} - +#[cfg(feature = "rhai")] impl SnapshotFn { /// Call the function with a [`SnapshotFile`] /// @@ -62,6 +70,75 @@ impl SnapshotFn { } } +#[cfg(feature = "rhai")] +#[cached(key = "String", convert = r#"{ s.to_string() }"#, size = 1)] +fn string_to_fn(s: &str) -> Option { + match SnapshotFn::from_str(s) { + Ok(filter_fn) => Some(filter_fn), + Err(err) => { + warn!("Error evaluating filter-fn {s}: {err}",); + None + } + } +} + +#[cfg(feature = "jq")] +#[derive(Clone)] +pub(crate) struct SnapshotJq(Filter>); + +#[cfg(feature = "jq")] +impl FromStr for SnapshotJq { + type Err = anyhow::Error; + fn from_str(s: &str) -> Result { + let programm = File { code: s, path: () }; + let loader = Loader::new(jaq_std::defs().chain(jaq_json::defs())); + let arena = Arena::default(); + let modules = loader + .load(&arena, programm) + .map_err(|errs| anyhow!("errors loading modules in jq: {errs:?}"))?; + let filter = Compiler::<_, Native<_>>::default() + .with_funs(jaq_std::funs().chain(jaq_json::funs())) + .compile(modules) + .map_err(|errs| anyhow!("errors during compiling filters in jq: {errs:?}"))?; + + Ok(Self(filter)) + } +} + +#[cfg(feature = "jq")] +impl SnapshotJq { + fn call(&self, snap: &SnapshotFile) -> Result { + let input = serde_json::to_value(snap)?; + + let inputs = RcIter::new(core::iter::empty()); + let res = self.0.run((Ctx::new([], &inputs), Val::from(input))).next(); + + match res { + Some(Ok(val)) => { + let val: Value = val.into(); + match val.as_bool() { + Some(true) => Ok(true), + Some(false) => Ok(false), + None => bail!("expression does not return bool"), + } + } + _ => bail!("expression does not return bool"), + } + } +} + +#[cfg(feature = "jq")] +#[cached(key = "String", convert = r#"{ s.to_string() }"#, size = 1)] +fn string_to_jq(s: &str) -> Option { + match SnapshotJq::from_str(s) { + Ok(filter_jq) => Some(filter_jq), + Err(err) => { + warn!("Error evaluating filter-fn {s}: {err}",); + None + } + } +} + #[serde_as] #[derive(Clone, Default, Debug, Serialize, Deserialize, Merge, clap::Parser)] #[serde(default, rename_all = "kebab-case", deny_unknown_fields)] @@ -125,10 +202,18 @@ pub struct SnapshotFilter { filter_size_added: Option, /// Function to filter snapshots + #[cfg(feature = "rhai")] #[clap(long, global = true, value_name = "FUNC")] #[serde_as(as = "Option")] #[merge(strategy=conflate::option::overwrite_none)] filter_fn: Option, + + /// jq to filter snapshots + #[cfg(feature = "jq")] + #[clap(long, global = true, value_name = "JQ")] + #[serde_as(as = "Option")] + #[merge(strategy=conflate::option::overwrite_none)] + filter_jq: Option, } impl SnapshotFilter { @@ -143,6 +228,7 @@ impl SnapshotFilter { /// `true` if the snapshot matches the filter, `false` otherwise #[must_use] pub fn matches(&self, snapshot: &SnapshotFile) -> bool { + #[cfg(feature = "rhai")] if let Some(filter_fn) = &self.filter_fn { if let Some(func) = string_to_fn(filter_fn) { match func.call::(snapshot) { @@ -156,6 +242,26 @@ impl SnapshotFilter { "Error evaluating filter-fn for snapshot {}: {err}", snapshot.id ); + return false; + } + } + } + } + #[cfg(feature = "jq")] + if let Some(filter_jq) = &self.filter_jq { + if let Some(jq) = string_to_jq(filter_jq) { + match jq.call(snapshot) { + Ok(result) => { + if !result { + return false; + } + } + Err(err) => { + warn!( + "Error evaluating filter-jq for snapshot {}: {err}", + snapshot.id + ); + return false; } } } diff --git a/src/snapshots/rustic_rs__config__tests__default_config_passes.snap b/src/snapshots/rustic_rs__config__tests__default_config_passes.snap index bb66646be..07fc79059 100644 --- a/src/snapshots/rustic_rs__config__tests__default_config_passes.snap +++ b/src/snapshots/rustic_rs__config__tests__default_config_passes.snap @@ -61,6 +61,7 @@ RusticConfig { filter_size: None, filter_size_added: None, filter_fn: None, + filter_jq: None, }, backup: BackupCmd { cli_sources: [], @@ -161,6 +162,7 @@ RusticConfig { filter_size: None, filter_size_added: None, filter_fn: None, + filter_jq: None, }, keep: KeepOptions { keep_tags: [], diff --git a/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap b/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap index d21701c41..e1f40db77 100644 --- a/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap +++ b/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap @@ -72,6 +72,7 @@ RusticConfig { filter_size: None, filter_size_added: None, filter_fn: None, + filter_jq: None, }, backup: BackupCmd { cli_sources: [], @@ -172,6 +173,7 @@ RusticConfig { filter_size: None, filter_size_added: None, filter_fn: None, + filter_jq: None, }, keep: KeepOptions { keep_tags: [],