diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.yml index 432fee3..171c13b 100644 --- a/.github/workflows/ci-rust.yml +++ b/.github/workflows/ci-rust.yml @@ -35,6 +35,7 @@ jobs: uses: Swatinem/rust-cache@v2 with: workspaces: . -> target + key: default-features save-if: ${{ github.ref == 'refs/heads/develop' }} - name: Build Rust run: cargo check --verbose --locked @@ -43,6 +44,32 @@ jobs: - name: Check dirty git uses: ./.github/actions/check-dirty-git + # builds and tests rust code, with --no-default-features, with --deny warnings. tests for dirty git + build-and-test-rust-no-default-features: + runs-on: ubuntu-latest + + env: + RUSTFLAGS: --deny warnings + + steps: + - uses: actions/checkout@v4 + - name: Rustup update + run: rustup update + - name: Show cargo version + run: cargo --version + - name: rust build caching + uses: Swatinem/rust-cache@v2 + with: + workspaces: . -> target + key: no-default-features + save-if: ${{ github.ref == 'refs/heads/develop' }} + - name: Build Rust + run: cargo check --no-default-features --verbose --locked + - name: Test Rust + run: cargo test --no-default-features --verbose --locked + - name: Check dirty git + uses: ./.github/actions/check-dirty-git + # lints and checks formatting of rust code lint-rust: runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 2f8f2de..acbe17b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -57,6 +57,27 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" +[[package]] +name = "atomic" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "bytemuck" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae" + +[[package]] +name = "bytes" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" + [[package]] name = "clap" version = "4.5.9" @@ -96,11 +117,14 @@ version = "0.1.1" dependencies = [ "assert_matches", "clap", + "clap_lex", "conf_derive", "escargot", + "figment", + "http", "serde", "serde_json", - "url", + "toml", ] [[package]] @@ -113,6 +137,12 @@ dependencies = [ "syn", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "escargot" version = "0.5.11" @@ -125,6 +155,32 @@ dependencies = [ "serde_json", ] +[[package]] +name = "figment" +version = "0.10.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" +dependencies = [ + "atomic", + "serde", + "serde_json", + "toml", + "uncased", + "version_check", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" + [[package]] name = "heck" version = "0.5.0" @@ -132,14 +188,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] -name = "idna" -version = "0.1.5" +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "indexmap" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", + "equivalent", + "hashbrown", ] [[package]] @@ -161,10 +227,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] -name = "matches" -version = "0.1.10" +name = "memchr" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "once_cell" @@ -172,12 +238,6 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" -[[package]] -name = "percent-encoding" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" - [[package]] name = "proc-macro2" version = "1.0.86" @@ -233,6 +293,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "strsim" version = "0.11.1" @@ -251,51 +320,53 @@ dependencies = [ ] [[package]] -name = "tinyvec" -version = "1.7.0" +name = "toml" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6b6a2fb3a985e99cebfaefa9faa3024743da73304ca1c683a36429613d3d22" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" dependencies = [ - "tinyvec_macros", + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", ] [[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "unicode-bidi" -version = "0.3.15" +name = "toml_datetime" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] [[package]] -name = "unicode-ident" -version = "1.0.12" +name = "toml_edit" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] [[package]] -name = "unicode-normalization" -version = "0.1.23" +name = "uncased" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" dependencies = [ - "tinyvec", + "version_check", ] [[package]] -name = "url" -version = "1.7.2" +name = "unicode-ident" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" -dependencies = [ - "idna", - "matches", - "percent-encoding", -] +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "utf8parse" @@ -303,6 +374,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "windows-sys" version = "0.52.0" @@ -375,3 +452,12 @@ name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +dependencies = [ + "memchr", +] diff --git a/Cargo.toml b/Cargo.toml index 4f74c3b..dd52fdb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,8 +21,8 @@ include = [ [package] name = "conf" version = "0.1.1" -description = "A derive-based config parser for CLI args and env parameters" -categories = ["command-line-interface"] +description = "A derive-based config parser for CLI args, env, and structured config files" +categories = ["configuration"] keywords = [ "argument", "config", @@ -42,10 +42,17 @@ bench = false [dependencies] conf_derive = { path = "./conf_derive", version = "0.1.1" } clap = { version = "4.5.8", features = ["string"] } +clap_lex = { version = "0.7" } +serde = { version = "1", optional = true } + +[features] +default = ["serde"] [dev-dependencies] assert_matches = "1.5" escargot = "0.5" +figment = { version = "0.10", features = ["json", "toml"] } +http = { version = "1.1" } serde = { version = "1", features = ["derive"] } serde_json = "1" -url = "1" +toml = "0.8" diff --git a/MOTIVATION.md b/MOTIVATION.md index b895762..6765144 100644 --- a/MOTIVATION.md +++ b/MOTIVATION.md @@ -801,6 +801,8 @@ The initial feature set was the features of `clap-derive` I had used most heavil I ended up adding more features besides this before the first `crates.io` release as I started migrating more of my projects to this, and encountered things that were either harder to migrate, or were just additional features that I realized I wanted and could fit into the framework with relative ease. +In version 0.1.1, we added support for subcommands. + ## Testing If we change the same simple program that we used for testing `clap-derive` to use `conf` instead, we can see that the error handling in these scenarios becomes better. diff --git a/README.md b/README.md index c70dbe6..38d4377 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # conf -`conf` is a `derive`-based env-and-argument parser aimed at the practically-minded web developer building large web projects. +`conf` is a `derive`-based config parser aimed at the practically-minded web developer building large web projects. [![Crates.io](https://img.shields.io/crates/v/conf?style=flat-square)](https://crates.io/crates/conf) [![Crates.io](https://img.shields.io/crates/d/conf?style=flat-square)](https://crates.io/crates/conf) @@ -8,7 +8,7 @@ [![License](https://img.shields.io/badge/license-MIT-blue?style=flat-square)](LICENSE-MIT) [![Build Status](https://img.shields.io/github/actions/workflow/status/cbeck88/conf-rs/ci-rust.yml?branch=develop&style=flat-square)](https://github.com/cbeck88/conf-rs/actions/workflows/ci-rust.yml?query=branch%3Adevelop) -[API Docs](https://docs.rs/conf/latest/conf/) | [Proc-macro Reference](./REFERENCE.md) +[API Docs](https://docs.rs/conf/latest/conf/) | [Proc-macro Reference](./REFERENCE.md) | [Examples](./examples) ## Overview @@ -30,12 +30,32 @@ The features that you get for this bargain are: * **You can declare fields which represent secrets.** This controls whether or not the entire value should be printed in error messages if it fails to parse. * **Support for an optional-flatten syntax**. This can be simpler and more idiomatic than using argument groups and such in `clap-derive`. * **Support for user-defined validation predicates**. This allows you to express constraints that can't be expressed in `clap`. +* **Support for layered config**. This means that you can use data loaded from a file as an additional source for config values, alongside args and env. + +As of version ??? `conf` supports using config content in any [`serde`](https://docs.rs/serde/latest/serde/)-compatible format, such as JSON, YAML, TOML, etc., as a hierarchical config layer. +The same commitment to "All the errors and not just one of them" holds. There are several advantages of this integrated approach: + +* Other popular approaches to hierarchical config include using [`clap`](https://docs.rs/clap/latest/clap/) for CLI argument parsing only, and then folding the + results of that into a library like [`figment`](https://docs.rs/figment/latest/figment) or [`config`](https://docs.rs/config/latest/config), which can also manage `env`, files, and compositing it all together. + * However, typically this creates a maintanence burden, because if a required field could be read via `clap` or could be read from `env` or a config file, it needs to be `Option` + for `clap` and `T` in the final config structure, so you end up needing to maintain two parallel structures. + * If these structures get out of sync, there isn't really any tooling to help you figure it out and the error messages may be confusing. + * Dividing the information between two structures this way means that `clap` isn't aware of the other ways that a value can be read. + But `clap` is responsible for generating the `--help` text, and so this causes the documentation of the config to be incomplete and makes it harder + for users to figure out how to use your program. + * It leads to poor quality error reporting, because crates like `figment` and `config` rely on `serde::Deserialize` to marshall the composited data onto your final structure. + This precludes giving multiple error reports if there are multiple problems in different parts of the config. (See [MOTIVATION.md](./MOTIVATION.md) for more discussion.) +* When using `conf` instead, all of these problems are avoided. Notably, `conf` provides its own proc-macro, and so we can walk the `serde::de::Deserializer` ourselves and + ensure that we get comprehensive error reporting, even if `serde_derive::Deserialize` would have stopped at the first error. +* `conf` can also be used together with `figment` advantageously. See [Multiple config files](#multiple-config-files) for more on this. + +------ `conf` is heavily influenced by [`clap-derive`](https://docs.rs/clap/latest/clap/) and the earlier [`struct-opt`](https://docs.rs/structopt/latest/structopt/) which I used for years. They are both great and became popular for a reason. -In most cases, `conf` tries to stay extremely close to `clap-derive` syntax and behavior, for familiarity and ease of migrating a large project. +Where there is overlap, `conf` tries to stay extremely close to `clap-derive` syntax and behavior, in most cases, for familiarity and ease of migrating a large project. In some cases, there are small deviations from the behavior of `clap-derive` to either help avoid mistakes, or to make the defaults closer to a good [12-factor app](https://12factor.net/config) behavior. -For some advanced features of `clap`, `conf` has a way to achieve the same thing, but we took a different approach. This is typically in an attempt to simplify how it works for the user of the `derive` macro, to have fewer named concepts, or to ease maintenance going forward. +For some advanced features of `clap`, `conf` has a way to achieve the same thing, but we took a different approach. This is typically in an attempt to simplify how it works for the user of the `derive` macro, to have fewer named concepts, or to ease maintenance going forward. (Because we don't offer an analogue of the `clap_builder` API, the design tradeoffs are different.) The public API here is restricted to the `Conf` and `Subcommands` traits, proc-macros to derive them, and one error type. It is hoped that this will both reduce the learning curve and ease future development and maintenance. @@ -65,6 +85,8 @@ Then, create a `struct` which represents the configuration data your application This struct should derive the `Conf` trait, and the `conf` attributes should be used to describe how each field can be read. ```rust +use conf::Conf; + #[derive(Conf)] pub struct Config { /// This is a string parameter, which can be read from args as `--my-param` or from env as `MY_PARAM`. @@ -271,7 +293,9 @@ This section discusses more advanced features and usage patterns, as well as alt ### Reading files -Sometimes, a web service needs to read a file on startup. `conf` supports this by using the `value_parser` feature, which works very similarly as in `clap`. +Sometimes, a web service needs to read a file on startup. + +One way this can be done in `conf` is by using the `value_parser` feature, which works very similarly as in `clap`. A `value_parser` is a function that takes a `&str` and returns either a value or an error. @@ -306,7 +330,11 @@ pub struct Config { } ``` -This can also be a good pattern for things like reading a certificate or a cryptographic key from a file, which you want to check on startup, failing fast if the file is not found or is invalid. +This can be a good pattern for things like reading a certificate or a cryptographic key from a file, which you want to check on startup. +This way you will fail fast if the file is not found or is invalid, but also report all other config problems at the same time. + +This kind of approach would always read the key from a file, but would allow you to specify the file path either in args or in env. +This is not the same thing as hierarchical config files though, which we'll discuss next. ### Hierarchical config @@ -321,18 +349,117 @@ This can also be a good pattern for things like reading a certificate or a crypt > 5. System-wide configuration > 6. Default configuration shipped with the program. -`conf` has built-in support for (1), (2), and (6) here. +`conf` has strong built-in support for (1), (2), and (6) here. To get the others, there are basically two approaches. + +#### .env files -To get the others when using something like `conf`, a common practice is to use a crate like [`dotenvy`](https://crates.io/crates/dotenvy). This crate can search for an `.env` file, and then set `env` values if they are not already set in your program. -You can do this right before calling `Config::parse()`, and in this manner achieve hierarchical config. You can load multiple `.env` files this way if you need to. +The simplest approach to hierarchical config, IMO, is to use a crate like [`dotenvy`](https://crates.io/crates/dotenvy). This crate can search for an `.env` file, and then set `env` values if they are not already set in your program. +You can do this right before calling `Config::parse()`, and in this manner achieve hierarchical config, with `args > env > .env file > defaults`. You can load multiple `.env` files this way if you need to, searching user-provided paths, default paths, and so on. -In web applications, I often use this approach for *development* rather than production. +In web applications, I often use this approach for *development* rather than production, and I recommend this approach especially for smaller projects. If your application has a lot of required values, it may take an engineer a while to figure out how to just run it locally. But you may not want to provide default values in the program that would not be appropriate in production, for safety. Instead, you can provide a `.env` file which is checked in to the repo, with values which are appropriate for local testing / CI. Then an engineer can use `cargo run` and it will just work. When you go to build docker containers, you can leave out these `.env` files, and then be sure that in the deployed environment, kubernetes or similar is in total control, and any missing or misspelled values in the helm charts and whatnot will be loud and fail fast. These `.env` files work well if you are using [`diesel`](https://crates.io/crates/diesel), because the `diesel` cli tool also [uses `dotenvy` to search for a `.env` file](https://diesel.rs/guides/getting-started) and find the `DATABASE_URL` when manging database migrations locally. -This approach to hierarchical config is much less general than what crates like [`config`](https://crates.io/crates/config) and [`figment`](https://crates.io/crates/figment) offer, but it's also simpler, and it's easy to change and debug. There are other reasons discussed in [MOTIVATION.md](./MOTIVATION.md) that I personally favor this approach. This of course is highly opinionated -- over time `conf` may add more features that support other ways of using it. To start, I only built what I felt I needed. Your mileage may vary. +You can also pass `.env` files directly to `docker run` if you want to test docker containers locally. + +This is a very traditional approach to configuring 12-factor apps. +You get most of the benefits of having config files, but it's also typically easier to deploy the app if it doesn't require files to be mounted into a container, and the config is typically easier to change in a deployed environment if it is based on environment variables. + +The biggest drawback of this approach is that you are limited to things that can easily be expressed in a `.env` format. +If your config structure logically contains arrays of structs, it may not be very natural to express that in `.env`. + +Another drawback is that the `.env` format doesn't really have a spec, and there are many divergent parser implementations. Eventually you may run into incompatibilities between what `docker` does, what `bash` does, +and what the numerous `dotenv` libraries in different programming languages do. This is typically annoying but not insurmountable. + +#### General config files + +Alternatively, you may prefer that your application can load layered config from a file in a more structured format. + +In the `conf` API, self-describing structured data like this is called a "document". (`conf` doesn't really care if it actually came from a file.) + +To use a document as a source for layered config in `conf`, you can do the following: + +0. You must have the `serde` feature enabled in `conf`, which is on by default. + + You must annotate your structs with `#[conf(serde)]`. This can create additional build-time requirements -- fields in your structs might need to implement `serde::Deserialize` depending on how they are annotated. + +1. First, determine the file path and load the document content. For example, + + ```rust + let config_path = std::env::var("CONFIG").ok().or_else("config.yaml".to_owned()); + + let doc_content: serde_yaml::Value = serde_yaml::from_reader(fs::File::open(&config_path).unwrap()).unwrap(); + ``` + + Note that `conf` doesn't force you to use any particular library or error handling discipline here. + You may prefer to skip the file if it is not specified, or not found, or invalid, and try to proceed without it. + +2. Next, use the builder API to parse an instance of your structure. + + ```rust + let config = MyConfig::conf_builder() + .doc(config_path, doc_content) + .parse(); + ``` + + The builder uses `std::env::vars_os` and `std::env::args_os` as env and args sources by default, but these can be overrided if desired. + The `config_path` string parameter is used in error messages. + +Intuitively what happens is, `conf` attempts to initialize your struct, mapping the yaml data onto it, similar to `serde::Deserialize`. +However, for any fields in your `Conf` struct, if there are multiple value sources, the priority is `args > env > serde > defaults`. So values +from the `serde::Deserializer` can be shadowed, and also holes in the `serde` data can be filled from defaults and so on. + +Any `value_parser` is run only after the available value sources and their priorities have been resolved. + +`conf` will work best if you use a "self-describing" format, which has a type like `serde_yaml::Value` or `serde_json::Value` +which can hold any valid yaml or json, and you deserialize into that first. In particular, it's not recommended to do the following, even if it would avoid some copies: + +```rust + // Not recommended + let config = MyConfig::conf_builder() + .doc(config_path, serde_yaml::Deserializer::from_reader(fs::File::open(&config_path).unwrap())) + .parse(); +``` + +If the file is not valid yaml or json, then at some point in the middle of the walk, the deserializer may be in a broken state, and any further attempts to interact with it will yield errors. +Then `conf` may report numerous errors as it tries to read data for different parts of your structure, giving up on failing branches and continuing to try on other branches. +These errors may distract from the root cause. By deserializing into a `Value` type first, and failing fast if that doesn't work, you can avoid this scenario. + +See a [worked example](./examples/serde/basic.rs) which is under test if you like. + +#### Multiple config files + +A limitation of `conf` is that you can only pass it one document in this manner -- you can't call [`ConfBuilder::doc`] multiple times and pass a series of progressively lower-priority file contents. + +However, you can use other libraries to help with this. + +```rust + let content: figment::Value + = Figment::new() + .merge(Json::file("file1")) + .merge(Json::file("file2")) + .extract()?; +``` + +The [`Figment::extract` function](https://docs.rs/figment/latest/figment/struct.Figment.html#method.extract) invokes [`serde::Deserialize`](https://docs.rs/serde/latest/serde/trait.Deserialize.html), and so can only report one error. But extracting into a [`figment::Value`](https://docs.rs/figment/latest/figment/value/enum.Value.html) is not expected to fail, since this is the internal representation that `figment` uses. +The `figment::Value` can then be passed to `conf` as a document, since it implements [`serde::de::Deserializer`](https://docs.rs/serde/latest/serde/trait.Deserializer.html). Then `conf` is driving the initialization of your struct, and not `serde_derive`, which retains all the benefits of `conf`'s design. + +In this manner, you can get all 6 categories of hierarchical config in your app if needed, without significant restrictions on config file formats. + +You can see a more complete [example](./example/serde/figment.rs) and tests in the repo. + +In the future, we may extend our API so that the [`figment::Metadata`](https://docs.rs/figment/latest/figment/struct.Metadata.html), which tracks the provenance of individual values, can also be passed on to `conf` and used in error messages. + +#### Documenting the config file format + +The suggested way to help users of your program understand the config file format is: + +* Have some examples committed to your repo, and have tests that they parse correctly +* Either distribute these with the documentation, or along with the release artifacts, or bake them into the binary and add a CLI option which makes the binary emit them. + +For example, the AWS CLI tool provides options to emit a config skeleton for many commands, such as, [`aws ecs register-task-definition --generate-cli-skeleton`](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-definition-template.html). ### Secrets @@ -355,7 +482,7 @@ We'll offer just three points of guidance around this tool. 1. The more valuable the secrets are, and the more challenging the threat model is, the more time it makes sense to spend working on defensive measures. The converse is also true. No one really has context to judge this except you, so instead of offering one-size-fits-all guidance, I prefer to think in terms of a sliding scale. -2. If you're at a point where systematically marking things `secret` seems like a good idea, then you should also be using special types to manage the secrets. +2. If you're at a point where *systematically marking things `secret`* seems like a good idea, then you should also be *using special types to manage the secrets*. For example, using [`SecretString` from the `secrecy` crate](https://docs.rs/secrecy/0.8.0/secrecy/type.SecretString.html) instead of `String` will prevent your password from appearing in debug logs *after* it has been loaded. There are alternatives out there if `secrecy` crate doesn't work for your use-case. This is usually a pretty low-effort improvement, and it goes hand-in-hand with what the `secret` marking does. * It's very easy to expose your secret by accident if you don't do something like this. For example, just by putting a `#[tracing::instrument]` annotation on a function that some day takes a `config` struct, you could accidentally log your password. @@ -368,8 +495,8 @@ We'll offer just three points of guidance around this tool. ### Argument groups and constraints -`clap` has support for the concept of "argument groups" (`ArgGroup`) and also "dependencies" among `Arg`'s. This is used to create additional conditions that must be satisfied for the config to be valid, and error messages if it is invalid. -`clap` provides many functions on `Arg` and on `ArgGroup` which can be used to define various kinds of constraints, such as conditional dependency or mutual exclusion, between `Arg`'s or `ArgGroup`'s. +`clap` has support for the concept of "argument groups" ([`ArgGroup`](https://docs.rs/figment/latest/figment/struct.Metadata.html)) and also "dependencies" among [`Arg`](https://docs.rs/clap/latest/clap/struct.Arg.html)'s. This is used to create additional conditions that must be satisfied for the config to be valid, and error messages if it is invalid. +`clap` [provides](https://docs.rs/clap/latest/clap/struct.Arg.html#method.conflicts_with) [many](https://docs.rs/clap/latest/clap/struct.Arg.html#method.exclusive) [functions](https://docs.rs/clap/latest/clap/struct.Arg.html#method.overrides_with) [on](https://docs.rs/clap/latest/clap/struct.Arg.html#method.required_if_eq) [`Arg`](https://docs.rs/clap/latest/clap/struct.Arg.html) [and](https://docs.rs/clap/latest/clap/struct.Arg.html#method.requires_if) [on](https://docs.rs/clap/latest/clap/struct.Arg.html#method.required_unless_present) [`ArgGroup`](https://docs.rs/figment/latest/figment/struct.Metadata.html) which can be used to define various kinds of constraints, such as conditional dependency or mutual exclusion, between `Arg`'s or `ArgGroup`'s. The main reason to use these features in `clap` is that it will generate nicely formatted errors if these constraints are violated, and then you don't have to worry about handling the situation in your application code. diff --git a/REFERENCE_derive_conf.md b/REFERENCE_derive_conf.md index c7f8510..4e76dc9 100644 --- a/REFERENCE_derive_conf.md +++ b/REFERENCE_derive_conf.md @@ -15,6 +15,9 @@ The `#[conf(...)]` attributes conform to [Rust’s structured attribute conventi * [env](#flag-env) * [aliases](#flag-aliases) * [env_aliases](#flag-env-aliases) + * [serde](#flag-serde) + * [rename](#flag-serde-rename) + * [skip](#flag-serde-skip) * [Parameter](#parameter) * [short](#parameter-short) * [long](#parameter-long) @@ -25,6 +28,10 @@ The `#[conf(...)]` attributes conform to [Rust’s structured attribute conventi * [value_parser](#parameter-value-parser) * [allow_hyphen_values](#parameter-allow-hyphen-values) * [secret](#parameter-secret) + * [serde](#parameter-serde) + * [rename](#parameter-serde-rename) + * [skip](#parameter-serde-skip) + * [use_value_parser](#parameter-serde-use-value-parser) * [Repeat](#repeat) * [long](#repeat-long) * [env](#repeat-env) @@ -35,18 +42,29 @@ The `#[conf(...)]` attributes conform to [Rust’s structured attribute conventi * [no_env_delimiter](#repeat-no-env-delimiter) * [allow_hyphen_values](#repeat-allow-hyphen-values) * [secret](#repeat-secret) + * [serde](#repeat-serde) + * [rename](#repeat-serde-rename) + * [skip](#repeat-serde-skip) + * [use_value_parser](#repeat-serde-use-value-parser) * [Flatten](#flatten) * [prefix](#flatten-prefix) * [long_prefix](#flatten-long-prefix) * [env_prefix](#flatten-env-prefix) * [help_prefix](#flatten-help-prefix) * [skip_short](#flatten-skip-short) + * [serde](#flatten-serde) + * [rename](#flatten-serde-rename) + * [skip](#flatten-serde-skip) * [Subcommands](#subcommands) + * [serde](#subcommands-serde) + * [skip](#subcommands-serde-skip) * [Struct-level attributes](#struct-level-attributes) * [no_help_flag](#struct-no-help-flag) * [about](#struct-about) * [name](#struct-name) * [env_prefix](#struct-env-prefix) + * [serde](#struct-serde) + * [allow_unknown_fields](#struct-serde-allow-unknown-fields) * [one_of_fields](#struct-one-of-fields) * [at_most_one_of_fields](#struct-at-most-one-of-fields) * [at_least_one_of_fields](#struct-at-least-one-of-fields) @@ -100,27 +118,26 @@ When `derive(Conf)` encounters a field, the first thing it must determine *what * **Flag**: A flag corresponds to a boolean program option. It is either set or it isn't. For example, `./my_prog --flag1 --flag2`. * **Parameter**: A parameter corresponds to a program option that expects a string value to be found during parsing. For example `./my_prog --param1 value1 --param2 value2`. -* **Repeat**: A repeat option represents a list of values. It has special parsing -- it is allowed to be specified multiple times on the command-line, and the results are parsed separately and aggregated into a `Vec`. This is similar to what `clap` calls a multi-option, and what `clap-derive` does by default if the field type is a `Vec`. For example, `./my_prog --can-repeat value1 --can-repeat value2`. +* **Repeat**: A repeat field represents a list of values. It has special parsing -- it is allowed to be specified multiple times on the command-line, and the results are parsed separately and aggregated into a `Vec`. This is similar to what `clap` calls a multi-option, and what `clap-derive` does by default if the field type is a `Vec`. For example, `./my_prog --can-repeat value1 --can-repeat value2`. * **Flatten**: A flatten field doesn't correspond to an option, but to a collection of options that come from another `Conf` structure, and may be adjusted before being merged in. * **Subcommands**: A subcommands field doesn't correspond to an option, but to a collection of subcommands defined by a `Subcommands` enum. When a subcommand is used, any values parsed by the subcommand parser appear at the associated enum variant. If the *first attribute* is `flag`, `parameter`, `repeat`, `flatten`, or `subcommands`, then `conf` will handle the field that way. -If none of these is found, then the *type* of the field is used to classify it. +If none of these is found, then the *type* of the field is used to classify it [^1]. * If the field is `bool`, then it is a flag * Otherwise it is a parameter. -Each kind of field then supports a different set of attributes. +In `conf` the only way to specify a repeat parameter is to use the `repeat` attribute. There is, intentionally, no type-based inference for that [^compat-note-1]. -*Note*: In `clap`, `repeat` parameters are inferred by setting the type to `Vec`, and this is the only way to specify a repeat parameter. It also changes the meaning of `value_parser` in a subtle way. -However, this can become confusing and so `conf` deviates from `clap` here. Instead, in `conf` the only way to specify a repeat parameter is to use the `repeat` attribute. +Each kind of field then supports a different set of attributes. ### Flag A flag corresponds to a switch that doesn't take any parameters. It's presence on the command line means the value is `true`, otherwise it is `false`. -*Requirements*: A flag field must have type `bool`. +**Requirements**: A flag field must have type `bool`. * `short` (optional char argument) @@ -175,11 +192,29 @@ A flag corresponds to a switch that doesn't take any parameters. It's presence o example command-line: `OLD_FLAG_NAME=1 ./my_prog` sets the flag to true +* `serde` (optional additional attributes) + + example: `#[conf(serde(rename = "foo"))]` + + Configuration specific to the serde integration. + + * `rename` (string argument) + + example: `#[conf(serde(rename = "foo"))]` + + Similar to `#[serde(rename)]`, changes the name used in serialization, which by default is the field name. + + * `skip` (no arguments) + + example: `#[conf(serde(skip))]` + + Similar to `#[serde(skip)]`, this field won't be read from the serde value source. + ### Parameter A parameter represents a single value that can be parsed from a string. -*Requirements*: A parameter field can have any type as long as it implements `FromStr` or `value_parser` is used. +**Requirements**: A parameter field can have any type as long as it implements `FromStr` or `value_parser` is used. * `short` (optional char argument) @@ -244,8 +279,7 @@ A parameter represents a single value that can be parsed from a string. By default, `conf` invokes the trait function `std::str::FromStr::from_str` to convert the parsed string to the type of the field. This can be overrided by setting `value_parser`. Any function expression can be used as long as any generic parameters are either specified or inferred. - *Note*: This is very similar to `clap-derive`, but it seems to work a little better in this crate at time of writing. For instance `value_parser = serde_json::from_str` just works, - while at `clap` version 4.5.8 it doesn't work. I'm not totally sure why that is, but it seems to be something about lifetime inferences. + *Note*: This is very similar to `clap-derive`, but there are technical differences [^compat-note-2]. * `allow_hyphen_values` (no arguments) @@ -270,6 +304,32 @@ A parameter represents a single value that can be parsed from a string. If the `bool` argument is not specified when this attribute appears, it is considered `true`. Values not marked secret are considered not to be secrets. +* `serde` (optional additional attributes) + + example: `#[conf(serde(use_value_parser, rename = "foo"))]` + + Configuration specific to the serde integration. + + * `rename` (string argument) + + example: `#[conf(serde(rename = "foo"))]` + + Similar to `#[serde(rename)]`, changes the name used in serialization, which by default is the field name. + + * `skip` (no arguments) + + example: `#[conf(serde(skip))]` + + Similar to `#[serde(skip)]`, this field won't be read from the serde value source. + This can be useful if the type doesn't implement `serde::Deserialize`. + + * `use_value_parser` (no arguments) + + example: `#[conf(serde(use_value_parser))]` + + If used, then instead of asking `serde` to deserialize the field type, `serde` will deserialize a `String`, + and then the `value_parser` will convert the string to the field type. The default value parser is `FromStr`. + #### Notes When a parameter is parsed from CLI arguments, the parser expects one of two syntaxes `--param=value` or `--param value` to be used. If `value` is not found as expected then parsing will fail. @@ -288,7 +348,7 @@ Currently none of the other special [type-based intent inferences that clap does A repeat field is similar to a parameter, except that it may appear multiple times on the command line, and the collection of string arguments are then parsed individually and aggregated. -*Requirements*: A repeat field must have type `Vec`, where `T` implements `FromStr`, or `value_parser` must be supplied that produces a `T`. +**Requirements**: A repeat field must have type `Vec`, where `T` implements `FromStr`, or `value_parser` must be supplied that produces a `T`. *Note*: A repeat option produces one `T` for each time the option appears in the CLI arguments, and unlike a parameter the option can appear multiple times. If it does not appear, and an `env` variable is specified, then that variable is read and split on a delimiter character which defaults to `','`, to produce a series of `T` values. @@ -374,19 +434,45 @@ is read and split on a delimiter character which defaults to `','`, to produce a If the `bool` argument is not specified when this attribute appears, it is considered `true`. Values not marked secret are considered not to be secrets. +* `serde` (optional additional attributes) + + example: `#[conf(serde(use_value_parser, rename = "foos"))]` + + Configuration specific to the serde integration. + + * `rename` (string argument) + + example: `#[conf(serde(rename = "foos"))]` + + Similar to `#[serde(rename)]`, changes the name used in serialization, which by default is the field name. + + * `skip` (no arguments) + + example: `#[conf(serde(skip))]` + + Similar to `#[serde(skip)]`, this field won't be read from the serde value source. + This can be useful if the value doesn't implement `serde::Deserialize`. + + * `use_value_parser` (no arguments) + + example: `#[conf(serde(use_value_parser))]` + + If used, then instead of asking `serde` to deserialize `Vec`, `serde` will deserialize a `Vec`, + and then the `value_parser` will convert each string to `T`. The default value parser is `FromStr`. + #### Notes `clap-derive`'s multi-option's don't work that well in a 12-factor app, because there's a mismatch between, getting multiple strings from the CLI arguments, and getting one string from env. `clap-derive`'s behavior for a typical case like -```ignore +```rust ignore #[clap(long, env)] my_list: Vec, ``` when there is no CLI arg and only env is set, is that the entire env value becomes the one element of `my_list`, and there is no way to configure a list with multiple items by setting only `env`. -So, most likely an app that was using `clap` this way was only using the CLI arguments to configure this value. +So, most likely an app that was using `clap` this way was only using the CLI arguments to configure this value. For `conf`, we consider that this is not a good default behavior. `clap` does have an additional option for this case called `value_delimiter`, which will cause it to split both CLI arguments and `env` values on a given character. In `conf` however, at this point the field can just be `parameter` instead of a `repeat`, and a `value_parser` can be used which does the splitting. @@ -404,7 +490,7 @@ and you can customize this if another choice of delimiter is more appropriate. ### Flatten -*Note*: A flatten field's type `T` must implement `Conf`, or the field type must be `Option` where `T` implements `Conf`. +**Requirements**: A flatten field's type must be `T` or` Option` where `T: Conf`. * `env_prefix` (optional string argument) @@ -457,20 +543,40 @@ and you can customize this if another choice of delimiter is more appropriate. at this flattening site. So, when you see this attribute appearing, you can be sure that all of the named short flags are actually being removed at this location, and not by some other `skip_short` attribute appearing in another location. +* `serde` (optional additional attributes) + + example: `#[conf(serde(rename = "foo"))]` + + Configuration specific to the serde integration. + + * `rename` (string argument) + + example: `#[conf(serde(rename = "foo"))]` + + Similar to `#[serde(rename)]`, changes the name used in serialization, which by default is the field name. + + * `skip` (no arguments) + + example: `#[conf(serde(skip))]` + + Similar to `#[serde(skip)]`, this substructure won't be read from the serde value source. + #### Notes Using `flatten` with no additional attributes behaves the same as `clap(flatten)`. When using `flatten` with `Option`, the parsing behavior is: -* If none of the fields of `T` (after flattening and prefixes) are present among the CLI arguments or env, then the result is `None`. -* If any of the fields of `T` are present, then we must succeed in parsing a `T` as usual, and the result is `Some`. +* If none of the fields of `T` (after flattening and prefixes) are present among the CLI arguments or env, and the substructure doesn't appear in the `serde` document, then the result is `None`. +* If any of the fields of `T` are present, or if the substructure appears in the serde document, then we must succeed in parsing a `T` as usual, and the result is `Some`. ### Subcommands A `subcommands` field works similarly to a `#[clap(subcommand)]` field, and represents one or more subcommands that can be used with this `Conf`. -* The type of a `subcommands` field is `T` or `Option` where `T` is an enum with `#[derive(Subcommands)]`. `Option` means that use of a subcommand is optional. +**Requirements**: A `subcommands` field type must be `T` or `Option` where `T: Subcommands`. + +* `Option` means that use of a subcommand is optional, otherwise, one of the subcommands must appear. * Each enum variant corresponds to one subcommand. The enum variant determines the name of the subcommand, and the value is a `Conf` structure. * If the subcommand appears on the command-line, then the subcommand is active, and the remaining arguments are handled by the subcommand parser. The result of this parse is stored as the enum value. @@ -483,11 +589,20 @@ For example, in one mode, you might run a webserver, and in another you might ru When you use subcommands, the top-level `--help` won't show any subcommand-specific configuration options, and the help of each subcommand only shows options relevant to that subcommand. This can help users navigate the help more easily. -`subcommands` fields do not support any additional attributes at this time. +See also the [`Subcommands`] trait and proc-macro documentation. -See also the [`Subcommands`] trait and documentation. +* `serde` (optional additional attributes) -*Restrictions*: + Configuration specific to the serde integration. + + * `skip` (no arguments) + + example: `#[conf(serde(skip))]` + + These subcommands won't support reading anything from the serde value source, and not need have `#[serde(conf)]` + when derived. + +**Restrictions**: * At most one `subcommands` field can appear in a given struct. * `subcommands` fields are only valid at top-level, and cannot be used in a struct that is flattened. @@ -534,6 +649,16 @@ works on that struct. Attributes that are not "top-level only" will still have a The given string is concatenated to the beginning of every env form and env alias of every program option associated to this struct. +* `serde` (optional additional attributes) + + example: `#[conf(serde)]`, `#[conf(serde(allow_unknown_fields))]` + + Enable serde as a value source for this struct, for additional layered config patterns. + + * `allow_unknown_fields` (no arguments) + + Similar to `#[serde(deny_unknown_fields)]`, except that the default is reversed here, to avoid configuration mistakes. + * `one_of_fields` (parenthesized identifier list) example: `#[conf(one_of_fields(a, b, c))]` @@ -547,7 +672,7 @@ works on that struct. Attributes that are not "top-level only" will still have a The total number of these fields which are "present" (`true` or `Some` or `non-empty`) must be exactly one, otherwise an error will be generated describing the offending / missing fields, with context. - Note that any of the field kinds is potentially supported (`flag`, `parameter`, `repeat`, `flatten`). + Note that any of the field kinds is potentially supported (`flag`, `parameter`, `repeat`, `flatten`, `subcommands`). * `at_most_one_of_fields` (parenthesized identifier list) @@ -571,3 +696,15 @@ works on that struct. Attributes that are not "top-level only" will still have a Creates a validation constraint that must be satisfied after parsing this struct succeeds, from a user-defined function. The function should have signature `fn(&T) -> Result<(), impl Display>`. + + +[^1]: Actually, the *tokens* of the type are used, so e.g. it must be `bool` and not an alias for `bool`. + +[^compat-note-1]: In `clap`, `repeat` parameters are inferred by setting the type to `Vec`, and this is the only way to specify a repeat parameter. It also changes the meaning of `value_parser` in a subtle way. +However, this can become confusing and so `conf` deviates from `clap` here. Instead, in `conf` the only way to specify a repeat parameter is to use the `repeat` attribute. + +[^compat-note-2]: Our `value_parser` feature is very similar to `clap-derive`, but it seems to work a little better in this crate at time of writing. For instance `value_parser = serde_json::from_str` just works, +while at `clap` version 4.5.8 it doesn't work. The reason seems to be that in clap v4, the `value_parser!` macro was introduced, and it uses auto-ref specialization to try to detect +features of the value parser type at build time and handle special cases. However, this adds more layers of complexity and prevents the compiler from inferring things like lifetime parameters, afaict, so it makes the +UX of the `derive` API somewhat worse. Our criteria for how to implement `value_parser` are also a bit different because we don't have a need for the solution to work with the `clap_builder` API as well. + diff --git a/REFERENCE_derive_subcommands.md b/REFERENCE_derive_subcommands.md index 0957710..3acea53 100644 --- a/REFERENCE_derive_subcommands.md +++ b/REFERENCE_derive_subcommands.md @@ -7,6 +7,15 @@ These are documented here. The `#[conf(...)]` attributes conform to [Rust’s structured attribute convention](https://doc.rust-lang.org/reference/attributes.html#meta-item-attribute-syntax). +* [Where can conf attributes be used?](#where-can-conf-attributes-be-used) +* [Enum-level attributes](#enum-level-attributes) + * [serde](#enum-serde) +* [Variant-level attributes](#variant-level-attributes) + * [name](#variant-name) + * [serde](#variant-serde) + * [rename](#variant-serde-rename) + * [skip](#variant-serde-skip) + ## Where can conf attributes be used? The `#[conf(...)]` attributes can appear on an `enum` or a `variant` of the `enum`. @@ -15,6 +24,7 @@ The `#[conf(...)]` attributes can appear on an `enum` or a `variant` of the `enu use conf::Subcommands; #[derive(Subcommands)] +#[conf(serde)] // This is an enum-level attribute pub enum MySubcommands { Run(RunConfig), // This is a variant-level attribute @@ -26,15 +36,45 @@ pub enum MySubcommands { } ``` -Each enum variant must have one unnamed field, which is a `struct` type which implements [`Conf`]. - -This is more restrictive than the corresponding `clap` system, which allows named fields in the enum variants, -decorated with attributes equivalent to those that appear on struct fields. For now, to do that in `conf` -you have actually make separate structs. This is equally expressive from the user's point of view, and is easier for us to maintain. +Each enum variant must have one unnamed field, which is a `struct` type which implements [`Conf`] [^compat-note-1]. ## Enum-level attributes -There are no enum-level attributes at this time. +* `serde` (no arguments) + + example: `#[conf(serde)]`, `#[subcommands(serde)]` + + Enable the serde integration on this enum. + + The interaction with `serde` is: + + * Each subcommand that is not `#[conf(serde(skip))]` now has a serialization name as well. + * If that key appears in the serde document, *and* the subcommand appears in the CLI args, + then the subcommand variant reads from the corresponding corresponding value in the serde document. + * If the key appears in the serde document, but the subcommand *does not* appear in the CLI args, + then this serde value is simply ignored, and it is not an error. + + This allows the previous example to work with a TOML config file structured like this: + + ```toml + [run] + run_param = "..." + + [run_migrations] + migrations_param = "..." + + [run_validation] + validation_param = "..." + ``` + + If you invoke `./my_prog run`, the `run` subcommand will pick up values from the `[run]` block, + and the other sections won't cause an error even though they are unused. + Similarly `./my_prog migrate` would pick up values from the `[run_migrations]` block, without errors. + + You can change the serialization name of a subcommand using `#[conf(serde(rename = "..."))]`. + + You may also prefer that two or more subcommands that have a lot of overlap read from the same + section of the config file. For this, you can just make the serialization names the same [^2]. ## Variant-level attributes @@ -45,3 +85,30 @@ There are no enum-level attributes at this time. Set the name of this subcommand, which is used to activate the subcommand and is documented in the help. If this attribute is not present, the name is the lower snake-case of the variant name. + +* `serde` (optional additional attributes) + + example: `#[conf(serde(rename = "foo"))]` + + Configuration specific to the serde integration. + + * `rename` (string argument) + + example: `#[conf(serde(rename = "foo"))]` + + Similar to `#[serde(rename)]`, changes the name used in serialization. + + If this attribute is not present, the serialization name is the lower snake-case of the variant name. + + * `skip` (no arguments) + + example: `#[conf(serde(skip))]` + + Similar to `#[serde(skip)]`, this subcommand won't read data from the serde value source. + +[^compat-note-1]: This is more restrictive than the corresponding `clap` system for subcommands, which allows named fields in the enum variants, +decorated with attributes equivalent to those that appear on struct fields. For now, to do that in `conf` +you have to declare separate structs. This is equally expressive from the user's point of view, and is easier for us to maintain. +[^2]: Normally, making two fields have the same serialization name won't work in `serde`. In `serde` it is only possible to deserialize a value at most once, + so you can't populate two different fields with the same deserializer content. Also it would likely break `Serialize`. In this case, we aren't serializing + anything, and the enum semantics ensure that we will only deserialize this value at most once. diff --git a/conf_derive/src/lib.rs b/conf_derive/src/lib.rs index 86b294c..18f710a 100644 --- a/conf_derive/src/lib.rs +++ b/conf_derive/src/lib.rs @@ -1,14 +1,14 @@ use proc_macro::TokenStream as TokenStream1; -use proc_macro2::{Span, TokenStream}; +use proc_macro2::TokenStream; use quote::quote; use syn::{parse_macro_input, DeriveInput}; -use syn::{Data, DataEnum, DataStruct, Error, Fields, Generics, Ident}; +use syn::{Data, DataEnum, DataStruct, Error, Fields}; mod proc_macro_options; -use proc_macro_options::{collect_args_fields, FieldItem, StructItem}; +use proc_macro_options::GenConfStruct; mod subcommand_proc_macro_options; -use subcommand_proc_macro_options::{collect_enum_variants, VariantItem}; +use subcommand_proc_macro_options::GenSubcommandsEnum; pub(crate) mod util; @@ -29,9 +29,15 @@ fn derive_conf(input: &DeriveInput) -> Result { fields: Fields::Named(fields), .. }) => { - let struct_item = StructItem::new(ident, &input.attrs)?; - let fields = collect_args_fields(&struct_item, fields)?; - gen_conf_impl_for_struct(&struct_item, ident, &input.generics, &fields) + let gen = GenConfStruct::new(ident, &input.attrs, fields)?; + let conf_impl = gen.gen_conf_impl(&input.generics)?; + let maybe_serde = gen.maybe_gen_conf_serde_impl(&input.generics)?; + + Ok(quote! { + #conf_impl + + #maybe_serde + }) } _ => Err(Error::new( @@ -41,274 +47,6 @@ fn derive_conf(input: &DeriveInput) -> Result { } } -fn gen_conf_impl_for_struct( - struct_item: &StructItem, - item_name: &Ident, - generics: &Generics, - fields: &[FieldItem], -) -> Result { - let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); - - let get_parser_config_impl = gen_conf_get_parser_config_impl_for_struct(struct_item, fields)?; - let get_program_options_impl = - gen_conf_get_program_options_impl_for_struct(struct_item, fields)?; - let get_subcommands_impl = gen_conf_get_subcommands_impl_for_struct(struct_item, fields)?; - let from_conf_context_impl = gen_conf_from_conf_context_impl_for_struct(struct_item, fields)?; - let get_name_impl = gen_conf_get_name_impl_for_struct(struct_item)?; - - Ok(quote! { - #[automatically_derived] - #[allow( - unused_qualifications, - )] - impl #impl_generics conf::Conf for #item_name #ty_generics #where_clause { - #get_parser_config_impl - #get_program_options_impl - #get_subcommands_impl - #from_conf_context_impl - #get_name_impl - } - }) -} - -fn gen_conf_get_name_impl_for_struct(struct_item: &StructItem) -> Result { - let struct_name = struct_item.get_ident().to_string(); - - Ok(quote! { - fn get_name() -> &'static str { - #struct_name - } - }) -} - -fn gen_conf_get_parser_config_impl_for_struct( - struct_item: &StructItem, - _fields: &[FieldItem], -) -> Result { - // To implement Conf::get_parser_config, we need to get parser_config - // for this struct, (top-level config essentially). - let parser_config = struct_item.gen_parser_config()?; - - Ok(quote! { - fn get_parser_config() -> Result { - let parser_config = #parser_config; - - Ok(parser_config) - } - }) -} - -fn gen_conf_get_program_options_impl_for_struct( - struct_item: &StructItem, - fields: &[FieldItem], -) -> Result { - // To implement Conf::get_program_options, we need to - // get all the program options for our constituents. To do this, we create - // an ident for the list of program options, which is going to be Vec. - // Then we pass that ident to every constitutent field, and aggregate all their code gen. - let program_options_ident = Ident::new("program_options", Span::call_site()); - let fields_push_program_options: Vec = fields - .iter() - .map(|field| field.gen_push_program_options(&program_options_ident)) - .collect::, syn::Error>>()?; - - // To implement #[conf(env_prefix="ACME_")] on a struct (rather than on a flattened field), - // the code gen associated to the struct needs to be able to add its own prefixing during - // get_program_options and during from_conf_context. - // To do this, we allow the struct_item to "post-process" the Vec, (to add a - // prefix to them all) and to "pre-process" the ConfContext (to add a matching prefix to - // that before it is used) Note: The preprocessing no longer does anything since we switched - // to using id's like clap does. - let struct_post_process_program_options = - struct_item.gen_post_process_program_options(&program_options_ident)?; - - // Note: fields_push_program_options is allowed to early return with ? on an error - Ok(quote! { - fn get_program_options() -> Result<&'static [conf::ProgramOption], conf::Error> { - static CACHED: ::std::sync::OnceLock> = ::std::sync::OnceLock::new(); - - if CACHED.get().is_none() { - let mut #program_options_ident = vec![]; - - #(#fields_push_program_options)* - - #struct_post_process_program_options - - let _ = CACHED.set(#program_options_ident); - } - - let cached = CACHED.get().unwrap(); - - Ok(cached.as_ref()) - } - }) -} - -fn gen_conf_get_subcommands_impl_for_struct( - _struct_item: &StructItem, - fields: &[FieldItem], -) -> Result { - let parsers_ident = Ident::new("__parsers__", Span::call_site()); - let parsed_env_ident = Ident::new("__parsed_env__", Span::call_site()); - let fields_push_subcommands: Vec = fields - .iter() - .map(|field| field.gen_push_subcommands(&parsers_ident, &parsed_env_ident)) - .collect::, syn::Error>>()?; - - Ok(quote! { - fn get_subcommands(#parsed_env_ident: &::conf::ParsedEnv) -> Result, conf::Error> { - let mut #parsers_ident = vec![]; - - #(#fields_push_subcommands)* - - Ok(#parsers_ident) - } - }) -} - -fn gen_conf_from_conf_context_impl_for_struct( - struct_item: &StructItem, - fields: &[FieldItem], -) -> Result { - // To implement Conf::from_conf_context, we need to take a conf context, - // and then return Ok(Self { ... }). For each constituent field, we need it - // to generate code to initialize itself properly. We pass the ConfContext ident - // to each consittuent field, and then aggregate all their code gen. - // Their code-gen is allowed to use `?` or `return Err(...)` to early return, - // but we still need to aggregate all the errors. Sample code gen is like. - // - // struct Sample { - // a: i32, - // b: i64, - // } - // - // from_conf_context(conf_context: &conf::ConfContext) -> Result> { - // let conf_context = #preprocess_conf_context; - // let mut errors = Vec::::new(); - // - // let a = match || -> Result { ... }() { - // Ok(val) => Some(val), - // Err(err) => { - // errors.push(err); - // None - // } - // }; - // - // let b = match || -> Result { ... }() { - // Ok(val) => Some(val), - // Err(err) => { - // errors.push(err); - // None - // } - // }; - // - // let return_value = match (a, b) { - // (Some(a), Some(b)) => Ok(Self { - // a, - // b, - // }), - // _ => Err(errors), - // }?; - // - // validation_predicate(&return_value).map_err(|err| - // vec![conf::InnerError::validation(&conf_context.id, err)])?; - // - // Ok(return_value) - // } - // - // The list of let a, let b... is called #initializations - // The match (a,b, ...) { ... } is called #return_value - // The validation_predicate(...) part is called #apply_validation_predicate - let conf_context_ident = Ident::new("__conf_context__", Span::call_site()); - let errors_ident = Ident::new("__errors__", Span::call_site()); - // For each field, intialize a local variable with Option which is some if it worked and None - // if there were errors. Push all errors into #errors_ident. - let initializations: Vec = fields - .iter() - .map(|field| -> Result { - let field_name = field.get_field_name(); - let field_type = field.get_field_type(); - let (initializer, returns_multiple_errors) = - field.gen_initializer(&conf_context_ident)?; - // The initializer is the portion e.g. - // || -> Result { ... } - // - // It returns `Result>` if returns_multiple_errors is true, otherwise it's Result - // It is allowed to read #conf_context_ident but not modify it - // We have to put it inside a locally defined fn so that it cannot modify the errors buffer etc. - - Ok(if returns_multiple_errors { - quote! { - fn #field_name(#conf_context_ident: &::conf::ConfContext<'_>) -> Result<#field_type, Vec<::conf::InnerError>> { - #initializer - } - let #field_name = match #field_name(&#conf_context_ident) { - Ok(val) => Some(val), - Err(errs) => { - #errors_ident.extend(errs); - None - } - }; - } - } else { - quote! { - fn #field_name(#conf_context_ident: &::conf::ConfContext<'_>) -> Result<#field_type, ::conf::InnerError> { - #initializer - } - let #field_name = match #field_name(&#conf_context_ident) { - Ok(val) => Some(val), - Err(err) => { - #errors_ident.push(err); - None - }, - }; - } - }) - }) - .collect::, syn::Error>>()?; - - let field_names = fields - .iter() - .map(|field| field.get_field_name()) - .collect::>(); - - let return_value: TokenStream = quote! { - match (#(#field_names),*) { - (#(Some(#field_names)),*) => Ok( Self { #(#field_names),* } ), - _ => Err(#errors_ident) - } - }; - - let instance_ident = Ident::new("__instance__", Span::call_site()); - - let validation_routine = - struct_item.gen_validation_routine(&instance_ident, &conf_context_ident, fields)?; - - let struct_ident = struct_item.get_ident(); - let struct_pre_process_conf_context = - struct_item.gen_pre_process_conf_context(&conf_context_ident)?; - - Ok(quote! { - fn from_conf_context<'a>(#conf_context_ident: ::conf::ConfContext<'a>) -> Result> { - #struct_pre_process_conf_context - - let mut #errors_ident = Vec::<::conf::InnerError>::new(); - - #(#initializations)* - - let return_value = #return_value?; - - fn validation<'a>(#instance_ident: & #struct_ident, #conf_context_ident: ::conf::ConfContext<'a>) -> Result<(), Vec<::conf::InnerError>> { - #validation_routine - } - - validation(&return_value, #conf_context_ident)?; - - Ok(return_value) - } - }) -} - /// Derive a `Subcommands` implementation for an item with `#[conf(...)]` attributes #[proc_macro_derive(Subcommands, attributes(conf))] pub fn subcommands(input: TokenStream1) -> TokenStream1 { @@ -323,8 +61,15 @@ fn derive_subcommands(input: &DeriveInput) -> Result { match &input.data { Data::Enum(DataEnum { variants, .. }) => { - let variants = collect_enum_variants(ident, variants.into_iter())?; - gen_subcommands_impl_for_enum(ident, &input.generics, &variants) + let gen = GenSubcommandsEnum::new(ident, &input.attrs, variants.into_iter())?; + let subcommands_impl = gen.gen_subcommands_impl(&input.generics)?; + let maybe_serde = gen.maybe_gen_subcommands_serde_impl(&input.generics)?; + + Ok(quote! { + #subcommands_impl + + #maybe_serde + }) } _ => Err(Error::new( @@ -333,90 +78,3 @@ fn derive_subcommands(input: &DeriveInput) -> Result { )), } } - -fn gen_subcommands_impl_for_enum( - item_name: &Ident, - generics: &Generics, - variants: &[VariantItem], -) -> Result { - let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); - - let get_parsers_impl = gen_subcommands_get_parsers_impl_for_enum(item_name, variants)?; - let get_subcommand_names_impl = - gen_subcommands_get_subcommand_names_impl_for_enum(item_name, variants)?; - let from_conf_context_impl = - gen_subcommands_from_conf_context_impl_for_enum(item_name, variants)?; - - Ok(quote! { - #[automatically_derived] - #[allow( - unused_qualifications, - )] - impl #impl_generics conf::Subcommands for #item_name #ty_generics #where_clause { - #get_parsers_impl - #get_subcommand_names_impl - #from_conf_context_impl - } - }) -} - -fn gen_subcommands_get_parsers_impl_for_enum( - _item_name: &Ident, - variants: &[VariantItem], -) -> Result { - let parsers_ident = Ident::new("__parsers__", Span::call_site()); - let parsed_env_ident = Ident::new("__parsed_env__", Span::call_site()); - let variants_push_parsers: Vec = variants - .iter() - .map(|var| var.gen_push_parsers(&parsers_ident, &parsed_env_ident)) - .collect::, syn::Error>>()?; - - Ok(quote! { - fn get_parsers(#parsed_env_ident: &::conf::ParsedEnv) -> Result, ::conf::Error> { - let mut #parsers_ident = vec![]; - - #(#variants_push_parsers)* - - Ok(#parsers_ident) - } - }) -} - -fn gen_subcommands_get_subcommand_names_impl_for_enum( - _item_name: &Ident, - variants: &[VariantItem], -) -> Result { - let command_names: Vec<_> = variants.iter().map(|var| var.get_command_name()).collect(); - - Ok(quote! { - fn get_subcommand_names() -> &'static [&'static str] { - &[ #(#command_names,)* ] - } - }) -} - -fn gen_subcommands_from_conf_context_impl_for_enum( - _item_name: &Ident, - variants: &[VariantItem], -) -> Result { - let variant_match_arms: Vec = variants - .iter() - .map(|var| { - let name = var.get_name(); - let command_name = var.get_command_name(); - let ty = var.get_type(); - quote! { - #command_name => Ok(Self::#name(<#ty as Conf>::from_conf_context(conf_context)?)) - } - }) - .collect(); - - Ok(quote! { - fn from_conf_context(command_name: String, conf_context: ::conf::ConfContext<'_>) -> Result> { - match command_name.as_str() { - #(#variant_match_arms,)* - _ => { panic!("Unknown command name '{command_name}'. This is an internal error. Expected '{:?}'", ::get_subcommand_names()) } - } - } - }) -} diff --git a/conf_derive/src/proc_macro_options/field_item/flag_item.rs b/conf_derive/src/proc_macro_options/field_item/flag_item.rs index 36eef5e..62375c1 100644 --- a/conf_derive/src/proc_macro_options/field_item/flag_item.rs +++ b/conf_derive/src/proc_macro_options/field_item/flag_item.rs @@ -1,9 +1,56 @@ use super::StructItem; use crate::util::*; -use proc_macro2::TokenStream; +use proc_macro2::{Span, TokenStream}; use quote::quote; -use syn::{parse_quote, spanned::Spanned, Error, Field, Ident, LitChar, LitStr, Type}; +use syn::{ + meta::ParseNestedMeta, parse_quote, spanned::Spanned, token, Error, Field, Ident, LitChar, + LitStr, Type, +}; +/// #[conf(serde(...))] options listed on a field of Flag kind +pub struct FlagSerdeItem { + pub rename: Option, + pub skip: bool, + span: Span, +} + +impl FlagSerdeItem { + pub fn new(meta: ParseNestedMeta<'_>) -> Result { + let mut result = Self { + rename: None, + skip: false, + span: meta.input.span(), + }; + + if meta.input.peek(token::Paren) { + meta.parse_nested_meta(|meta| { + let path = meta.path.clone(); + if path.is_ident("rename") { + set_once( + &path, + &mut result.rename, + Some(parse_required_value::(meta)?), + ) + } else if path.is_ident("skip") { + result.skip = true; + Ok(()) + } else { + Err(meta.error("unrecognized conf(serde) option")) + } + })?; + } + + Ok(result) + } +} + +impl GetSpan for FlagSerdeItem { + fn get_span(&self) -> Span { + self.span + } +} + +/// Proc macro annotations parsed from a field of Flag kind pub struct FlagItem { field_name: Ident, short_switch: Option, @@ -11,6 +58,7 @@ pub struct FlagItem { aliases: Option, env_name: Option, env_aliases: Option, + serde: Option, doc_string: Option, } @@ -28,6 +76,7 @@ impl FlagItem { aliases: None, env_name: None, env_aliases: None, + serde: None, doc_string: None, }; @@ -71,6 +120,8 @@ impl FlagItem { &mut result.env_aliases, Some(parse_required_value::(meta)?), ) + } else if path.is_ident("serde") { + set_once(&path, &mut result.serde, Some(FlagSerdeItem::new(meta)?)) } else { Err(meta.error("unrecognized conf flag option")) } @@ -85,7 +136,11 @@ impl FlagItem { .map(LitStrArray::is_empty) .unwrap_or(true) { - return Err(Error::new(field.span(), "Setting aliases without setting a long-switch is an error, make one of the aliases the primary switch name.")); + return Err(Error::new( + field.span(), + "Setting aliases without setting a long-switch is an error, \ + make one of the aliases the primary switch name.", + )); } if result.env_name.is_none() @@ -95,7 +150,11 @@ impl FlagItem { .map(LitStrArray::is_empty) .unwrap_or(true) { - return Err(Error::new(field.span(), "Setting env_aliases without setting an env is an error, make one of the aliases the primary env.")); + return Err(Error::new( + field.span(), + "Setting env_aliases without setting an env is an error, \ + make one of the aliases the primary env.", + )); } Ok(result) @@ -109,6 +168,21 @@ impl FlagItem { parse_quote! { bool } } + pub fn get_serde_name(&self) -> LitStr { + self.serde + .as_ref() + .and_then(|serde| serde.rename.clone()) + .unwrap_or_else(|| LitStr::new(&self.field_name.to_string(), self.field_name.span())) + } + + pub fn get_serde_type(&self) -> Type { + parse_quote! { bool } + } + + pub fn get_serde_skip(&self) -> bool { + self.serde.as_ref().map(|serde| serde.skip).unwrap_or(false) + } + pub fn gen_push_program_options( &self, program_options_ident: &Ident, @@ -164,4 +238,25 @@ impl FlagItem { false, )) } + + pub fn gen_initializer_with_doc_val( + &self, + conf_context_ident: &Ident, + _doc_name: &Ident, + doc_val: &Ident, + ) -> Result<(TokenStream, bool), Error> { + let id = self.field_name.to_string(); + + Ok(( + quote! { + let (src, val) = #conf_context_ident.get_boolean_opt(#id)?; + if src.is_default() { + Ok(#doc_val) + } else { + Ok(val) + } + }, + false, + )) + } } diff --git a/conf_derive/src/proc_macro_options/field_item/flatten_item.rs b/conf_derive/src/proc_macro_options/field_item/flatten_item.rs index c3fc195..e22799d 100644 --- a/conf_derive/src/proc_macro_options/field_item/flatten_item.rs +++ b/conf_derive/src/proc_macro_options/field_item/flatten_item.rs @@ -4,8 +4,52 @@ use heck::{ToKebabCase, ToShoutySnakeCase}; use proc_macro2::{Span, TokenStream}; use quote::quote; use std::fmt::Display; -use syn::{spanned::Spanned, Error, Field, Ident, LitStr, Type}; +use syn::{meta::ParseNestedMeta, spanned::Spanned, token, Error, Field, Ident, LitStr, Type}; +/// #[conf(serde(...))] options listed on a field of Flatten kind +pub struct FlattenSerdeItem { + pub rename: Option, + pub skip: bool, + span: Span, +} + +impl FlattenSerdeItem { + pub fn new(meta: ParseNestedMeta<'_>) -> Result { + let mut result = Self { + rename: None, + skip: false, + span: meta.input.span(), + }; + + if meta.input.peek(token::Paren) { + meta.parse_nested_meta(|meta| { + let path = meta.path.clone(); + if path.is_ident("rename") { + set_once( + &path, + &mut result.rename, + Some(parse_required_value::(meta)?), + ) + } else if path.is_ident("skip") { + result.skip = true; + Ok(()) + } else { + Err(meta.error("unrecognized conf(serde) option")) + } + })?; + } + + Ok(result) + } +} + +impl GetSpan for FlattenSerdeItem { + fn get_span(&self) -> Span { + self.span + } +} + +/// Proc macro annotations parsed from a field of Flatten kind pub struct FlattenItem { field_name: Ident, field_type: Type, @@ -14,6 +58,7 @@ pub struct FlattenItem { env_prefix: Option, description_prefix: Option, skip_short: Option, + serde: Option, } fn make_long_prefix(ident: &impl Display, span: Span) -> Option { @@ -43,6 +88,7 @@ impl FlattenItem { env_prefix: None, description_prefix: None, skip_short: None, + serde: None, }; // These two variables are used to set description_prefix at the end. @@ -99,6 +145,8 @@ impl FlattenItem { &mut result.skip_short, Some(parse_required_value::(meta)?), ) + } else if path.is_ident("serde") { + set_once(&path, &mut result.serde, Some(FlattenSerdeItem::new(meta)?)) } else { Err(meta.error("unrecognized conf flatten option")) } @@ -128,8 +176,19 @@ impl FlattenItem { self.field_type.clone() } + fn get_serde_name(&self) -> LitStr { + self.serde + .as_ref() + .and_then(|serde| serde.rename.clone()) + .unwrap_or_else(|| LitStr::new(&self.field_name.to_string(), self.field_name.span())) + } + + pub fn get_serde_skip(&self) -> bool { + self.serde.as_ref().map(|serde| serde.skip).unwrap_or(false) + } + // Body of a routine which extends #program_options_ident to hold any program options associated - // to this field + // to this field. pub fn gen_push_program_options( &self, program_options_ident: &Ident, @@ -158,52 +217,62 @@ impl FlattenItem { .map(|array| array.elements.len()) .unwrap_or(0); + // Identifier for was_skipped variable, used with skip_short_forms sanity checks. + // This is an array of bools, one for each skip-short parameter. + let was_skipped_ident = Ident::new("__was_skipped__", Span::call_site()); + // Common modifications we have to make to program options whether the flatten is optional // or required let common_program_option_modifications = quote! { - .apply_flatten_prefixes(#id_prefix, #long_prefix, #env_prefix, #description_prefix) - .skip_short_forms(&[#skip_short], &mut was_skipped[..]) + .apply_flatten_prefixes(#id_prefix, #long_prefix, #env_prefix, #description_prefix) + .skip_short_forms(&[#skip_short], &mut #was_skipped_ident[..]) }; - Ok(if let Some(inner_type) = self.is_optional_type.as_ref() { - // This is flatten-optional. We have to request inner-type program options, - // and do the same things to them, except also call make_optional() on them at the end. + // When using flatten optional, we have to make all program options optional + // before passing them on to lower layers. If not, there are no additional mods needed. + let modify_program_option = if self.is_optional_type.is_some() { quote! { - let mut was_skipped = [false; #skip_short_len]; - #program_options_ident.extend( - #inner_type::get_program_options()?.iter().cloned().map( - |program_option| - program_option - #common_program_option_modifications - .make_optional() - ) - ); - if !was_skipped.iter().all(|x| *x) { - let not_skipped: Vec = [#skip_short].into_iter().zip(was_skipped.into_iter()).filter_map( - |(short_form, was_skipped)| if was_skipped { None } else { Some(short_form) } - ).collect(); - return Err(::conf::Error::skip_short_not_found(not_skipped, #field_name, <#inner_type as ::conf::Conf>::get_name())); - } + #common_program_option_modifications + .make_optional() } } else { - // This is a regular flatten - quote! { - let mut was_skipped = [false; #skip_short_len]; - #program_options_ident.extend( - #field_type::get_program_options()?.iter().cloned().map( - |program_option| - program_option - #common_program_option_modifications - ) - ); - if !was_skipped.iter().all(|x| *x) { - let not_skipped: Vec = [#skip_short].into_iter().zip(was_skipped.into_iter()).filter_map( - |(short_form, was_skipped)| if was_skipped { None } else { Some(short_form) } + common_program_option_modifications + }; + + // For flatten-optional with Option, inner_type is T and implements Conf. + // For regular flaten, inner_type is simply the field type and implements Conf. + let inner_type = self.is_optional_type.as_ref().unwrap_or(field_type); + + // The initializer simply gets all program options, modifies as needed, + // and then checks for a skip-short error. + let push_expr = quote! { + let mut #was_skipped_ident = [false; #skip_short_len]; + #program_options_ident.extend( + <#inner_type as ::conf::Conf>::get_program_options()?.iter().cloned().map( + |program_option| + program_option + #modify_program_option + ) + ); + if #was_skipped_ident.iter().any(|x| !x) { + let not_skipped: Vec = + [#skip_short] + .into_iter() + .zip(#was_skipped_ident.into_iter()) + .filter_map( + |(short_form, was_skipped)| if was_skipped { None } else { Some(short_form) } ).collect(); - return Err(::conf::Error::skip_short_not_found(not_skipped, #field_name, <#field_type as ::conf::Conf>::get_name())); - } - } - }) + return Err( + ::conf::Error::skip_short_not_found( + not_skipped, + #field_name, + <#inner_type as ::conf::Conf>::get_name() + ) + ); + } + }; + + Ok(push_expr) } // Flatten fields don't add subcommands to the conf structure, because we don't support that @@ -215,17 +284,22 @@ impl FlattenItem { ) -> Result { let inner_type: &Type = self.is_optional_type.as_ref().unwrap_or(&self.field_type); let type_name = quote! { inner_type }.to_string(); - let panic_message = format!("It is not supported to declare subcommands in a flattened structure '{type_name}', only at top level. (Needs design work around prefixing.) See conf-rs discussion #4"); + let panic_message = format!( + "It is not supported to declare subcommands in a flattened structure '{type_name}', only \ + at top level. (Needs design work around prefixing.)"); Ok(quote! { if !<#inner_type as conf::Conf>::get_subcommands(#parsed_env_ident)?.is_empty() { - panic!(#panic_message); + panic!(#panic_message); } }) } - // Body of a function taking a &ConfContext returning Result<#field_type, - // Vec<::conf::InnerError>> + // Body of a function taking a &ConfContext returning + // Result<#field_type, Vec<::conf::InnerError>> + // + // Arguments: + // * conf_context_ident is the identifier of a &ConfContext variable in scope pub fn gen_initializer( &self, conf_context_ident: &Ident, @@ -234,33 +308,40 @@ impl FlattenItem { let id_prefix = self.get_id_prefix(); - if let Some(inner_type) = self.is_optional_type.as_ref() { + let initializer = if let Some(inner_type) = self.is_optional_type.as_ref() { // This is flatten-optional - Ok(( - quote! { - Ok(if let Some(option_appeared_result) = <#inner_type as ::conf::Conf>::any_program_options_appeared(&#conf_context_ident.for_flattened(#id_prefix)).map_err(|err| vec![err])? { - let #conf_context_ident = #conf_context_ident.for_flattened_optional(#id_prefix, <#inner_type as ::conf::Conf>::get_name(), option_appeared_result); - Some(<#inner_type as ::conf::Conf>::from_conf_context(#conf_context_ident)?) - } else { - None - }) - }, - true, - )) + quote! { + let option_appeared_result = + <#inner_type as ::conf::Conf>::any_program_options_appeared( + &#conf_context_ident.for_flattened(#id_prefix) + ).map_err(|err| vec![err])?; + Ok(if let Some(option_appeared) = option_appeared_result { + let #conf_context_ident = #conf_context_ident.for_flattened_optional( + #id_prefix, + <#inner_type as ::conf::Conf>::get_name(), + option_appeared + ); + Some(<#inner_type as ::conf::Conf>::from_conf_context(#conf_context_ident)?) + } else { + None + }) + } } else { // Non-optional flatten - Ok(( - quote! { - let #conf_context_ident = #conf_context_ident.for_flattened(#id_prefix); - <#field_type as ::conf::Conf>::from_conf_context(#conf_context_ident) - }, - true, - )) - } + quote! { + let #conf_context_ident = #conf_context_ident.for_flattened(#id_prefix); + <#field_type as ::conf::Conf>::from_conf_context(#conf_context_ident) + } + }; + Ok((initializer, true)) } // Returns an expression which calls any_program_options_appeared with given conf context. // This is used to get errors for one_of constraint failures. + // + // Arguments: + // * conf_context_ident is the identifier of a ConfContext variable that is in scope that we + // won't consume. pub fn any_program_options_appeared_expr( &self, conf_context_ident: &Ident, @@ -268,15 +349,109 @@ impl FlattenItem { let field_type = &self.field_type; let id_prefix = self.get_id_prefix(); - if let Some(inner_type) = self.is_optional_type.as_ref() { - Ok( - quote! { <#inner_type as ::conf::Conf>::any_program_options_appeared(& #conf_context_ident .for_flattened(#id_prefix)) }, - ) + let inner_type = self.is_optional_type.as_ref().unwrap_or(field_type); + + Ok(quote! { + <#inner_type as ::conf::Conf>::any_program_options_appeared( + & #conf_context_ident .for_flattened(#id_prefix) + ) + }) + } + + // This is used by ConfSerde + // + // When walking a map access, we match on the key, and if we get the key for this field, + // try to deserialize using DeserializeSeed. + // + // We have to use DeserializeSeed with the inner-type if this is flatten-optional, because + // DeserializeSeed isn't implemented for type on Option. + // + // We don't use the "any program options appeared" stuff here because, the key for the flattened + // structure appeared, so that turns the group on. If that key doesn't appear when we do the + // serde walk, then later we will try to initialize the group normally. + // + // Arguments: + // * ctxt: identifier of a &ConfSerdeContext in scope + // * map_access: identifier of a MapAccess in scope + // * map_access_type: identifier of the MapAccess type in this scope + // * errors_ident: identifier of a mut Vec errors buffer to which we can push + pub fn gen_serde_match_arm( + &self, + ctxt: &Ident, + map_access: &Ident, + map_access_type: &Ident, + errors_ident: &Ident, + ) -> Result<(TokenStream, Vec), Error> { + let field_name = &self.field_name; + let field_name_str = field_name.to_string(); + let field_type = &self.field_type; + let serde_name_str = self.get_serde_name(); + let id_prefix = self.get_id_prefix(); + + // It's necessary to do special handling for optional flattened structs here, + // because ConfSerdeContext is only implemented on the inner one. + // + // We also don't *have* to do any of the "any program option appeared" stuff here, + // because we only reach this line if serde provided a value for this struct, + // and that should mean that the optional group is enabled, since serde mentioned it. + // + // Note: We may want to consider an attribute that changes this behavior, so + // that if the group is not mentioned in args or env then it gets skipped even if + // serde has some values. + // + // As it stands, if serde doesn't mention it, then the `Conf::any_program_option_appeared` + // stuff runs in the deserializer_finalizer routine when we call Conf::from_conf_context + // And if serde does mention it, then we have to try to deserialize regardless of what + // `Conf::any_program_option_appeared` says. + let inner_type = self.is_optional_type.as_ref().unwrap_or(field_type); + + let val_expr = if self.is_optional_type.is_some() { + quote! { Some(__val__) } } else { - Ok( - quote! { <#field_type as ::conf::Conf>::any_program_options_appeared(& #conf_context_ident .for_flattened(#id_prefix)) }, - ) - } + quote! { __val__ } + }; + + // Note: If next_value_seed returns Err rather than Ok(Err), then I believe it means + // that our DeserializeSeed implementation never ran, since it never does that. + // But it's possible that the MapAccess will fail before even getting to that point, + // and then it could return a singular D::Error. So we should not unwrap such errors. + let match_arm = quote! { + #serde_name_str => { + if #field_name.is_some() { + #errors_ident.push( + InnerError::serde( + #ctxt.document_name, + #field_name_str, + #map_access_type::Error::duplicate_field(#serde_name_str) + ) + ); + } else { + let __seed__ = <#inner_type as ConfSerde>::Seed::from( + #ctxt.for_flattened(#id_prefix) + ); + #field_name = Some(match #map_access.next_value_seed(__seed__) { + Ok(Ok(__val__)) => { + Some(#val_expr) + } + Ok(Err(__errs__)) => { + #errors_ident.extend(__errs__); + None + } + Err(__err__) => { + #errors_ident.push( + InnerError::serde( + #ctxt.document_name, + #field_name_str, + __err__ + ) + ); + None + } + }); + } + }, + }; + Ok((match_arm, vec![serde_name_str])) } } diff --git a/conf_derive/src/proc_macro_options/field_item/mod.rs b/conf_derive/src/proc_macro_options/field_item/mod.rs index 95f4c4f..85a478e 100644 --- a/conf_derive/src/proc_macro_options/field_item/mod.rs +++ b/conf_derive/src/proc_macro_options/field_item/mod.rs @@ -1,8 +1,9 @@ use super::StructItem; use crate::util::type_is_bool; -use proc_macro2::TokenStream; -use syn::{punctuated::Punctuated, Field, Ident, Meta, Token, Type}; +use proc_macro2::{Span, TokenStream}; +use quote::quote; +use syn::{punctuated::Punctuated, Error, Field, Ident, LitStr, Meta, Token, Type}; mod flag_item; mod flatten_item; @@ -26,7 +27,7 @@ pub enum FieldItem { } impl FieldItem { - pub fn new(field: &Field, struct_item: &StructItem) -> Result { + pub fn new(field: &Field, struct_item: &StructItem) -> Result { // First, inspect the first field attribute. // If the first attribute is 'flag', 'parameter', 'repeat', or 'flatten', then that's how // we're going to handle it. @@ -95,7 +96,7 @@ impl FieldItem { pub fn gen_push_program_options( &self, program_options_ident: &Ident, - ) -> Result { + ) -> Result { match self { Self::Flag(item) => item.gen_push_program_options(program_options_ident), Self::Parameter(item) => item.gen_push_program_options(program_options_ident), @@ -111,7 +112,7 @@ impl FieldItem { &self, subcommands_ident: &Ident, parsed_env: &Ident, - ) -> Result { + ) -> Result { match self { Self::Flag(item) => item.gen_push_subcommands(subcommands_ident, parsed_env), Self::Parameter(item) => item.gen_push_subcommands(subcommands_ident, parsed_env), @@ -121,16 +122,13 @@ impl FieldItem { } } - /// Generate code for a struct initializer for this field + /// Generate code for a struct initializer for this field, reading from conf_context /// /// Returns: /// * a TokenStream for initializer expression, which can use `?` to return errors, /// * a bool which is true if the error type is `Vec` and false if it is /// `InnerError` - pub fn gen_initializer( - &self, - conf_context_ident: &Ident, - ) -> Result<(TokenStream, bool), syn::Error> { + fn gen_initializer(&self, conf_context_ident: &Ident) -> Result<(TokenStream, bool), Error> { match self { Self::Flag(item) => item.gen_initializer(conf_context_ident), Self::Parameter(item) => item.gen_initializer(conf_context_ident), @@ -139,4 +137,305 @@ impl FieldItem { Self::Subcommands(item) => item.gen_initializer(conf_context_ident), } } + + /// Generate code for a struct initializer for this field, reading from conf_context + /// + /// Returns: + /// * a TokenStream for initializer expression, which can use `?` to return errors, + /// * a bool which is true if the error type is `Vec` and false if it is + /// `InnerError` + fn gen_initializer_with_doc_val( + &self, + conf_context_ident: &Ident, + doc_name_ident: &Ident, + doc_val_ident: &Ident, + ) -> Result<(TokenStream, bool), Error> { + match self { + Self::Flag(item) => { + item.gen_initializer_with_doc_val(conf_context_ident, doc_name_ident, doc_val_ident) + } + Self::Parameter(item) => { + item.gen_initializer_with_doc_val(conf_context_ident, doc_name_ident, doc_val_ident) + } + Self::Repeat(item) => { + item.gen_initializer_with_doc_val(conf_context_ident, doc_name_ident, doc_val_ident) + } + Self::Flatten(_item) => unimplemented!("uses a custom match arm"), + Self::Subcommands(_item) => unimplemented!("would have to use a custom match arm"), + } + } + + /// Generate code of the form + /// + /// { + /// fn #field_name(conf_context: &...) -> Result<#field_type, InnerError> { .. } + /// match field_name(...) { + /// Ok(t) => Some(t), + /// Err(err) => { errors.push(err); None }, + /// } + /// } + /// + /// This value can then be assigned to a variable, e.g. + /// + /// let #field_name = #initializer; + /// + /// The code block reads from a conf context and pushes any errors to given errors buffer. + /// + /// Arguments: + /// * conf_context_ident is a variable of type ConfContext which is in scope, which we won't + /// consume + /// * errors_ident is a variable of type mut Vec which is in scope, which we can + /// push to. + pub fn gen_initialize_from_conf_context_and_push_errors( + &self, + conf_context_ident: &Ident, + errors_ident: &Ident, + ) -> Result { + let field_name = self.get_field_name(); + let field_type = self.get_field_type(); + let (initializer, returns_multiple_errors) = self.gen_initializer(conf_context_ident)?; + // The initializer is the portion e.g. + // fn(conf_context: &...) -> Result { ... } + // + // It returns `Result>` if returns_multiple_errors is true, + // otherwise it's Result It is allowed to read + // #conf_context_ident but not modify it We have to put it inside a locally defined + // fn so that it cannot modify the errors buffer etc. + + let (error_type, extend_fn) = if returns_multiple_errors { + ( + quote! { ::std::vec::Vec<::conf::InnerError> }, + quote! { extend }, + ) + } else { + (quote! { ::conf::InnerError }, quote! { push }) + }; + + Ok(quote! { + { + fn #field_name( + #conf_context_ident: &::conf::ConfContext<'_> + ) -> Result<#field_type, #error_type> { + #initializer + } + match #field_name(&#conf_context_ident) { + Ok(val) => Some(val), + Err(errs) => { + #errors_ident.#extend_fn(errs); + None + } + } + } + }) + } + + /// Generate code of the form + /// + /// { + /// fn #field_name(c: &ConfContext, d: ...) -> Result<#field_type, Vec { + /// .. + /// } + /// match #field_name(...) { + /// Ok(val) => Some(val), + /// Err(err) => #errors_ident.extend(err); + /// } + /// } + /// + /// which can then be assigned to a variable, e.g. #field_name + /// + /// The code block reads from a conf context and a value from document, resolves it, and pushes + /// any errors to given errors ident. + /// + /// Arguments: + /// * conf_context_ident is a variable of type ConfContext which is in scope, which we won't + /// consume + /// * doc_name_ident is a variable of type &str, which is the name of the document this value + /// came from + /// * doc_val_ident is a variable of type #serde_type, which was parsed from serde successfully. + /// * errors_ident is a variable of type mut Vec which is in scope, which we can + /// push to. + pub fn gen_initialize_from_conf_context_and_doc_val_and_push_errors( + &self, + conf_context_ident: &Ident, + doc_name_ident: &Ident, + doc_val_ident: &Ident, + errors_ident: &Ident, + ) -> Result { + let field_name = self.get_field_name(); + let field_type = self.get_field_type(); + let serde_type = self.get_serde_type(); + let (initializer, returns_multiple_errors) = + self.gen_initializer_with_doc_val(conf_context_ident, doc_name_ident, doc_val_ident)?; + // The initializer is the portion e.g. + // fn(conf_context: &..., doc_val: T) -> Result { ... } + // + // It returns `Result>` if returns_multiple_errors is true, + // otherwise it's Result + // It is allowed to read #conf_context_ident but not modify it + // We have to put it inside a locally defined + // fn so that it cannot modify the errors buffer etc. + + let (error_type, extend_fn) = if returns_multiple_errors { + ( + quote! { ::std::vec::Vec<::conf::InnerError> }, + quote! { extend }, + ) + } else { + (quote! { ::conf::InnerError }, quote! { push }) + }; + + Ok(quote! { + { + fn #field_name( + #conf_context_ident: &::conf::ConfContext<'_>, + #doc_name_ident: &str, + #doc_val_ident: #serde_type + ) -> Result<#field_type, #error_type> { + #initializer + } + match #field_name(&#conf_context_ident, #doc_name_ident, #doc_val_ident) { + Ok(val) => Some(val), + Err(errs) => { + #errors_ident.#extend_fn(errs); + None + } + } + } + }) + } + + /// Generate (one or more) match arms for iterating a serde MapAccess object. + /// The match takes place on a `&str` representing the struct key. + /// The match arm(s) generated here should identify one or more `&str` + /// and perform initialization appropriately. + /// + /// Returns a TokenStream for the match arm, and a list of `serde_name`'s which + /// we want to advertise in error messages. + /// + /// Arguments: + /// * Ident for conf_serde_context which is in scope and may be consumed + /// * Ident for map_access object which is in scope and may be consumed + /// * Ident for map_access type + /// * Ident for errors buffer which is in scope, to which we may push. + pub fn gen_serde_match_arm( + &self, + ctxt: &Ident, + map_access: &Ident, + map_access_type: &Ident, + errors_ident: &Ident, + ) -> Result<(TokenStream, Vec), Error> { + if self.get_serde_skip() { + return Ok((quote! {}, vec![])); + } + match self { + Self::Flag(_) | Self::Parameter(_) | Self::Repeat(_) => { + self.gen_simple_serde_match_arm(ctxt, map_access, map_access_type, errors_ident) + } + Self::Flatten(item) => { + item.gen_serde_match_arm(ctxt, map_access, map_access_type, errors_ident) + } + Self::Subcommands(item) => { + item.gen_serde_match_arm(ctxt, map_access, map_access_type, errors_ident) + } + } + } + + // A simple serde match arm is used at a "terminal", i.e. a flag, parameter, or repeat field. + // These are fields that represent only a single program option. + // This is a terminal in the sense that we don't recurse further using `DeserializeSeed` and our + // own traits. + // + // The field has controls on what happens in this match arm via: + // * get_serde_name() + // * get_serde_type() + // * gen_initializer_with_doc_val() + fn gen_simple_serde_match_arm( + &self, + ctxt: &Ident, + map_access: &Ident, + map_access_type: &Ident, + errors_ident: &Ident, + ) -> Result<(TokenStream, Vec), Error> { + let field_name = self.get_field_name(); + let field_name_str = field_name.to_string(); + + let serde_name_str = self.get_serde_name(); + let serde_type = self.get_serde_type(); + + let conf_context_ident = Ident::new("__conf_context__", Span::call_site()); + let doc_name_ident = Ident::new("__doc_name__", Span::call_site()); + let doc_val_ident = Ident::new("__doc_val__", Span::call_site()); + let initializer = self.gen_initialize_from_conf_context_and_doc_val_and_push_errors( + &conf_context_ident, + &doc_name_ident, + &doc_val_ident, + errors_ident, + )?; + + let match_arm = quote! { + #serde_name_str => { + if #field_name.is_some() { + #errors_ident.push( + InnerError::serde( + #ctxt.document_name, + #field_name_str, + #map_access_type::Error::duplicate_field(#serde_name_str) + ) + ); + } else { + #field_name = Some(match #map_access.next_value::<#serde_type>() { + Ok(#doc_val_ident) => { + let #conf_context_ident: &::conf::ConfContext = &#ctxt.conf_context; + let #doc_name_ident: &str = #ctxt.document_name; + #initializer + } + Err(__err__) => { + #errors_ident.push( + InnerError::serde( + #ctxt.document_name, + #field_name_str, + __err__ + ) + ); + None + } + }); + } + }, + }; + Ok((match_arm, vec![serde_name_str])) + } + + /// Get the serde name (only when "is_single_option" is true) + fn get_serde_name(&self) -> LitStr { + match self { + Self::Flag(item) => item.get_serde_name(), + Self::Parameter(item) => item.get_serde_name(), + Self::Repeat(item) => item.get_serde_name(), + Self::Flatten(_item) => unimplemented!(), + Self::Subcommands(_item) => unimplemented!(), + } + } + + /// Get the serde type (only when "is_single_option" is true) + fn get_serde_type(&self) -> Type { + match self { + Self::Flag(item) => item.get_serde_type(), + Self::Parameter(item) => item.get_serde_type(), + Self::Repeat(item) => item.get_serde_type(), + Self::Flatten(_item) => unimplemented!(), + Self::Subcommands(_item) => unimplemented!(), + } + } + + /// Get the serde(skip) option + fn get_serde_skip(&self) -> bool { + match self { + Self::Flag(item) => item.get_serde_skip(), + Self::Parameter(item) => item.get_serde_skip(), + Self::Repeat(item) => item.get_serde_skip(), + Self::Flatten(item) => item.get_serde_skip(), + Self::Subcommands(item) => item.get_serde_skip(), + } + } } diff --git a/conf_derive/src/proc_macro_options/field_item/parameter_item.rs b/conf_derive/src/proc_macro_options/field_item/parameter_item.rs index f96b230..48de6aa 100644 --- a/conf_derive/src/proc_macro_options/field_item/parameter_item.rs +++ b/conf_derive/src/proc_macro_options/field_item/parameter_item.rs @@ -1,11 +1,61 @@ use super::StructItem; use crate::util::*; -use proc_macro2::TokenStream; +use proc_macro2::{Span, TokenStream}; use quote::quote; use syn::{ - parse_quote, spanned::Spanned, Error, Expr, Field, Ident, LitBool, LitChar, LitStr, Type, + meta::ParseNestedMeta, parse_quote, spanned::Spanned, token, Error, Expr, Field, Ident, + LitBool, LitChar, LitStr, Type, }; +/// #[conf(serde(...))] options listed on a parameter +pub struct ParameterSerdeItem { + pub rename: Option, + pub skip: bool, + pub use_value_parser: bool, + span: Span, +} + +impl ParameterSerdeItem { + pub fn new(meta: ParseNestedMeta<'_>) -> Result { + let mut result = Self { + rename: None, + skip: false, + use_value_parser: false, + span: meta.input.span(), + }; + + if meta.input.peek(token::Paren) { + meta.parse_nested_meta(|meta| { + let path = meta.path.clone(); + if path.is_ident("rename") { + set_once( + &path, + &mut result.rename, + Some(parse_required_value::(meta)?), + ) + } else if path.is_ident("skip") { + result.skip = true; + Ok(()) + } else if path.is_ident("use_value_parser") { + result.use_value_parser = true; + Ok(()) + } else { + Err(meta.error("unrecognized conf(serde) option")) + } + })?; + } + + Ok(result) + } +} + +impl GetSpan for ParameterSerdeItem { + fn get_span(&self) -> Span { + self.span + } +} + +/// Proc macro annotations parsed from a field of Parameter kind pub struct ParameterItem { field_name: Ident, field_type: Type, @@ -19,18 +69,20 @@ pub struct ParameterItem { env_aliases: Option, default_value: Option, value_parser: Option, + serde: Option, doc_string: Option, } impl ParameterItem { - pub fn new(field: &Field, _struct_item: &StructItem) -> Result { + pub fn new(field: &Field, struct_item: &StructItem) -> Result { let field_name = field .ident .clone() .ok_or_else(|| Error::new(field.span(), "missing identifier"))?; let field_type = field.ty.clone(); let is_optional_type = type_is_option(&field.ty)?; - let allow_hyphen_values = type_is_signed_number(&field.ty); // signed numbers often start with hyphens + // signed numbers often start with hyphens + let allow_hyphen_values = type_is_signed_number(&field.ty); let mut result = Self { field_name, @@ -45,6 +97,7 @@ impl ParameterItem { env_aliases: None, default_value: None, value_parser: None, + serde: None, doc_string: None, }; @@ -109,6 +162,12 @@ impl ParameterItem { .unwrap_or(LitBool::new(true, path.span())), ), ) + } else if path.is_ident("serde") { + set_once( + &path, + &mut result.serde, + Some(ParameterSerdeItem::new(meta)?), + ) } else { Err(meta.error("unrecognized conf parameter option")) } @@ -121,8 +180,14 @@ impl ParameterItem { && result.long_switch.is_none() && result.env_name.is_none() && result.default_value.is_none() + && struct_item.serde.is_none() { - return Err(Error::new(field.span(), "There is no way for the user to give this parameter a value. Trying using #[conf(short)], #[conf(long)], or #[conf(env)] to specify a switch or an env associated to this value, or specify a default value.")); + return Err(Error::new( + field.span(), + "There is no way for the user to give this parameter a value. \ + Trying using #[arg(short)], #[arg(long)], or #[arg(env)] to specify a switch \ + or an env associated to this value, or specify a default value.", + )); } if result.long_switch.is_none() @@ -132,7 +197,11 @@ impl ParameterItem { .map(LitStrArray::is_empty) .unwrap_or(true) { - return Err(Error::new(field.span(), "Setting aliases without setting a long-switch is an error, make one of the aliases the primary switch name.")); + return Err(Error::new( + field.span(), + "Setting aliases without setting a long-switch is an error, \ + make one of the aliases the primary switch name.", + )); } if result.env_name.is_none() @@ -142,7 +211,11 @@ impl ParameterItem { .map(LitStrArray::is_empty) .unwrap_or(true) { - return Err(Error::new(field.span(), "Setting env_aliases without setting an env is an error, make one of the aliases the primary env.")); + return Err(Error::new( + field.span(), + "Setting env_aliases without setting an env is an error, \ + make one of the aliases the primary env.", + )); } Ok(result) @@ -160,6 +233,31 @@ impl ParameterItem { self.default_value.as_ref() } + pub fn get_serde_name(&self) -> LitStr { + self.serde + .as_ref() + .and_then(|serde| serde.rename.clone()) + .unwrap_or_else(|| LitStr::new(&self.field_name.to_string(), self.field_name.span())) + } + + pub fn get_serde_type(&self) -> Type { + let use_value_parser = self + .serde + .as_ref() + .map(|serde| serde.use_value_parser) + .unwrap_or(false); + + if use_value_parser { + parse_quote! { ::std::string::String } + } else { + self.field_type.clone() + } + } + + pub fn get_serde_skip(&self) -> bool { + self.serde.as_ref().map(|serde| serde.skip).unwrap_or(false) + } + pub fn gen_push_program_options( &self, program_options_ident: &Ident, @@ -205,62 +303,157 @@ impl ParameterItem { Ok(quote! {}) } - pub fn gen_initializer( + fn get_value_parser(&self) -> Expr { + // Value parser is FromStr::from_str if not specified + self.value_parser + .clone() + .unwrap_or_else(|| parse_quote! { std::str::FromStr::from_str }) + } + + fn gen_initializer_helper( &self, conf_context_ident: &Ident, + if_no_conf_context_val: Option, + before_value_parser: Option, ) -> Result<(TokenStream, bool), syn::Error> { let field_type = &self.field_type; let id = self.field_name.to_string(); - // Value parser is FromStr::from_str if not specified - let value_parser: Expr = self - .value_parser - .clone() - .unwrap_or_else(|| parse_quote! { ::core::str::FromStr::from_str }); + let value_parser = self.get_value_parser(); + + // Code gen is slightly different if the field type is Option + // Inner_type is T in that case, or just field_type otherwise. + // Value parser will produce inner_type. + let inner_type = self.is_optional_type.as_ref().unwrap_or(field_type); + + // If the conf context doesn't find a value, its an error if this field is required, + // and Ok(None) if this field is optional. + // This gets overrided when serde provides a value. + let if_no_conf_context_val = if_no_conf_context_val.unwrap_or_else(|| { + if self.is_optional_type.is_some() { + quote! { return Ok(None); } + } else { + quote! { return Err(#conf_context_ident.missing_required_parameter_error(opt)); } + } + }); + + // Value parser produces #inner_type, so we have to massage a success result to #field_type + let value_parser_ok_arm = if self.is_optional_type.is_some() { + quote! { Ok(t) => Ok(Some(t)), } + } else { + quote! { Ok(t) => Ok(t), } + }; - // Code gen is slightly different if the field type is Optional // The part around value parser needs to be very simple if we want type inference to work // We also stick the user-provided expression inside a function to prevent it from mutating // anything in the surrounding scope. // But we are reading conf_context_ident from our caller's scope, outside of the // user-provided expression - Ok(( - if let Some(inner_type) = self.is_optional_type.as_ref() { - quote! { - { - fn value_parser(__arg__: &str) -> Result<#inner_type, impl ::core::fmt::Display> { - #value_parser(__arg__) - } - - let (maybe_val, opt): (Option<_>, &conf::ProgramOption) = #conf_context_ident.get_string_opt(#id)?; - match maybe_val { - Some((value_source, val_str)) => { - match value_parser(val_str) { - Ok(t) => Ok(Some(t)), - Err(err) => Err(::conf::InnerError::invalid_value(value_source, val_str, opt, err)), - } - }, - None => Ok(None), - } - } - } - } else { - quote! { - { - fn value_parser(__arg__: &str) -> Result<#field_type, impl ::core::fmt::Display> { - #value_parser(__arg__) - } - - let (maybe_val, opt): (Option<_>, &::conf::ProgramOption) = #conf_context_ident.get_string_opt(#id)?; - let (value_source, val_str): (::conf::ConfValueSource<&str>, &str) = maybe_val.ok_or_else(|| #conf_context_ident.missing_required_parameter_error(opt))?; - match value_parser(val_str) { - Ok(t) => Ok(t), - Err(err) => Err(::conf::InnerError::invalid_value(value_source, val_str, opt, err)), - } - } - } - }, - false, - )) + let initializer = quote! { + { + fn __value_parser__( + __arg__: &str + ) -> Result<#inner_type, impl ::core::fmt::Display> { + #value_parser(__arg__) + } + + use ::conf::{ConfValueSource, ProgramOption, InnerError}; + + let (maybe_val, opt): (Option<_>, &ProgramOption) + = #conf_context_ident.get_string_opt(#id)?; + let (value_source, val_str): (ConfValueSource<&str>, &str) + = if let Some(val) = maybe_val { + val + } else { + #if_no_conf_context_val + }; + #before_value_parser + match __value_parser__(val_str) { + #value_parser_ok_arm + Err(err) => Err( + InnerError::invalid_value( + value_source, + val_str, + opt, + err + ) + ), + } + } + }; + Ok((initializer, false)) + } + + pub fn gen_initializer( + &self, + conf_context_ident: &Ident, + ) -> Result<(TokenStream, bool), syn::Error> { + self.gen_initializer_helper(conf_context_ident, None, None) + } + + // Gen initializer with a provided document value. + // + // Like gen_initializer, but in this case, serde has provided a value for this field. + // The value is a variable of type #serde_type and the identifier is #doc_val. + // + // Here, we should return #doc_val if the basic initializer would have produced no value, + // or would have produced the default_value string, because the document is higher priority. + // But the document value should be ignored if args or env is the value source. + pub fn gen_initializer_with_doc_val( + &self, + conf_context_ident: &Ident, + doc_name: &Ident, + doc_val: &Ident, + ) -> Result<(TokenStream, bool), Error> { + let use_value_parser = self + .serde + .as_ref() + .map(|serde| serde.use_value_parser) + .unwrap_or(false); + + if use_value_parser { + // When use_value_parser is true, then #doc_val has type String. + // To pick this value for the field, we have to set value_source and val_str + // to indicate that we are selecting the document value. + let if_no_conf_context_val = quote! { + (ConfValueSource::Document(#doc_name), #doc_val.as_str()) + }; + + // When the value source is a default, but we have a doc val, + // we should prefer the doc val. + let before_value_parser = quote! { + let (value_source, val_str) = if value_source.is_default() { + #if_no_conf_context_val + } else { + (value_source, val_str) + }; + }; + + self.gen_initializer_helper( + conf_context_ident, + Some(if_no_conf_context_val), + Some(before_value_parser), + ) + } else { + // When use_value_parser is false, then #doc_val has type #field_type. + // To pick this value for the field, we just return it. + let if_no_conf_context_val = quote! { + return Ok(#doc_val); + }; + + // After we have a conf context value, but before we run the value parser, + // check if the value that was obtained should be lower priority than the doc val. + // If so then early return in the same way. + let before_value_parser = quote! { + if value_source.is_default() { + #if_no_conf_context_val + } + }; + self.gen_initializer_helper( + conf_context_ident, + Some(if_no_conf_context_val), + Some(before_value_parser), + ) + } } } diff --git a/conf_derive/src/proc_macro_options/field_item/repeat_item.rs b/conf_derive/src/proc_macro_options/field_item/repeat_item.rs index 072e9bc..989d961 100644 --- a/conf_derive/src/proc_macro_options/field_item/repeat_item.rs +++ b/conf_derive/src/proc_macro_options/field_item/repeat_item.rs @@ -1,11 +1,61 @@ use super::StructItem; use crate::util::*; -use proc_macro2::TokenStream; +use proc_macro2::{Span, TokenStream}; use quote::quote; use syn::{ - parse_quote, spanned::Spanned, Error, Expr, Field, Ident, LitBool, LitChar, LitStr, Type, + meta::ParseNestedMeta, parse_quote, spanned::Spanned, token, Error, Expr, Field, Ident, + LitBool, LitChar, LitStr, Type, }; +/// #[conf(serde(...))] options listed on a field of Repeat kind +pub struct RepeatSerdeItem { + pub rename: Option, + pub skip: bool, + pub use_value_parser: bool, + span: Span, +} + +impl RepeatSerdeItem { + pub fn new(meta: ParseNestedMeta<'_>) -> Result { + let mut result = Self { + rename: None, + skip: false, + use_value_parser: false, + span: meta.input.span(), + }; + + if meta.input.peek(token::Paren) { + meta.parse_nested_meta(|meta| { + let path = meta.path.clone(); + if path.is_ident("rename") { + set_once( + &path, + &mut result.rename, + Some(parse_required_value::(meta)?), + ) + } else if path.is_ident("skip") { + result.skip = true; + Ok(()) + } else if path.is_ident("use_value_parser") { + result.use_value_parser = true; + Ok(()) + } else { + Err(meta.error("unrecognized conf(serde) option")) + } + })?; + } + + Ok(result) + } +} + +impl GetSpan for RepeatSerdeItem { + fn get_span(&self) -> Span { + self.span + } +} + +/// Proc macro annotations parsed from a field of Repeat kind pub struct RepeatItem { field_name: Ident, field_type: Type, // This is needed to help with type inference in code gen @@ -18,6 +68,7 @@ pub struct RepeatItem { value_parser: Option, env_delimiter: Option, no_env_delimiter: bool, + serde: Option, description: Option, } @@ -49,6 +100,7 @@ impl RepeatItem { value_parser: None, env_delimiter: None, no_env_delimiter: false, + serde: None, description: None, }; @@ -112,6 +164,8 @@ impl RepeatItem { .unwrap_or(LitBool::new(true, path.span())), ), ) + } else if path.is_ident("serde") { + set_once(&path, &mut result.serde, Some(RepeatSerdeItem::new(meta)?)) } else { Err(meta.error("unrecognized conf repeat option")) } @@ -147,7 +201,11 @@ impl RepeatItem { .map(LitStrArray::is_empty) .unwrap_or(true) { - return Err(Error::new(field.span(), "Setting aliases without setting a long-switch is an error, make one of the aliases the primary switch name.")); + return Err(Error::new( + field.span(), + "Setting aliases without setting a long-switch is an error, \ + make one of the aliases the primary switch name.", + )); } if result.env_name.is_none() @@ -157,7 +215,11 @@ impl RepeatItem { .map(LitStrArray::is_empty) .unwrap_or(true) { - return Err(Error::new(field.span(), "Setting env_aliases without setting an env is an error, make one of the aliases the primary env.")); + return Err(Error::new( + field.span(), + "Setting env_aliases without setting an env is an error, \ + make one of the aliases the primary env.", + )); } Ok(result) @@ -171,6 +233,36 @@ impl RepeatItem { self.field_type.clone() } + pub fn get_serde_name(&self) -> LitStr { + self.serde + .as_ref() + .and_then(|serde| serde.rename.clone()) + .unwrap_or_else(|| LitStr::new(&self.field_name.to_string(), self.field_name.span())) + } + + pub fn get_serde_type(&self) -> Type { + let use_value_parser = self + .serde + .as_ref() + .map(|serde| serde.use_value_parser) + .unwrap_or(false); + + if use_value_parser { + parse_quote! { ::std::vec::Vec<::std::string::String> } + } else { + self.field_type.clone() + } + } + + pub fn get_serde_skip(&self) -> bool { + self.serde.as_ref().map(|serde| serde.skip).unwrap_or(false) + } + + /// Generate a routine that pushes a ::conf::ProgramOption corresponding to + /// this field, onto a mut Vec that is in scope. + /// + /// Arguments: + /// * program_options_ident is the ident of this buffer of ProgramOption to push to. pub fn gen_push_program_options( &self, program_options_ident: &Ident, @@ -188,19 +280,19 @@ impl RepeatItem { let secret = quote_opt(&self.secret); Ok(quote! { - #program_options_ident.push(conf::ProgramOption { - id: #id.into(), - parse_type: conf::ParseType::Repeat, - description: #description, - short_form: None, - long_form: #long_form, - aliases: vec![#aliases], - env_form: #env_form, - env_aliases: vec![#env_aliases], - default_value: None, - is_required: false, - allow_hyphen_values: #allow_hyphen_values, - secret: #secret, + #program_options_ident.push(::conf::ProgramOption { + id: #id.into(), + parse_type: ::conf::ParseType::Repeat, + description: #description, + short_form: None, + long_form: #long_form, + aliases: vec![#aliases], + env_form: #env_form, + env_aliases: vec![#env_aliases], + default_value: None, + is_required: false, + allow_hyphen_values: #allow_hyphen_values, + secret: #secret, }); }) } @@ -213,14 +305,8 @@ impl RepeatItem { Ok(quote! {}) } - pub fn gen_initializer( - &self, - conf_context_ident: &Ident, - ) -> Result<(TokenStream, bool), syn::Error> { - let field_type = &self.field_type; - let id = self.field_name.to_string(); - - let delimiter = quote_opt(&if self.no_env_delimiter { + fn get_delimiter(&self) -> TokenStream { + quote_opt(&if self.no_env_delimiter { None } else { Some( @@ -228,43 +314,131 @@ impl RepeatItem { .clone() .unwrap_or_else(|| LitChar::new(',', self.field_name.span())), ) - }); + }) + } + fn get_value_parser(&self) -> Expr { // Value parser is FromStr::from_str if not specified - let value_parser = self - .value_parser + self.value_parser .clone() - .unwrap_or_else(|| parse_quote! { std::str::FromStr::from_str }); + .unwrap_or_else(|| parse_quote! { std::str::FromStr::from_str }) + } + + fn gen_initializer_helper( + &self, + conf_context_ident: &Ident, + before_value_parser: Option, + ) -> Result<(TokenStream, bool), syn::Error> { + let field_type = &self.field_type; + let id = self.field_name.to_string(); + + let delimiter = self.get_delimiter(); + let value_parser = self.get_value_parser(); // Note: We can't use rust into_iter, collect, map_err because sometimes it messes with type - // inference around the value parser Note: The line `let mut result: #field_type = - // Default::default();` is expected to be default initializing a Vec. If it fails - // because the user put another funky type there, imo this should not really be supported. - // It's more compelling to make the value_parser option easier to use (easier type - // inference) than to support user-defined containers here, and try to use - // `.collect` etc. directly into their container. The user's code can do .iter().collect() - // after our code runs if they want. - Ok(( - quote! { - || -> Result<#field_type, Vec> { - let (value_source, strs, opt): (conf::ConfValueSource<&str>, Vec<&str>, &conf::ProgramOption) = #conf_context_ident.get_repeat_opt(#id, #delimiter).map_err(|err| vec![err])?; - let mut result: #field_type = Default::default(); - let mut errors = Vec::::new(); - result.reserve(strs.len()); - for val_str in strs { - match #value_parser(val_str) { - Ok(val) => result.push(val), - Err(err) => errors.push(conf::InnerError::invalid_value(value_source.clone(), val_str, opt, err.to_string()).into()), - } - } - if errors.is_empty() { - Ok(result) - } else { - Err(errors) - } - }() - }, - true, - )) + // inference around the value parser + // + // Note: The line `let mut result: #field_type = Default::default();` + // is expected to be default initializing a Vec. + // It can't be `#field_type::default()` because it requires turbofish syntax. + // + // If it fails because the user put another funky type there, imo this should not really be + // supported. It's more compelling to make the value_parser option easier to use + // (easier type inference) than to support user-defined containers here, and try to + // use `.collect` etc. directly into their container. The user's code can do + // .iter().collect() after our code runs if they want. + let initializer = quote! { + { + fn __value_parser__( + __arg__: &str + ) -> Result<<#field_type as ::conf::InnerTypeHelper>::Ty, impl ::core::fmt::Display> { + #value_parser(__arg__) + } + + use ::conf::{ConfValueSource, ProgramOption, InnerError}; + use ::std::vec::Vec; + + let (value_source, strs, opt): (ConfValueSource<&str>, Vec<&str>, &ProgramOption) + = #conf_context_ident.get_repeat_opt(#id, #delimiter).map_err(|err| vec![err])?; + + #before_value_parser + + let mut result: #field_type = Default::default(); + let mut errors = Vec::::new(); + result.reserve(strs.len()); + for val_str in strs { + match __value_parser__(val_str) { + Ok(val) => result.push(val), + Err(err) => errors.push( + InnerError::invalid_value( + value_source.clone(), + val_str, + opt, + err.to_string() + ) + ), + } + } + if errors.is_empty() { + Ok(result) + } else { + Err(errors) + } + } + }; + Ok((initializer, true)) + } + + // Gen initializer + // + // Create an expression that returns initialized #field_type value, or errors. + pub fn gen_initializer( + &self, + conf_context_ident: &Ident, + ) -> Result<(TokenStream, bool), syn::Error> { + self.gen_initializer_helper(conf_context_ident, None) + } + + // Gen initializer with a provided doc val + // + // This should work similar to gen_initializer, but if the conf context produces a default + // value, we should return the doc_val instead because it has higher priority than default, + // but lower than args and env. + pub fn gen_initializer_with_doc_val( + &self, + conf_context_ident: &Ident, + doc_name: &Ident, + doc_val: &Ident, + ) -> Result<(TokenStream, bool), Error> { + let use_value_parser = self + .serde + .as_ref() + .map(|serde| serde.use_value_parser) + .unwrap_or(false); + + if use_value_parser { + // When use_value_parser is enabled, the behavior is, if conf_context produced a default + // value, we should overwrite it with the document value. `val_strs` is a `Vec<&str>`, + // and #doc_val is a `Vec`. + let before_value_parser = quote! { + let (value_source, strs) = if value_source.is_default() { + (ConfValueSource::Document(#doc_name), #doc_val.iter().map(String::as_str).collect()) + } else { + (value_source, strs) + }; + }; + + self.gen_initializer_helper(conf_context_ident, Some(before_value_parser)) + } else { + // When use_value_parser is not enabled, the behavior is, if conf context produced a + // default value, we should instead simply return the doc value. + let before_value_parser = quote! { + if value_source.is_default() { + return Ok(#doc_val); + } + }; + + self.gen_initializer_helper(conf_context_ident, Some(before_value_parser)) + } } } diff --git a/conf_derive/src/proc_macro_options/field_item/subcommands_item.rs b/conf_derive/src/proc_macro_options/field_item/subcommands_item.rs index 22b7055..7c6cd07 100644 --- a/conf_derive/src/proc_macro_options/field_item/subcommands_item.rs +++ b/conf_derive/src/proc_macro_options/field_item/subcommands_item.rs @@ -1,14 +1,51 @@ use super::StructItem; use crate::util::*; -use proc_macro2::TokenStream; +use proc_macro2::{Span, TokenStream}; use quote::quote; -use syn::{spanned::Spanned, Error, Field, Ident, Type}; +use syn::{meta::ParseNestedMeta, spanned::Spanned, token, Error, Field, Ident, LitStr, Type}; +/// #[conf(serde(...))] options listed on a field of Subcommands kind +pub struct SubcommandsSerdeItem { + pub skip: bool, + span: Span, +} + +impl SubcommandsSerdeItem { + pub fn new(meta: ParseNestedMeta<'_>) -> Result { + let mut result = Self { + skip: false, + span: meta.input.span(), + }; + + if meta.input.peek(token::Paren) { + meta.parse_nested_meta(|meta| { + let path = meta.path.clone(); + if path.is_ident("skip") { + result.skip = true; + Ok(()) + } else { + Err(meta.error("unrecognized conf(serde) option")) + } + })?; + } + + Ok(result) + } +} + +impl GetSpan for SubcommandsSerdeItem { + fn get_span(&self) -> Span { + self.span + } +} + +/// #[conf(...)] options listed on a field of Subcommands kind pub struct SubcommandsItem { struct_name: Ident, field_name: Ident, field_type: Type, is_optional_type: Option, + serde: Option, doc_string: Option, } @@ -27,6 +64,7 @@ impl SubcommandsItem { field_name, field_type, is_optional_type, + serde: None, doc_string: None, }; @@ -37,6 +75,12 @@ impl SubcommandsItem { let path = meta.path.clone(); if path.is_ident("subcommands") { Ok(()) + } else if path.is_ident("serde") { + set_once( + &path, + &mut result.serde, + Some(SubcommandsSerdeItem::new(meta)?), + ) } else { Err(meta.error("unrecognized conf subcommands option")) } @@ -55,6 +99,10 @@ impl SubcommandsItem { self.field_type.clone() } + pub fn get_serde_skip(&self) -> bool { + self.serde.as_ref().map(|serde| serde.skip).unwrap_or(false) + } + // Subcommands fields don't add any program options to the conf structure. pub fn gen_push_program_options( &self, @@ -79,14 +127,19 @@ impl SubcommandsItem { // but it's not clear that it's useful. Ok(quote! { if !#parsers_ident.is_empty() { - panic!(#panic_message); + panic!(#panic_message); } - #parsers_ident.extend(<#inner_type as ::conf::Subcommands>::get_parsers(#parsed_env_ident)?); + #parsers_ident.extend( + <#inner_type as ::conf::Subcommands>::get_parsers(#parsed_env_ident)? + ); }) } - // Body of a function taking a &ConfContext returning Result<#field_type, - // Vec<::conf::InnerError>> + // Body of a function taking a &ConfContext and returning + // Result<#field_type, Vec<::conf::InnerError>> + // + // Arguments: + // * conf_context_ident is the identifier of a &ConfContext variable in scope pub fn gen_initializer( &self, conf_context_ident: &Ident, @@ -95,23 +148,94 @@ impl SubcommandsItem { let field_name = self.field_name.to_string(); let field_type = &self.field_type; - let result = if let Some(inner_type) = self.is_optional_type.as_ref() { + let initializer = if let Some(inner_type) = self.is_optional_type.as_ref() { quote! { - if let Some((name, conf_context)) = #conf_context_ident.for_subcommand() { - Ok(Some(<#inner_type as ::conf::Subcommands>::from_conf_context(name, conf_context)?)) - } else { - Ok(None) - } + Ok(if let Some((name, conf_context)) = #conf_context_ident.for_subcommand() { + Some( + <#inner_type as ::conf::Subcommands>::from_conf_context(name, conf_context)? + ) + } else { + None + }) } } else { quote! { - let Some((name, conf_context)) = #conf_context_ident.for_subcommand() else { - return Err(vec![ ::conf::InnerError::missing_required_subcommand( #struct_name, #field_name, <#field_type as ::conf::Subcommands>::get_subcommand_names() ) ]); - }; - Ok(<#field_type as ::conf::Subcommands>::from_conf_context(name, conf_context)?) + use ::conf::{InnerError, Subcommands}; + let Some((name, conf_context)) = #conf_context_ident.for_subcommand() else { + return Err(vec![ + InnerError::missing_required_subcommand( + #struct_name, + #field_name, + <#field_type as Subcommands>::get_subcommand_names() + ) + ]); + }; + <#field_type as Subcommands>::from_conf_context(name, conf_context) } }; - Ok((result, true)) + Ok((initializer, true)) + } + + // A serde match arm for the subcommand. + pub fn gen_serde_match_arm( + &self, + ctxt: &Ident, + map_access: &Ident, + map_access_type: &Ident, + errors_ident: &Ident, + ) -> Result<(TokenStream, Vec), Error> { + let field_name = self.get_field_name(); + + let inner_type = self.is_optional_type.as_ref().unwrap_or(&self.field_type); + + let val_expr = if self.is_optional_type.is_some() { + quote! { Some(__val__) } + } else { + quote! { __val__ } + }; + + // For a subcommand to be active, it must appear in the args, we can't activate a subcommand + // based only on the conf file. + // We'd like to allow that each subcommand could have its own section in the conf file, and + // sections that don't correspond to the currently selected one aren't an error. + // Multiple subcommands could have the same section as well (if their serde name were + // equal). + // + // So the test is: + // * If this key matches any serde_name for any of the subcommands, enter this match arm. + // * Check if the conf context has a subcommand name, and if that matches any of these + // commands. If not, then we ignore this serde value. + // * Otherwise, we are attempting to recurse into the subcommand. + let match_arm = quote! { + key__ if <#inner_type as SubcommandsSerde>::SERDE_NAMES.iter().any(|(_c, s)| *s == key__) => { + let Some((command_name, conf_context_serde)) = #ctxt.for_subcommand() else { continue }; + + let Some((static_command_name, static_serde_name)) = <#inner_type as SubcommandsSerde>::SERDE_NAMES.iter().find(|(c, s)| *c == command_name && *s == key__) else { continue }; + + if #field_name.is_some() { + #errors_ident.push( + InnerError::serde( + #ctxt.document_name, + static_command_name, + #map_access_type::Error::duplicate_field(static_serde_name) + ) + ); + } else { + #field_name = Some(match <#inner_type as SubcommandsSerde>::from_conf_serde_context(&command_name, conf_context_serde, &mut #map_access) { + Ok(__val__) => { + Some(#val_expr) + }, + Err(__errs__) => { + #errors_ident.extend(__errs__); + None + } + }); + } + }, + }; + // We don't know the SERDE_NAMES as string literals in this proc_macro, they are only in the + // proc_macro invocation for the enum. + Ok((match_arm, vec![])) } } diff --git a/conf_derive/src/proc_macro_options/mod.rs b/conf_derive/src/proc_macro_options/mod.rs index d6a32d4..3c62e4f 100644 --- a/conf_derive/src/proc_macro_options/mod.rs +++ b/conf_derive/src/proc_macro_options/mod.rs @@ -1,24 +1,732 @@ -//! These are helper structures which: +//! GenConfStruct helps with parsing syn data for a Conf Struct, and generating Conf trait +//! implementation bits. +//! +//! This module also provides StructItem, FieldItem helper structures which: //! * Parse the `#[conf(...)]` attributes that appear on different types of items //! * Store the results and make them easily available //! * Assist with subsequent codegen -use syn::FieldsNamed; +use crate::util::{make_lifetime, prepend_generic_lifetimes}; +use proc_macro2::{Span, TokenStream}; +use quote::quote; +use syn::{parse_quote, Attribute, Error, FieldsNamed, Generics, Ident, LitStr, Token, Type}; mod field_item; +use field_item::FieldItem; + mod struct_item; +use struct_item::StructItem; + +/// Helper which generates individual functions related to `#[derive(Conf)]` +/// on a struct. +/// +/// Calling "new" parses all the proc macro attributes for struct and fields. +/// Calling individual functions returns code gen. +pub struct GenConfStruct { + struct_item: StructItem, + fields: Vec, +} + +impl GenConfStruct { + /// Parse syn data for a struct with derive(Conf) on it + pub fn new(ident: &Ident, attrs: &[Attribute], fields: &FieldsNamed) -> Result { + let struct_item = StructItem::new(ident, attrs)?; + let fields = fields + .named + .iter() + .map(|f| FieldItem::new(f, &struct_item)) + .collect::, Error>>()?; + Ok(Self { + struct_item, + fields, + }) + } + + /// Generate an impl Conf block for this struct + /// + /// Takes generics associated to the struct. + pub fn gen_conf_impl(&self, generics: &Generics) -> Result { + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + + let ident = self.struct_item.get_ident(); + let conf_fns = vec![ + self.get_parser_config_impl()?, + self.get_program_options_impl()?, + self.get_subcommands_impl()?, + self.from_conf_context_impl()?, + self.get_name_impl()?, + ]; + + Ok(quote! { + #[automatically_derived] + #[allow( + unused_qualifications, + )] + impl #impl_generics ::conf::Conf for #ident #ty_generics #where_clause { + #(#conf_fns)* + } + }) + } + + /// Generate Conf::get_name implementation + fn get_name_impl(&self) -> Result { + let struct_name = self.struct_item.get_ident().to_string(); + + Ok(quote! { + fn get_name() -> &'static str { + #struct_name + } + }) + } + + /// Generate Conf::get_parser_config implementation + fn get_parser_config_impl(&self) -> Result { + // To implement Conf::get_parser_config, we need to get a ParserConfig object + // for this struct, (top-level config essentially). + let parser_config = self.struct_item.gen_parser_config()?; + + Ok(quote! { + fn get_parser_config() -> Result<::conf::ParserConfig, ::conf::Error> { + let parser_config = #parser_config; + + Ok(parser_config) + } + }) + } + + /// Generate Conf::get_program_options implementation + fn get_program_options_impl(&self) -> Result { + // To implement Conf::get_program_options, we need to + // get all the program options for our constituents. To do this, we create + // an ident for the list of program options, which is going to be Vec. + // Then we pass that ident to every constitutent field, and aggregate all their code gen. + let program_options_ident = Ident::new("__program_options__", Span::call_site()); + let fields_push_program_options: Vec = self + .fields + .iter() + .map(|field| field.gen_push_program_options(&program_options_ident)) + .collect::, Error>>()?; + + // To implement #[conf(env_prefix="ACME_")] on a struct (rather than on a flattened field), + // the code gen associated to the struct needs to be able to add its own prefixing during + // get_program_options and during from_conf_context. + // To do this, we allow the struct_item to "post-process" the Vec, (to add a + // prefix to them all) and to "pre-process" the ConfContext (to add a matching prefix to + // that before it is used) Note: The preprocessing no longer does anything since we switched + // to using id's like clap does. + let struct_post_process_program_options = self + .struct_item + .gen_post_process_program_options(&program_options_ident)?; + + // Note: fields_push_program_options is allowed to early return with ? on an error + Ok(quote! { + fn get_program_options() -> Result<&'static [::conf::ProgramOption], ::conf::Error> { + static CACHED: ::std::sync::OnceLock> = ::std::sync::OnceLock::new(); + + if CACHED.get().is_none() { + let mut #program_options_ident = vec![]; + + #(#fields_push_program_options)* + + #struct_post_process_program_options + + let _ = CACHED.set(#program_options_ident); + } + + let cached = CACHED.get().unwrap(); + + Ok(cached.as_ref()) + } + }) + } + + /// Generate Conf::get_subcommands implementation + fn get_subcommands_impl(&self) -> Result { + let parsers_ident = Ident::new("__parsers__", Span::call_site()); + let parsed_env_ident = Ident::new("__parsed_env__", Span::call_site()); + let fields_push_subcommands: Vec = self + .fields + .iter() + .map(|field| field.gen_push_subcommands(&parsers_ident, &parsed_env_ident)) + .collect::, Error>>()?; + + Ok(quote! { + fn get_subcommands(#parsed_env_ident: &::conf::ParsedEnv) -> Result, ::conf::Error> { + let mut #parsers_ident = vec![]; + + #(#fields_push_subcommands)* + + Ok(#parsers_ident) + } + }) + } + + // Generate Conf::from_conf_context implementation + #[allow(clippy::wrong_self_convention)] + fn from_conf_context_impl(&self) -> Result { + // To implement Conf::from_conf_context, we need to take a conf context, + // and then return Ok(Self { ... }). For each constituent field, we need it + // to generate code to initialize itself properly. We pass the ConfContext ident + // to each constituent field, and then aggregate all their code gen. + // Their code-gen is allowed to use `?` or `return Err(...)` to early return, + // but we still need to aggregate all the errors. Sample code gen is like. + // + // struct Sample { + // a: i32, + // b: i64, + // } + // + // from_conf_context(conf_context: conf::ConfContext) -> Result> + // { let conf_context = #preprocess_conf_context; + // let mut errors = Vec::::new(); + // + // fn a(conf_context: &conf::ConfContext) -> Result { + // .. + // } + // let a = match a(&conf_context) { + // Ok(val) => Some(val), + // Err(err) => { + // errors.push(err); + // None + // } + // }; + // + // fn b(conf_context: &conf::ConfContext) -> Result { + // .. + // } + // let b = match b(&conf_context) { + // Ok(val) => Some(val), + // Err(err) => { + // errors.push(err); + // None + // } + // }; + // + // let return_value = match (a, b) { + // (Some(a), Some(b)) => Ok(Self { + // a, + // b, + // }), + // _ => Err(errors), + // }?; + // + // validation_predicate(&return_value).map_err(|err| { + // vec![conf::InnerError::validation(&conf_context.id, err)] + // })?; + // + // Ok(return_value) + // } + // + // The list of let a, let b... is called #initializations + // The match (a,b, ...) { ... } is called #return_value + // The validation_predicate(...) part is called #apply_validation_predicate + let conf_context_ident = Ident::new("__conf_context__", Span::call_site()); + let errors_ident = Ident::new("__errors__", Span::call_site()); + + // For each field, intialize a local variable with Option which is some if it worked and + // None if there were errors. Push all errors into #errors_ident. + let initializations: Vec = self + .fields + .iter() + .map(|field| -> Result { + let field_name = field.get_field_name(); + let initializer = field.gen_initialize_from_conf_context_and_push_errors( + &conf_context_ident, + &errors_ident, + )?; + Ok(quote! { + let #field_name = #initializer; + }) + }) + .collect::, Error>>()?; + + let gather_and_validate = self.gather_and_validate(&conf_context_ident, &errors_ident)?; + + Ok(quote! { + fn from_conf_context<'a>(#conf_context_ident: ::conf::ConfContext<'a>) -> Result> { + let mut #errors_ident = Vec::<::conf::InnerError>::new(); + + #(#initializations)* + + #gather_and_validate + } + }) + } + + // Generate a routine which gathers struct fields + // (local variables represented as Option<#field type>). + // + // Bails if we can't produce a struct, or if the constituted struct fails validation. + // Otherwise returns it. + // + // Arguments: + // * conf_context_ident: The identifier of a ConfContext variable that we can use (and consume) + // * errors_ident: the identifier of a `mut Vec` buffer variable which is in scope. + fn gather_and_validate( + &self, + conf_context_ident: &Ident, + errors_ident: &Ident, + ) -> Result { + let struct_ident = self.struct_item.get_ident(); + + let field_names: Vec<&Ident> = self + .fields + .iter() + .map(|field| field.get_field_name()) + .collect(); + + let return_value: TokenStream = quote! { + match (#(#field_names),*) { + (#(Some(#field_names)),*) => #struct_ident { #(#field_names),* }, + _ => panic!("Internal error: no errors encountered but struct was incomplete") + } + }; + + let instance_ident = Ident::new("__instance__", Span::call_site()); + + let validation_routine = self.struct_item.gen_validation_routine( + &instance_ident, + conf_context_ident, + &self.fields, + )?; + + Ok(quote! { + if !#errors_ident.is_empty() { + return Err(#errors_ident); + } + + let return_value = #return_value; + + fn validation<'ctxctx>(#instance_ident: & #struct_ident, #conf_context_ident: ::conf::ConfContext<'ctxctx>) -> Result<(), Vec<::conf::InnerError>> { + #validation_routine + } + + validation(&return_value, #conf_context_ident)?; + + Ok(return_value) + }) + } + + /// Generate an impl ConfSerde block for this struct (if requested via attributes) + /// Also, the requisite DeserializeSeed impl's and such. + /// + /// Takes generics associated to this struct. + pub fn maybe_gen_conf_serde_impl( + &self, + generics: &Generics, + ) -> Result, Error> { + // If serde is not requested, fugeddaboutit + if self.struct_item.serde.is_none() { + return Ok(None); + }; + + // To generate a ConfSerde impl on S, we need to designate a Seed, + // which will implement serde::DeserializeSeed. + // + // The Seed type can't exist within conf crate. If it did, then the + // type is an conf, and the trait is in serde, so the impl would have to + // be in conf, due to the orphan rules. + // But the trait implementations need to be code-genned because they depend + // on the user-defined type, and are going to live in the user's crate. + // So, the Seed type needs to be a new-type of some kind, defined by this proc macro, + // in the user's crate. + // + // In order to hide it, we define it in a "private module", and put the impl's there too: + // const _: () = { ... }; + // + // The Seed is just a newtype around ConfSerdeContext, which is what we would + // have used if not for orphan rules. + // (But actually, it needs phantom data pointing back to the user-type as well, + // so that we can implement DeserializeSeed unambiguously. The user-type is an associated + // type of the DeserializeSeed trait, and not a type parameter.) + // + // There are a few things we impl on the Seed: + // + // impl From for Seed + // impl Visitor<'de> for &Seed + // impl DeserializeSeed<'de> for Seed + // + // Then we impl ConfSerde on Self, naming Seed as the associated type. + // + // * DeserializeSeed is the main workhorse here. + // * From is necessary so that the ConfSerde impl has a way to actually + // construct the seed. + // * We do need something to impl Visitor, but it didn't have to be &Seed. It was just + // convenient to do it that way. + + let ident = self.struct_item.get_ident(); + let seed_ident = Ident::new("__SEED__", Span::call_site()); + + let visitor_impl = self.gen_serde_visitor_impl(&seed_ident, generics)?; + let deserialize_seed_impl = self.gen_serde_deserialize_seed_impl(&seed_ident, generics)?; + + // These generics are used to impl ConfSerde on the user's type. + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + + // The Seed struct needs an additional lifetime (the conf context lifetime) + // These generics are used when declaring the seed and implementing traits on it. + let ct = make_lifetime("'ctctct"); + let seed_generics = prepend_generic_lifetimes(generics, [&ct]); + let (seed_impl_generics, seed_ty_generics, seed_where_clause) = + seed_generics.split_for_impl(); + + Ok(Some(quote! { + const _: () = { + use ::core::{fmt, option::Option, marker::PhantomData, result::Result}; + use ::std::vec::Vec; + use ::conf::{ConfSerdeContext, ConfSerde, InnerError, serde::de}; + + pub struct #seed_ident #seed_generics { + ctxt: ConfSerdeContext<#ct>, + marker: PhantomData #ident #ty_generics>, + }; + + impl #seed_impl_generics From> for #seed_ident #seed_ty_generics #seed_where_clause { + fn from(ctxt: ConfSerdeContext<#ct>) -> Self { + Self { + ctxt, + marker: Default::default(), + } + } + } + + #visitor_impl + #deserialize_seed_impl + + impl #impl_generics ConfSerde for #ident #ty_generics #where_clause { + type Seed<#ct> = #seed_ident #seed_generics; + } + }; + })) + } + + // Helper which generates the tuple type used as the serde::Visitor::Value, + // the "output type" of the visitor. + // + // ( ( Option< Option< #field_type > >... ), Vec ) + // + // For a given #field_name, + // * None means serde did not produce this key and so we still have to visit it without serde + // afterwards + // * Some(None) means serde visited it and it produced an error. + // * Some(Some(val)) means serde visited it and produced a value. + fn gen_visitor_tuple_type(&self) -> Type { + let field_types: Vec = self.fields.iter().map(|f| f.get_field_type()).collect(); + // ( #ty ) is not a tuple type in rust, it must be ( #ty , ) when the tuple size is one. + let extra_comma = if field_types.len() == 1 { + Some(::default()) + } else { + None + }; + parse_quote! { + ( ( #( Option< Option< #field_types > > ),* #extra_comma) , Vec ) + } + } + + /// Generate implementation of serde::Visitor for &Seed + /// Panics if serde was not requested on this struct + /// + /// Arguments: + /// * seed_ident is the identifier used in this scope for the Seed type + /// * generics associated to this struct declaration + fn gen_serde_visitor_impl( + &self, + seed_ident: &Ident, + generics: &Generics, + ) -> Result { + let serde_opts = self.struct_item.serde.as_ref().unwrap(); + + // We need to add two generic lifetimes to the lifetime list (but only in the impl) + // One is the "deserializer lifetime", and one is the "context lifetime". + let de = make_lifetime("'dedede"); + let ct = make_lifetime("'ctctct"); + + let seed_generics = prepend_generic_lifetimes(generics, [&ct]); + let visitor_generics = prepend_generic_lifetimes(&seed_generics, [&de]); + + let (impl_generics, _, _) = visitor_generics.split_for_impl(); + + let ident = self.struct_item.get_ident(); + let expecting_str = format!("Object with schema {ident}"); + let ident_str = ident.to_string(); + + let conf_serde_context_ident = Ident::new("__conf_serde_context__", Span::call_site()); + let errors_ident = Ident::new("__errors__", Span::call_site()); + let map_access_ident = Ident::new("__map_access__", Span::call_site()); + let map_access_type = Ident::new("MA__", Span::call_site()); + + let field_names: Vec<&Ident> = self.fields.iter().map(|f| f.get_field_name()).collect(); + let field_match_arms_and_serde_names = self + .fields + .iter() + .map(|f| { + f.gen_serde_match_arm( + &conf_serde_context_ident, + &map_access_ident, + &map_access_type, + &errors_ident, + ) + }) + .collect::, _>>()?; + let (field_match_arms, serde_names): (Vec, Vec>) = + field_match_arms_and_serde_names.into_iter().unzip(); + let serde_names: Vec = serde_names.into_iter().flatten().collect(); + + // The Visitor impl Value type. This is a tuple containing Option< #field_type> and + // Vec It represents the partially finished work done using the document + // values from serde. + let visitor_tuple_type = self.gen_visitor_tuple_type(); + // ( #id ) is not a tuple in rust, it must be ( #id , ) when the tuple size is one. + let extra_comma = if field_names.len() == 1 { + Some(::default()) + } else { + None + }; + + let handle_unknown_field = if !serde_opts.allow_unknown_fields { + Some(quote! { + #errors_ident.push( + InnerError::serde( + #conf_serde_context_ident.document_name, + #ident_str, + #map_access_type::Error::unknown_field(__other__, &[ #(#serde_names),* ]) + ) + ); + }) + } else { + None + }; + + Ok(quote! { + impl #impl_generics de::Visitor<#de> for &#seed_ident #seed_generics { + type Value = #visitor_tuple_type; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, #expecting_str) + } + + fn visit_map<#map_access_type>(self, mut #map_access_ident: #map_access_type) -> Result + where #map_access_type: de::MapAccess<#de> + { + use ::conf::{ConfSerdeContext, IdentString, InnerError, SubcommandsSerde, serde::de::Error}; + let #conf_serde_context_ident: &ConfSerdeContext = &self.ctxt; + let ( ( #(mut #field_names),* #extra_comma), mut #errors_ident ) = Self::Value::default(); + + while let Some(key) = #map_access_ident.next_key::()? { + match key.as_str() { + #(#field_match_arms)* + __other__ => { #handle_unknown_field } + } + } + + Ok( ( ( #(#field_names),* #extra_comma), #errors_ident ) ) + } + } + }) + } + + // Generate an implementation of serde::DeserializerSeed on Seed + // Panics if serde was not requested on this struct + // + // *** How does this work? *** + // + // In the basic, no-serde version, things are pretty simple. + // Each field has an "initializer" expression, generated by FieldItem, + // which can init the field (from conf context) or return errors. + // We do `let #field_name: Option<#field_type> = #initializer`; for each field. + // Then we have a local variable for every field. (And a conf context and an error buffer in + // scope.) + // + // Then we use `gather_and_validate`. + // If any errors occurred, we return the entire error buffer. + // If not, we can essentially unwrap all the #field_name variables, because every initializer + // expression either returns Some(#field_type) or produces an error. + // Then we move them into the struct type. + // Then we try to run any validation routines. If any of them fail, return all the errors. + // Otherwise we succeeded and can return the struct. + // + // Each #initalizer is responsible for checking all the possible value sources and making the + // priority work, and applying value_parser etc. A lot of that logic is in the `ConfContext` + // methods, but it's resolved independently on a per field basis. + // + // Any ordering of running the #initializers would be fine, since they are independent of + // eachother. We just happen to run them in the order that they were declared. + // + // --- + // + // Once serde is in the mix, things are a bit more complicated, because, we don't get to decide + // the order in which serde gives us values. The MapAccess object gives us the keys and + // values in whatever order it wants. + // + // Instead, what we do is, we make a serde Visitor which uses the ConfContext, and walks + // whichever of the fields serde MapAccess produces a value for. Then it returns + // `Option>` and `Vec`, which go on the stack. + // + // The field is: + // None if `serde` did not attempt to visit it + // Some(None) if `serde` visited it, but an error resulted + // Some(Some(...)) if `serde successfully produced a value. + // + // (The distinction between None and Some(None) is important because, it's possible that args or + // env supplies a String for a given field, but the ValueParser produces errors, but also + // that `serde` produces multiple value for the same key, which is independently a separate + // error. We don't ever want to run a ValueParser twice. For the same reason, if serde + // visited a field and it produced an error, we want to remember that and not visit it in round + // 2 in the post-serde phase. It's confusing if we try to deserialize the same field in two + // different ways and potentially get two different errors.) + // Then we do a modified version of the no-serde initialization routine -- we check which + // options are populated (were given a value by serde), and any that are not populated, we + // run the no-serde initializer for. At the end, either everything is populated or we have + // at least one error. We run the same gather_and_validate routine to return the resulting + // struct. + // + // Arguments: + // * seed_ident is the identifier used in this scope for the Seed type + // * generics associated to this struct declaration + fn gen_serde_deserialize_seed_impl( + &self, + seed_ident: &Ident, + generics: &Generics, + ) -> Result { + let _serde_opts = self.struct_item.serde.as_ref().unwrap(); + + // We need to add two generic lifetimes to the lifetime list (but only in the impl) + // One is the "deserializer lifetime", and one is the "context lifetime". + let de = make_lifetime("'dedede"); + let ct = make_lifetime("'ctctct"); + + let seed_generics = prepend_generic_lifetimes(generics, [&ct]); + let visitor_generics = prepend_generic_lifetimes(&seed_generics, [&de]); + + let (_, ty_generics, _) = generics.split_for_impl(); + let (impl_generics, _, _) = visitor_generics.split_for_impl(); + + let ident = self.struct_item.get_ident(); + let ident_str = ident.to_string(); + + let conf_context_ident = Ident::new("__conf_context__", Span::call_site()); + let errors_ident = Ident::new("__errors__", Span::call_site()); + + let deserialize_finalizer_body = + self.get_deserialize_finalizer_body(&conf_context_ident, &errors_ident)?; + + let field_names: Vec<&Ident> = self.fields.iter().map(|f| f.get_field_name()).collect(); + let field_name_strs: Vec = field_names.iter().map(ToString::to_string).collect(); + + // The Visitor impl Value type. This is a tuple containing Option< #field_type> and + // Vec. It represents the partially finished work done using the + // document values from serde. + let visitor_tuple_type = self.gen_visitor_tuple_type(); + // The type that will be the Value of this DeserializeSeed impl + let value_type: Type = parse_quote! { + Result<#ident #ty_generics, Vec> + }; + + // High level: + // + // To implement deserialize seed, we take the deserializer, and call deserialize_struct. + // We pass a reference to ourself as the visitor, which was implemented in + // gen_serde_visitor_impl. + // + // This produces a Visitor::Value, which is a tuple consisting of + // Option>, indicating which fields we successfully + // deserialized, which ones tried to deserialize and failed, and which ones were not + // attempted in the serde phase. The __deserialize_finalizer finishes the job, + // initializing everything that serde didn't attempt to initialize, and then running + // validators etc. + // + // It may also produce a serde::de::Error. See gen_serde_visitor_impl -- that only happens + // if there is an error getting the next key from the map. + // + // If there is an error getting the next key from the map, it means that the deserializer + // data is not very well-formed -- most likely, the user is not iterating + // serde_json::Value or serde_yaml::Value or similar, because that should not give + // such an error. There may be a bunch of other key-value pairs that are in the + // user's data file, but are not parseable due to a syntax issue. + // + // If we continue trying to initialize stuff, we're likely going to get nonsense -- missing + // field errors for all the subsequent keys in this map that serde could not read. + // + // We'd rather only report the root cause -- that we couldn't iterate the map properly. + // So we bail out in that case, and don't attempt to proceed with finalizing. + Ok(quote! { + impl #impl_generics de::DeserializeSeed<#de> for #seed_ident #seed_generics { + type Value = #value_type; + + fn deserialize(self, __deserializer: D__) -> Result + where D__: de::Deserializer<#de> { + + use ::conf::{ConfContext, ConfSerde, ConfSerdeContext, InnerError}; + + fn __deserialize_finalizer( + #conf_context_ident: ConfContext<'_>, + (( #( mut #field_names,)* ), mut #errors_ident): #visitor_tuple_type + ) -> #value_type { + #deserialize_finalizer_body + } + + Ok(match __deserializer.deserialize_struct(#ident_str, &[ #(#field_name_strs,)* ], &self) { + Ok(tuple_val) => __deserialize_finalizer(self.ctxt.conf_context, tuple_val), + Err(err) => { + let ctxt: ConfSerdeContext = self.ctxt; + Err(vec![ InnerError::serde(ctxt.document_name, #ident_str, err) ]) + }, + }) + } + } + }) + } + + // Body of the deserialize finalizer function + // In this scope, all #field_name variables have type Option>, + // and they are Some if the serde step encountered those fields, and None otherwise + // + // In this step, we initialize exactly those fields that were not initialized by the + // serde walk. + // + // Arguments: + // * conf_context_ident is the identifier of a ConfContext variable in scope, which we may + // consume + // * errors_ident is the identifier of a mut Vec variable in scope, which we may + // consume + fn get_deserialize_finalizer_body( + &self, + conf_context_ident: &Ident, + errors_ident: &Ident, + ) -> Result { + // For each field, #field_name is currently a local variable of type Option>. + // If serde produced a value, then don't change anything. + // Otherwise, use the initializer from the non-serde path. + // We use `unwrap_or_else` here to accomplish this, and pass the initializer expr + // inside a lambda function which prevents shadowing the let binding. + // Push all errors into #errors_ident. + let initializations: Vec = self + .fields + .iter() + .map(|field| { + let field_name = field.get_field_name(); + let initializer = field.gen_initialize_from_conf_context_and_push_errors( + conf_context_ident, + errors_ident, + )?; + + Ok(quote! { + let #field_name = #field_name.unwrap_or_else(|| { + #initializer + }); + }) + }) + .collect::, Error>>()?; + + // Now, every variable has either been initialized by the serde path or the non serde path, + // and if it is still None, it means there was an error. We can gather and validate as + // usual. + let gather_and_validate = self.gather_and_validate(conf_context_ident, errors_ident)?; + + Ok(quote! { + #(#initializations)* -pub use field_item::FieldItem; -pub use struct_item::StructItem; - -/// Helper for parsing field items out of a syn struct -pub fn collect_args_fields( - struct_item: &StructItem, - fields: &FieldsNamed, -) -> Result, syn::Error> { - fields - .named - .iter() - .map(|field| FieldItem::new(field, struct_item)) - .collect() + #gather_and_validate + }) + } } diff --git a/conf_derive/src/proc_macro_options/struct_item.rs b/conf_derive/src/proc_macro_options/struct_item.rs index d98ffca..49b8236 100644 --- a/conf_derive/src/proc_macro_options/struct_item.rs +++ b/conf_derive/src/proc_macro_options/struct_item.rs @@ -1,17 +1,55 @@ use super::FieldItem; use crate::util::*; -use proc_macro2::TokenStream; +use proc_macro2::{Span, TokenStream}; use quote::quote; use std::{cmp::Ordering, collections::HashMap}; -use syn::{Attribute, Expr, Ident, LitStr}; +use syn::{meta::ParseNestedMeta, token, Attribute, Error, Expr, Ident, LitStr}; + +/// #[conf(serde(...))] options listed on a struct which has `#[derive(Conf)]` +pub struct StructSerdeItem { + pub allow_unknown_fields: bool, + span: Span, +} + +impl StructSerdeItem { + pub fn new(meta: ParseNestedMeta<'_>) -> Result { + let mut result = Self { + allow_unknown_fields: false, + span: meta.input.span(), + }; + + if meta.input.peek(token::Paren) { + meta.parse_nested_meta(|meta| { + let path = meta.path.clone(); + if path.is_ident("allow_unknown_fields") { + result.allow_unknown_fields = true; + Ok(()) + } else { + Err(meta.error("unrecognized conf(serde) option")) + } + })?; + } + + Ok(result) + } +} + +impl GetSpan for StructSerdeItem { + fn get_span(&self) -> Span { + self.span + } +} /// #[conf(...)] options listed on a struct which has `#[derive(Conf)]` +/// +/// Also assists with code generation related to these, such as for validations pub struct StructItem { pub struct_ident: Ident, pub about: Option, pub name: Option, pub no_help_flag: bool, pub env_prefix: Option, + pub serde: Option, pub one_of_fields: Vec<(Ordering, List)>, pub validation_predicate: Option, pub doc_string: Option, @@ -19,13 +57,14 @@ pub struct StructItem { impl StructItem { /// Parse conf options out of attributes on a struct - pub fn new(struct_ident: &Ident, attrs: &[Attribute]) -> Result { + pub fn new(struct_ident: &Ident, attrs: &[Attribute]) -> Result { let mut result = Self { struct_ident: struct_ident.clone(), about: None, name: None, no_help_flag: false, env_prefix: None, + serde: None, one_of_fields: Vec::default(), validation_predicate: None, doc_string: None, @@ -57,6 +96,8 @@ impl StructItem { &mut result.env_prefix, Some(parse_required_value::(meta)?), ) + } else if path.is_ident("serde") { + set_once(&path, &mut result.serde, Some(StructSerdeItem::new(meta)?)) } else if path.is_ident("validation_predicate") { set_once( &path, @@ -106,7 +147,7 @@ impl StructItem { } /// Generate a conf::ParserConfig expression, based on top-level options in this struct - pub fn gen_parser_config(&self) -> Result { + pub fn gen_parser_config(&self) -> Result { // This default if name is not explicitly set matches what clap-derive does. let name = self .name @@ -134,7 +175,7 @@ impl StructItem { pub fn gen_post_process_program_options( &self, program_options_ident: &Ident, - ) -> Result, syn::Error> { + ) -> Result, Error> { if self.env_prefix.is_none() { return Ok(None); } @@ -152,25 +193,20 @@ impl StructItem { })) } - /// Generate an (optional) ConfContext pre-processing step. - pub fn gen_pre_process_conf_context( - &self, - _conf_context_ident: &Ident, - ) -> Result, syn::Error> { - Ok(None) - } - /// Generate tokens that apply any validations to an instance /// - /// These tokens are the body of a validation function with signature - /// fn validation(#instance_ident: &Self, #instance_id_prefix_ident: &str) -> Result<(), - /// Vec> + /// These tokens are the body of a validation function with signature: + /// + /// fn validation( + /// #instance_ident: &Self, + /// #instance_id_prefix_ident: &str + /// ) -> Result<(), Vec> pub fn gen_validation_routine( &self, instance: &Ident, conf_context_ident: &Ident, fields: &[FieldItem], - ) -> Result { + ) -> Result { let struct_ident = &self.struct_ident; let struct_name = self.struct_ident.to_string(); let mut predicate_evaluations = Vec::::new(); @@ -190,7 +226,13 @@ impl StructItem { .iter() .map(ToString::to_string) .collect::>(); - quote! { Err(#conf_context_ident.too_few_arguments_error(#struct_name, &[#(#id_list),*], &[#(#quoted_flattened_id_list),*])) } + quote! { + Err(#conf_context_ident.too_few_arguments_error( + #struct_name, + &[#(#id_list),*], + &[#(#quoted_flattened_id_list),*] + )) + } }; // Depending on ordering parameter, a count of > 1 is either okay or an error @@ -199,7 +241,7 @@ impl StructItem { } else { let quoted_flattened_id_and_value_source_list = flattened_list .iter() - .map(|ident| -> Result { + .map(|ident| -> Result { let field_name = ident.to_string(); let get_value_source_expr = fields_helper.make_get_value_source_expr(ident)?; @@ -212,14 +254,22 @@ impl StructItem { // fail and early return with ?, but this isn't really expected to happen. // The functions that it is calling will all be failing earlier in the process if // they fail at all. + // + // Early returning from this block may have unexpected consequences. quote! { { - let flattened_ids_and_value_sources: Result)>)>, ::conf::InnerError> = + let flattened_ids_and_value_sources: Result< + Vec<(&'static str, Option<(&str, ::conf::ConfValueSource::<&str>)>)>, + ::conf::InnerError + > = (|| Ok(vec![#(#quoted_flattened_id_and_value_source_list),*]))(); - match flattened_ids_and_value_sources { - Ok(flattened_ids_and_value_sources) => Err(#conf_context_ident.too_many_arguments_error(#struct_name, &[#(#id_list),*], flattened_ids_and_value_sources)), - Err(err) => Err(err) - } + flattened_ids_and_value_sources.and_then(|ids_and_sources| { + Err(#conf_context_ident.too_many_arguments_error( + #struct_name, + &[#(#id_list),*], + ids_and_sources + )) + }) } } }; @@ -241,10 +291,19 @@ impl StructItem { if let Some(user_validation_predicate) = self.validation_predicate.as_ref() { predicate_evaluations.push(quote! { { - fn __validation_predicate__(#instance: & #struct_ident) -> Result<(), impl ::core::fmt::Display> { + fn __validation_predicate__( + #instance: & #struct_ident + ) -> Result<(), impl ::core::fmt::Display> + { #user_validation_predicate(#instance) } - __validation_predicate__(#instance).map_err(|err| ::conf::InnerError::validation(#struct_name, & #conf_context_ident .get_id_prefix(), err)) + __validation_predicate__(#instance).map_err(|err| + ::conf::InnerError::validation( + #struct_name, + & #conf_context_ident .get_id_prefix(), + err + ) + ) } }); } @@ -256,7 +315,10 @@ impl StructItem { } } else { quote! { - let errors = [#(#predicate_evaluations),*].into_iter().filter_map(|result| result.err()).collect::>(); + let errors = [#(#predicate_evaluations),*] + .into_iter() + .filter_map(|result| result.err()) + .collect::>(); if errors.is_empty() { Ok(()) } else { @@ -268,7 +330,8 @@ impl StructItem { } // struct which caches lookup from ident to FieldItem, and generates tokenstreams for checking if -// these fields are present in the struct instance etc. +// these fields are present in the struct instance etc. This is used in code-gen for the built-in +// validation predicates. struct FieldsHelper<'a> { instance: &'a Ident, conf_context_ident: &'a Ident, @@ -290,7 +353,7 @@ impl<'a> FieldsHelper<'a> { } } - pub fn get_field(&mut self, ident: &Ident) -> Result<&'a FieldItem, syn::Error> { + pub fn get_field(&mut self, ident: &Ident) -> Result<&'a FieldItem, Error> { let field_item = if let Some(val) = self.cache.get(ident) { val } else { @@ -298,21 +361,25 @@ impl<'a> FieldsHelper<'a> { .fields .iter() .find(|field| field.get_field_name() == ident) - .ok_or_else(|| syn::Error::new(ident.span(), "identifier not found in struct"))?; + .ok_or_else(|| Error::new(ident.span(), "identifier not found in struct"))?; self.cache.insert(ident.clone(), field); field }; Ok(field_item) } - pub fn get_is_present_expr(&mut self, ident: &Ident) -> Result { + pub fn get_is_present_expr(&mut self, ident: &Ident) -> Result { let field_item = self.get_field(ident)?; let field_type = field_item.get_field_type(); let instance = &self.instance; if let FieldItem::Parameter(item) = field_item { if item.get_default_value().is_some() { - return Err(syn::Error::new(ident.span(), "using one_of_fields constraint with a field that has a default_value is invalid, since it will always be present.")); + return Err(Error::new( + ident.span(), + "using one_of_fields constraint with a field \ + that has a default_value is invalid, since it will always be present.", + )); } }; @@ -323,7 +390,7 @@ impl<'a> FieldsHelper<'a> { } else if type_is_vec(&field_type)?.is_some() { quote! { !#instance.#ident.is_empty() } } else { - return Err(syn::Error::new( + return Err(Error::new( ident.span(), "field must be bool, Option, or Vec to use with one_of_fields constraint", )); @@ -335,11 +402,11 @@ impl<'a> FieldsHelper<'a> { pub fn make_count_expr_for_field_list( &mut self, list: &List, - ) -> Result { + ) -> Result { let u32_exprs: Vec = list .elements .iter() - .map(|ident| -> Result { + .map(|ident| -> Result { let bool_expr = self.get_is_present_expr(ident)?; Ok(quote! { #bool_expr as u32 }) }) @@ -352,7 +419,7 @@ impl<'a> FieldsHelper<'a> { pub fn split_single_options_and_flattened( &mut self, list: &List, - ) -> Result<(Vec, Vec), syn::Error> { + ) -> Result<(Vec, Vec), Error> { let mut single_opts = Vec::::new(); let mut groups = Vec::::new(); @@ -367,13 +434,13 @@ impl<'a> FieldsHelper<'a> { Ok((single_opts, groups)) } - pub fn make_get_value_source_expr(&mut self, ident: &Ident) -> Result { + pub fn make_get_value_source_expr(&mut self, ident: &Ident) -> Result { let field_item = self.get_field(ident)?; match field_item { FieldItem::Flatten(flatten_item) => { Ok(flatten_item.any_program_options_appeared_expr(self.conf_context_ident)?) } - _ => Err(syn::Error::new( + _ => Err(Error::new( ident.span(), "field is not flattened, this is an internal error", )), diff --git a/conf_derive/src/subcommand_proc_macro_options/enum_item.rs b/conf_derive/src/subcommand_proc_macro_options/enum_item.rs new file mode 100644 index 0000000..6a11dde --- /dev/null +++ b/conf_derive/src/subcommand_proc_macro_options/enum_item.rs @@ -0,0 +1,44 @@ +use crate::util::*; +use syn::{Attribute, Error, Ident}; + +/// #[conf(...)] options listed on an enum which has `#[derive(Subcommands)]` +/// +/// Also assists with code generation related to these, such as for validations +pub struct EnumItem { + pub enum_ident: Ident, + pub serde: bool, + pub doc_string: Option, +} + +impl EnumItem { + /// Parse conf options out of attributes on an enum + pub fn new(enum_ident: &Ident, attrs: &[Attribute]) -> Result { + let mut result = Self { + enum_ident: enum_ident.clone(), + serde: false, + doc_string: None, + }; + + for attr in attrs { + maybe_append_doc_string(&mut result.doc_string, &attr.meta)?; + if attr.path().is_ident("conf") { + attr.parse_nested_meta(|meta| { + let path = meta.path.clone(); + if path.is_ident("serde") { + result.serde = true; + Ok(()) + } else { + Err(meta.error("unrecognized conf option")) + } + })?; + } + } + + Ok(result) + } + + /// Get the identifier of this enum + pub fn get_ident(&self) -> &Ident { + &self.enum_ident + } +} diff --git a/conf_derive/src/subcommand_proc_macro_options/mod.rs b/conf_derive/src/subcommand_proc_macro_options/mod.rs index f74acdf..ad6382c 100644 --- a/conf_derive/src/subcommand_proc_macro_options/mod.rs +++ b/conf_derive/src/subcommand_proc_macro_options/mod.rs @@ -5,17 +5,225 @@ //! //! This contains such helpers for the derive(Subcommand) macro. -use syn::{Ident, Variant}; +use proc_macro2::{Span, TokenStream}; +use quote::quote; +use syn::{Attribute, Error, Generics, Ident, Variant}; + +mod enum_item; +use enum_item::EnumItem; mod variant_item; -pub use variant_item::VariantItem; - -/// Helper for parsing variant items out of a syn enum, tagged as a group of subcommands -pub fn collect_enum_variants<'a>( - enum_ident: &Ident, - variants: impl Iterator, -) -> Result, syn::Error> { - variants - .map(|var| VariantItem::new(var, enum_ident)) - .collect() +use variant_item::VariantItem; + +/// Helper which generates individual functions related to `#[derive(Subcommands)]` +/// on an enum. +/// +/// Calling "new" parses all the proc macro attributes for enum and fields. +/// Calling individual functions returns code gen. +pub struct GenSubcommandsEnum { + enum_item: EnumItem, + variants: Vec, +} + +impl GenSubcommandsEnum { + /// Parse syn data for an enum with `#[derive(Subcommands)]` on it + pub fn new<'a>( + ident: &Ident, + attrs: &[Attribute], + variants: impl Iterator, + ) -> Result { + Ok(Self { + enum_item: EnumItem::new(ident, attrs)?, + variants: variants + .map(|var| VariantItem::new(var, ident)) + .collect::, _>>()?, + }) + } + + /// Generate a Subcommands impl for this enum + pub fn gen_subcommands_impl(&self, generics: &Generics) -> Result { + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + + let ident = self.enum_item.get_ident(); + + let subcommands_fns = vec![ + self.get_parsers_impl()?, + self.get_subcommand_names_impl()?, + self.from_conf_context_impl()?, + ]; + + Ok(quote! { + #[automatically_derived] + #[allow( + unused_qualifications, + )] + impl #impl_generics ::conf::Subcommands for #ident #ty_generics #where_clause { + #(#subcommands_fns)* + } + }) + } + + /// Generate Subcommands::get_parsers implementation + fn get_parsers_impl(&self) -> Result { + let parsers_ident = Ident::new("__parsers__", Span::call_site()); + let parsed_env_ident = Ident::new("__parsed_env__", Span::call_site()); + let variants_push_parsers: Vec = self + .variants + .iter() + .map(|var| var.gen_push_parsers(&parsers_ident, &parsed_env_ident)) + .collect::, syn::Error>>()?; + + Ok(quote! { + fn get_parsers(#parsed_env_ident: &::conf::ParsedEnv) -> Result, ::conf::Error> { + let mut #parsers_ident = vec![]; + + #(#variants_push_parsers)* + + Ok(#parsers_ident) + } + }) + } + + /// Generate Subcommands::get_subcommand_names implementation + fn get_subcommand_names_impl(&self) -> Result { + let command_names: Vec<_> = self + .variants + .iter() + .map(|var| var.get_command_name()) + .collect(); + + Ok(quote! { + fn get_subcommand_names() -> &'static [&'static str] { + &[ #(#command_names,)* ] + } + }) + } + + /// Generate Subcommands::from_conf_context implementation + #[allow(clippy::wrong_self_convention)] + fn from_conf_context_impl(&self) -> Result { + let variant_match_arms: Vec = self.variants + .iter() + .map(|var| { + let name = var.get_name(); + let command_name = var.get_command_name(); + let ty = var.get_type(); + quote! { + #command_name => Ok(Self::#name(<#ty as Conf>::from_conf_context(conf_context)?)) + } + }) + .collect(); + + Ok(quote! { + fn from_conf_context( + command_name: String, + conf_context: ::conf::ConfContext<'_> + ) -> Result> { + match command_name.as_str() { + #(#variant_match_arms,)* + _ => { + panic!( + "Unknown command name '{command_name}'. This is an internal error. Expected '{:?}'", + ::get_subcommand_names() + ) + } + } + } + }) + } + + /// Generate a SubcommandsSerde impl for this enum, if requested + pub fn maybe_gen_subcommands_serde_impl( + &self, + generics: &Generics, + ) -> Result, syn::Error> { + if !self.enum_item.serde { + return Ok(None); + } + + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + + let ident = self.enum_item.get_ident(); + + let subcommands_serde_items = + vec![self.gen_serde_names()?, self.gen_from_conf_serde_context()?]; + + Ok(Some(quote! { + #[automatically_derived] + #[allow( + unused_qualifications, + )] + impl #impl_generics ::conf::SubcommandsSerde for #ident #ty_generics #where_clause { + #(#subcommands_serde_items)* + } + })) + } + + fn gen_serde_names(&self) -> Result { + let tuples: Vec = self + .variants + .iter() + .filter(|var| !var.get_serde_skip()) + .map(|var| { + let command_name = var.get_command_name(); + let serde_name = var.get_serde_name(); + quote! { + (#command_name, #serde_name) + } + }) + .collect(); + Ok(quote! { + const SERDE_NAMES: &'static[(&'static str, &'static str)] = &[ #( #tuples ),* ]; + }) + } + + fn gen_from_conf_serde_context(&self) -> Result { + let next_value_producer_ident = Ident::new("__next_value_producer__", Span::call_site()); + + let variant_match_arms: Vec = self + .variants + .iter() + .filter(|var| !var.get_serde_skip()) + .map(|var| { + let name = var.get_name(); + let command_name = var.get_command_name(); + let serde_name = var.get_serde_name(); + let ty = var.get_type(); + quote! { + #command_name => { + let document_name = ctxt.document_name; + let seed = <#ty as ConfSerde>::Seed::from(ctxt); + Ok(Self::#name(#next_value_producer_ident.next_value_seed(seed).map_err(|err| { + vec![InnerError::serde( + document_name, + #serde_name, + err + )] + })??)) + } + } + }) + .collect(); + + Ok(quote! { + fn from_conf_serde_context<'de, NVP>( + command_name: &str, + ctxt: ::conf::ConfSerdeContext, + #next_value_producer_ident: NVP + ) -> Result> + where NVP: ::conf::NextValueProducer<'de> + { + use ::conf::{ConfSerde, InnerError}; + match command_name { + #(#variant_match_arms,)* + _ => { + panic!( + "Unknown command name '{command_name}'. This is an internal error. Expected '{:?}'", + ::SERDE_NAMES + ) + } + } + } + }) + } } diff --git a/conf_derive/src/subcommand_proc_macro_options/variant_item.rs b/conf_derive/src/subcommand_proc_macro_options/variant_item.rs index 8af0892..7c08621 100644 --- a/conf_derive/src/subcommand_proc_macro_options/variant_item.rs +++ b/conf_derive/src/subcommand_proc_macro_options/variant_item.rs @@ -1,14 +1,62 @@ use crate::util::*; -use heck::ToKebabCase; -use proc_macro2::TokenStream; +use heck::{ToKebabCase, ToSnakeCase}; +use proc_macro2::{Span, TokenStream}; use quote::quote; -use syn::{spanned::Spanned, Error, Fields, FieldsUnnamed, Ident, LitStr, Type, Variant}; +use syn::{ + meta::ParseNestedMeta, spanned::Spanned, token, Error, Fields, FieldsUnnamed, Ident, LitStr, + Type, Variant, +}; +/// #[conf(serde(...))] options listed on a field of Flatten kind +pub struct VariantSerdeItem { + pub rename: Option, + pub skip: bool, + span: Span, +} + +impl VariantSerdeItem { + pub fn new(meta: ParseNestedMeta<'_>) -> Result { + let mut result = Self { + rename: None, + skip: false, + span: meta.input.span(), + }; + + if meta.input.peek(token::Paren) { + meta.parse_nested_meta(|meta| { + let path = meta.path.clone(); + if path.is_ident("rename") { + set_once( + &path, + &mut result.rename, + Some(parse_required_value::(meta)?), + ) + } else if path.is_ident("skip") { + result.skip = true; + Ok(()) + } else { + Err(meta.error("unrecognized conf(serde) option")) + } + })?; + } + + Ok(result) + } +} + +impl GetSpan for VariantSerdeItem { + fn get_span(&self) -> Span { + self.span + } +} + +/// Proc macro annotations parsed from a variant within a Subcommands enum pub struct VariantItem { variant_name: Ident, variant_type: Type, is_optional_type: Option, command_name: LitStr, + serde: Option, doc_string: Option, } @@ -40,12 +88,13 @@ impl VariantItem { variant_name, variant_type, is_optional_type, + serde: None, doc_string: None, }; let mut command_name_override: Option = None; - for attr in &field.attrs { + for attr in &variant.attrs { maybe_append_doc_string(&mut result.doc_string, &attr.meta)?; if attr.path().is_ident("conf") || attr.path().is_ident("subcommands") { attr.parse_nested_meta(|meta| { @@ -58,6 +107,8 @@ impl VariantItem { &mut command_name_override, Some(parse_required_value::(meta)?), ) + } else if path.is_ident("serde") { + set_once(&path, &mut result.serde, Some(VariantSerdeItem::new(meta)?)) } else { Err(meta.error("unrecognized conf subcommands option")) } @@ -84,6 +135,22 @@ impl VariantItem { self.variant_type.clone() } + pub fn get_serde_name(&self) -> LitStr { + self.serde + .as_ref() + .and_then(|serde| serde.rename.clone()) + .unwrap_or_else(|| { + LitStr::new( + &self.variant_name.to_string().to_snake_case(), + self.variant_name.span(), + ) + }) + } + + pub fn get_serde_skip(&self) -> bool { + self.serde.as_ref().map(|serde| serde.skip).unwrap_or(false) + } + pub fn gen_push_parsers( &self, parsers_ident: &Ident, @@ -93,7 +160,10 @@ impl VariantItem { let command_name = &self.command_name; Ok(quote! { - #parsers_ident.push(<#inner_type as conf::Conf>::get_parser(#parsed_env_ident)?.rename(#command_name)); + #parsers_ident.push( + <#inner_type as ::conf::Conf>::get_parser(#parsed_env_ident)? + .rename(#command_name) + ); }) } } diff --git a/conf_derive/src/util.rs b/conf_derive/src/util.rs index 3e5a863..2cdd4a0 100644 --- a/conf_derive/src/util.rs +++ b/conf_derive/src/util.rs @@ -1,11 +1,11 @@ use heck::{ToKebabCase, ToShoutySnakeCase}; use proc_macro2::{Span, TokenStream}; use quote::{quote, ToTokens}; -use std::fmt::Display; +use std::{borrow::Borrow, fmt::Display}; use syn::{ bracketed, meta::ParseNestedMeta, parenthesized, parse::Parse, punctuated::Punctuated, - spanned::Spanned, Error, Expr, ExprLit, GenericArgument, Lit, LitChar, LitStr, Meta, Path, - PathArguments, Token, Type, + spanned::Spanned, Error, Expr, ExprLit, GenericArgument, GenericParam, Generics, Lifetime, + LifetimeParam, Lit, LitChar, LitStr, Meta, Path, PathArguments, Token, Type, }; /// Helper for determining if a type is likely bool @@ -169,7 +169,8 @@ pub fn set_once( /// Helper for appending a doc string attribute to the description string, if it is a doc string /// attribute. -// Based on code here: https://github.com/cyqsimon/documented/blob/e9a465c9e1666839ea08efbe9ce54480d7ee769f/documented-derive/src/lib.rs#L411 +// Based on code here: +// https://github.com/cyqsimon/documented/blob/e9a465c9e1666839ea08efbe9ce54480d7ee769f/documented-derive/src/lib.rs#L411 pub fn maybe_append_doc_string( description: &mut Option, attr_meta: &Meta, @@ -289,3 +290,25 @@ impl GetSpan for List { self.elements.span() } } + +/// Make a lifetime from a string +pub fn make_lifetime(lt: impl AsRef) -> Lifetime { + Lifetime::new(lt.as_ref(), Span::call_site()) +} + +/// Helper for adding new lifetime params to the beginning of a list of Generics +pub fn prepend_generic_lifetimes>( + generics: &Generics, + lifetimes: impl AsRef<[R]>, +) -> Generics { + let mut generics = generics.clone(); + + for lt in lifetimes.as_ref().iter().rev() { + generics.params.insert( + 0, + GenericParam::Lifetime(LifetimeParam::new(lt.borrow().clone())), + ); + } + + generics +} diff --git a/examples/figment.rs b/examples/figment.rs new file mode 100644 index 0000000..46a8209 --- /dev/null +++ b/examples/figment.rs @@ -0,0 +1,13 @@ +#[cfg(feature = "serde")] +#[path = "serde/figment.rs"] +mod serde_figment; + +#[cfg(feature = "serde")] +fn main() { + serde_figment::main() +} + +#[cfg(not(feature = "serde"))] +fn main() { + panic!("needs serde feature") +} diff --git a/examples/model_service.json b/examples/model_service.json new file mode 100644 index 0000000..6b40fe3 --- /dev/null +++ b/examples/model_service.json @@ -0,0 +1,5 @@ +{ + "db": { + "retries": 44 + } +} diff --git a/examples/model_service.toml b/examples/model_service.toml new file mode 100644 index 0000000..6f99269 --- /dev/null +++ b/examples/model_service.toml @@ -0,0 +1,5 @@ +listen_addr = "0.0.0.0:80" +db.retries = 3 + +[migrations] +sql_file = "xxx.sql" diff --git a/examples/model_service2.toml b/examples/model_service2.toml new file mode 100644 index 0000000..ef0a65f --- /dev/null +++ b/examples/model_service2.toml @@ -0,0 +1,7 @@ +# This file has intentional mistakes +listen_addr = "0.0.0.0:81" +db.retries = "xxx" + +[migrations] +sql_file = "foo.sql" +unexpected_arg = 4 diff --git a/examples/serde/basic.rs b/examples/serde/basic.rs new file mode 100644 index 0000000..08363db --- /dev/null +++ b/examples/serde/basic.rs @@ -0,0 +1,37 @@ +use conf::Conf; +use std::{env, ffi::OsString, fs}; + +#[path = "./model_service.rs"] +mod model_service; +use model_service::ModelServiceConfig; + +pub fn main() { + // In this example, the user may specify `--config PATH` or `--config=PATH` on the CLI args, + // or set an environment variable `CONFIG`. Then the file is loaded in json format if present, + // and used as a value-source during config parsing. + // + // This has to be done before calling `conf::Parse` because we have to supply the parsed config + // file to `conf` as one of the value sources if we want to use it. + // + // `conf::find_parameter` is a minimal function which uses `clap_lex` to search for just one + // parameter, without introducing any additional dependencies. + let config_path: Option = + conf::find_parameter("config", env::args_os()).or_else(|| env::var_os("CONFIG")); + + let config = if let Some(config_path) = config_path { + // When the config file is specified, its an error if it can't be found or can't be parsed + // in the expected format. + let file_contents = fs::read_to_string(&config_path).expect("Could not open config file"); + let doc_content: toml::Value = toml::from_str(&file_contents).expect("Config file format"); + // Now we pass the parsed content to the builder, with the file name so that it can be used + // in error messages. + ModelServiceConfig::conf_builder() + .doc(config_path.to_string_lossy(), doc_content) + .parse() + } else { + // There is no config file, so we can just try to parse from only args and env. + ModelServiceConfig::parse() + }; + + println!("{config:#?}"); +} diff --git a/examples/serde/figment.rs b/examples/serde/figment.rs new file mode 100644 index 0000000..5ce504f --- /dev/null +++ b/examples/serde/figment.rs @@ -0,0 +1,40 @@ +use conf::Conf; +use figment::{ + providers::{Format, Json, Toml}, + value::Value, + Figment, +}; +use std::env; + +#[path = "./model_service.rs"] +mod model_service; +use model_service::ModelServiceConfig; + +pub fn main() { + // In this example, we show how figment can be used together with conf + // in order to load content from multiple files, merge them according to + // some hierarchical order, and then supply the result to conf. + // + // This takes advantage of conf's comprehensive error reporting, and gives + // better results than if you just extract() directly into your final + // config structure. + let mut fig = Figment::new(); + + if let Some(path) = env::var_os("TOML") { + fig = fig.merge(Toml::file(path)); + } + if let Some(path) = env::var_os("TOML2") { + fig = fig.merge(Toml::file(path)); + } + if let Some(path) = env::var_os("JSON") { + fig = fig.merge(Json::file(path)); + } + + let doc_content: Value = fig.extract().unwrap(); + + let config = ModelServiceConfig::conf_builder() + .doc("files", &doc_content) + .parse(); + + println!("{config:#?}"); +} diff --git a/examples/serde/model_service.rs b/examples/serde/model_service.rs new file mode 100644 index 0000000..e430a08 --- /dev/null +++ b/examples/serde/model_service.rs @@ -0,0 +1,60 @@ +use conf::{Conf, Subcommands}; +use http::Uri as Url; +use std::net::SocketAddr; +use std::path::PathBuf; + +/// Configuration for an http client +#[derive(Conf, Debug)] +#[conf(serde)] +pub struct HttpClientConfig { + /// Base URL + #[arg(long, env, serde(use_value_parser))] + pub url: Url, + + /// Number of retries + #[arg(long, env)] + pub retries: u32, +} + +/// Configuration for model service +#[derive(Conf, Debug)] +#[conf(serde)] +pub struct ModelServiceConfig { + /// Listen address to bind to + #[arg(long, env, default_value = "127.0.0.1:9090")] + pub listen_addr: SocketAddr, + + /// Auth service: + #[arg(flatten, prefix, help_prefix)] + pub auth: Option, + + /// Database: + #[arg(flatten, prefix, help_prefix)] + pub db: HttpClientConfig, + + /// Config file path + #[arg(long = "config", env = "CONFIG")] + pub config_file: Option, + + /// Optional subcommands + #[arg(subcommands)] + pub command: Option, +} + +/// Subcommands that can be used with this service +#[derive(Subcommands, Debug)] +#[conf(serde)] +pub enum Command { + #[conf(serde(rename = "migrations"))] + RunMigrations(MigrationConfig), + #[conf(serde(rename = "migrations"))] + ShowPendingMigrations(MigrationConfig), +} + +#[derive(Conf, Debug)] +#[conf(serde)] +pub struct MigrationConfig { + /// Path to migrations file (instead of embedded migrations) + #[arg(long, env)] + pub sql_file: Option, +} diff --git a/examples/serde_basic.rs b/examples/serde_basic.rs new file mode 100644 index 0000000..beb89ce --- /dev/null +++ b/examples/serde_basic.rs @@ -0,0 +1,13 @@ +#[cfg(feature = "serde")] +#[path = "serde/basic.rs"] +mod basic; + +#[cfg(feature = "serde")] +fn main() { + basic::main() +} + +#[cfg(not(feature = "serde"))] +fn main() { + panic!("needs serde feature") +} diff --git a/examples/showcase.rs b/examples/showcase.rs index 4788d2b..d2f014e 100644 --- a/examples/showcase.rs +++ b/examples/showcase.rs @@ -1,6 +1,6 @@ use conf::Conf; +use http::Uri as Url; use std::net::SocketAddr; -use url::Url; /// Configuration for an http client #[derive(Conf, Debug)] diff --git a/examples/showcase_subcommands.rs b/examples/subcommands_example.rs similarity index 95% rename from examples/showcase_subcommands.rs rename to examples/subcommands_example.rs index 9db6074..573bc35 100644 --- a/examples/showcase_subcommands.rs +++ b/examples/subcommands_example.rs @@ -1,7 +1,7 @@ use conf::{Conf, Subcommands}; +use http::Uri as Url; use std::net::SocketAddr; use std::path::PathBuf; -use url::Url; /// Configuration for an http client #[derive(Conf, Debug)] @@ -17,6 +17,7 @@ pub struct HttpClientConfig { /// Configuration for model service #[derive(Conf, Debug)] +#[conf(name = "subcommands_example")] pub struct ModelServiceConfig { /// Listen address to bind to #[conf(long, env, default_value = "127.0.0.1:9090")] diff --git a/src/builder.rs b/src/builder.rs new file mode 100644 index 0000000..c7d1afa --- /dev/null +++ b/src/builder.rs @@ -0,0 +1,95 @@ +use crate::{parse_env, Conf, ConfContext, Error, InnerError, ParsedArgs, ParsedEnv}; +use std::{ffi::OsString, marker::PhantomData}; + +/// A builder which collects config value sources for the parse. +/// +/// Use any of [`ConfBuilder::args`], [`ConfBuilder::env`], [`ConfBuilder::doc`] to set sources, +/// and then call one of [`ConfBuilder::parse`] or [`ConfBuilder::try_parse`]. +/// +/// If `args` is not called, the default source is `std::env::args_os`. +/// If `env` is not called, the default source is `std::env::vars_os`. +pub struct ConfBuilder +where + S: Conf, +{ + collected_env: ParsedEnv, + inited_env: bool, + collected_args: Vec, + inited_args: bool, + _marker: PhantomData S>, +} + +impl Default for ConfBuilder +where + S: Conf, +{ + fn default() -> Self { + Self { + collected_env: Default::default(), + inited_env: false, + collected_args: Default::default(), + inited_args: false, + _marker: Default::default(), + } + } +} + +impl ConfBuilder +where + S: Conf, +{ + /// Set the CLI args used in this parse + pub fn args(mut self, args: impl IntoIterator>) -> Self { + assert!(!self.inited_args, "Cannot set args twice"); + self.collected_args = args.into_iter().map(Into::into).collect(); + self.inited_args = true; + self + } + + /// Set the env vars used in this parse + pub fn env(mut self, env: impl IntoIterator) -> Self + where + K: Into, + V: Into, + { + assert!(!self.inited_env, "Cannot set env twice"); + self.collected_env = parse_env(env); + self.inited_env = true; + self + } + + /// Parse based on supplied sources (or falling back to defaults), and exiting the program + /// with errors logged to stderr if parsing fails. + pub fn parse(self) -> S { + match self.try_parse() { + Ok(result) => result, + Err(err) => err.exit(), + } + } + + /// Try to parse an instance based on supplied sources (or falling back to defaults), + /// returning an error if parsing fails. + pub fn try_parse(self) -> Result { + let (parsed_env, args) = self.into_tuple(); + + let parser = S::get_parser(&parsed_env)?; + let arg_matches = parser.parse(args)?; + let parsed_args = ParsedArgs::new(&arg_matches, &parser); + let conf_context = ConfContext::new(parsed_args, &parsed_env); + S::from_conf_context(conf_context) + .map_err(|errs| InnerError::vec_to_clap_error(errs, parser.get_command())) + } + + /// Convert self into an args, env tuple, after setting defaults from std::env::* and such + /// if anything was not inited + pub(crate) fn into_tuple(mut self) -> (ParsedEnv, Vec) { + if !self.inited_args { + self = self.args(std::env::args_os()); + } + if !self.inited_env { + self = self.env(std::env::vars_os()); + } + + (self.collected_env, self.collected_args) + } +} diff --git a/src/conf_context.rs b/src/conf_context.rs index 94cfa92..c9339b7 100644 --- a/src/conf_context.rs +++ b/src/conf_context.rs @@ -13,6 +13,7 @@ where { Args, Env(S), + Document(S), Default, } @@ -21,9 +22,14 @@ impl<'a> ConfValueSource<&'a str> { match self { Self::Args => ConfValueSource::Args, Self::Env(s) => ConfValueSource::Env(s.to_owned()), + Self::Document(s) => ConfValueSource::Document(s.to_owned()), Self::Default => ConfValueSource::Default, } } + + pub fn is_default(&self) -> bool { + matches!(self, Self::Default) + } } impl<'a> From for ConfValueSource<&'a str> { diff --git a/src/conf_serde/builder.rs b/src/conf_serde/builder.rs new file mode 100644 index 0000000..48d5d56 --- /dev/null +++ b/src/conf_serde/builder.rs @@ -0,0 +1,104 @@ +use crate::{ + Conf, ConfBuilder, ConfContext, ConfSerde, ConfSerdeContext, Error, InnerError, ParsedArgs, +}; +use serde::de::{DeserializeSeed, Deserializer}; +use std::{ffi::OsString, marker::PhantomData}; + +impl ConfBuilder +where + S: ConfSerde, +{ + /// Set the document used in this parse. + /// + /// Requires a name for the document and a serde Deserializer representing the content. + /// + /// The name is typically the name of a file, and is used in error messages. + /// + /// The deserializer is, for example, a serde_json::Value, serde_yaml::Value, figment::Value, + /// etc. which you have already loaded from disk and parsed in an unstructured way. + pub fn doc<'de, D: Deserializer<'de>>( + self, + document_name: impl Into, + deserializer: D, + ) -> ConfSerdeBuilder<'de, S, D> { + ConfSerdeBuilder { + inner: self, + document_name: document_name.into(), + document: deserializer, + _marker: Default::default(), + } + } +} + +/// A ConfBuilder which additionally has serde-document content installed. +/// +/// This is only allowed when the target struct supports serde, i.e. has `#[conf(serde)]` attribute. +pub struct ConfSerdeBuilder<'de, S, D> +where + S: ConfSerde, + D: Deserializer<'de>, +{ + inner: ConfBuilder, + document_name: String, + document: D, + _marker: PhantomData<&'de u8>, +} + +impl<'de, S, D> ConfSerdeBuilder<'de, S, D> +where + S: ConfSerde, + D: Deserializer<'de>, +{ + /// Set the env vars used in this parse + pub fn env(mut self, env: impl IntoIterator) -> Self + where + K: Into, + V: Into, + { + self.inner = self.inner.env(env); + self + } + + /// Set the CLI args used in this parse + pub fn args(mut self, args: impl IntoIterator>) -> Self { + self.inner = self.inner.args(args); + self + } + + /// Parse based on supplied sources (or falling back to defaults), and exiting the program + /// with errors logged to stderr if parsing fails. + pub fn parse(self) -> S { + match self.try_parse() { + Ok(result) => result, + Err(err) => err.exit(), + } + } + + /// Try to parse an instance based on supplied sources (or falling back to defaults), + /// returning an error if parsing fails. + pub fn try_parse(self) -> Result { + let Self { + inner, + document, + document_name, + _marker, + } = self; + let (parsed_env, args) = inner.into_tuple(); + + let parser = ::get_parser(&parsed_env)?; + let arg_matches = parser.parse(args)?; + let parsed_args = ParsedArgs::new(&arg_matches, &parser); + let conf_context = ConfContext::new(parsed_args, &parsed_env); + let conf_serde_context = ConfSerdeContext::new(conf_context, document_name.as_str()); + let seed = ::Seed::from(conf_serde_context); + // Code gen should produce: + // impl<'de> DeserializeSeed for Seed { + // type Value = Result>; + // ... + // } + // So that the result of deserialize call is Result>, D::Error> + DeserializeSeed::<'de>::deserialize(seed, document) + .expect("Internal error, Deserializer Error should not be returned here") + .map_err(|errs| InnerError::vec_to_clap_error(errs, parser.get_command())) + } +} diff --git a/src/conf_serde/mod.rs b/src/conf_serde/mod.rs new file mode 100644 index 0000000..07978d3 --- /dev/null +++ b/src/conf_serde/mod.rs @@ -0,0 +1,111 @@ +//! Use a serde Deserializer as a value-soure for layered config. +//! +//! We discussed extensively in MOTIVATION.md why serde *deserializer* libraries +//! are unable to cause serde to return multiple errors when deserialization fails. +//! We also discussed why the ability to do that is so important to us. +//! +//! In this module, we create a way that we *can* return multiple errors when walking the +//! `Deserializer`, as long as we control the `Deserialize` side of things. +//! Since we control the proc-macro here, we have such control. +//! +//! We can even incorporate arbitrary other data, like a `ConfContext`, into the walk, +//! and use other value sources besides the `Deserializer` when initializing the struct. +//! +//! The main idea is to use the `DeserializeSeed` trait from `serde`, implement it +//! on (a variation of) ConfContext, and set `type Value = Result>;` +//! This means that when we implement `DeseralizeSeed::deserialize`, we get all the `ConfContext`, +//! meaning parsed args, env, any thing else that we want, and we get the `Deserializer`. +//! The type we have to return is `Result>, D::Error>`, but if +//! we simply always return the `Ok` variant, then we basically have exactly the signature that +//! we wanted to implement. +//! +//! So we can inspect all this data, walk the deserializer and see what it has, then walk the +//! fields it didn't have any data for, and initialize everything according to the hierachical +//! config priority. We can collect as many errors as we need to, and when we return, we always +//! return `Ok(Ok(instance))` or `Ok(Err(errors))`. We can also use this same `DeserializeSeed` +//! trick when we recurse into a substructure, because `serde::de::MapAccess` allows us to invoke +//! the `DeserializeSeed` API however we want when recursing. +//! +//! At the end, the `ConfBuilder` is able to manage the error trickery we performed here and +//! return what the user was expecting, so none of this has any negative impact on the public API. +//! +//! And the plus side is, we have only loose coupling with all the file-reading and format-parsing +//! code. Users can bring their own stuff, whatever `serde_json` or `serde_json_lenient` or any +//! other variation, and configure it however they want. And conf has no explicit dependencies on +//! anything that reads a file. +//! +//! *** +//! +//! Note that due to this technique, you will get better error reporting from +//! using `Conf` to deserialize a large structure from a `serde::Deserializer` than +//! `serde::Deserialize` would be able to (!), because we can collect errors from all over the +//! structure. +//! +//! However, the technique has some limitations -- it has to be acceptable to type-erase +//! the serde errors. In this crate, we are mapping all the errors that can occur to +//! `conf::InnerError` and then aggregating them as one `clap::Error` essentially. If you wanted to +//! generalize the technique, and you don't want to break compat with existing serde Deserializers, +//! you would need to use `Box` or something, but `serde` doesn't give you `'static` on +//! the error types, or convertibility between e.g. Deserializer errors and MapAccess errors. +//! +//! Another thing is, this is only really a good technique when you are deserializing a +//! self-describing format, like json, yaml, toml, etc, and you deserialize their type from +//! bytes-on-the-wire first. If you try to pass e.g. `serde_json::Deserializer::from_reader` as the +//! `Deserializer` here, you may get worse error reporting if the file is not well-formed +//! potentially, because once you have junk bytes on the wire and the wire protocol isn't working +//! right, trying to read more things is probably futile and just creates noise. +//! +//! Another drawback of this approach is, because we aren't using the `serde::Deserialize` derive +//! macro from `serde_derive`, we don't automatically get support for any of the `#[serde(...)]` +//! annotations. However, this is also a plus in some sense, because for things like +//! `#[serde(flatten)]`, we don't inherit any of the limitations or bugs, and we can choose to +//! implement it in a way that makes sense for our use-case later. +mod builder; +pub use builder::ConfSerdeBuilder; + +mod traits; +pub use traits::{ConfSerde, ConfSerdeContext, NextValueProducer, SubcommandsSerde}; + +/// Helper for deserializing serde MapAccess keys in a struct visitor. +/// Similar to `String`, but uses "Deserializer::deserializer_identifier" to hint. +#[doc(hidden)] +pub struct IdentString { + inner: String, +} + +impl IdentString { + /// Get identifier as a str + pub fn as_str(&self) -> &str { + self.inner.as_str() + } +} + +impl<'de> serde::de::Deserialize<'de> for IdentString { + fn deserialize(deserializer: D) -> Result + where + D: serde::de::Deserializer<'de>, + { + use core::fmt; + use serde::de; + + struct Visitor {} + impl<'de> de::Visitor<'de> for Visitor { + type Value = IdentString; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "a field identifier") + } + + fn visit_str(self, s: &str) -> Result + where + E: de::Error, + { + Ok(IdentString { + inner: s.to_owned(), + }) + } + } + + deserializer.deserialize_identifier(Visitor {}) + } +} diff --git a/src/conf_serde/traits.rs b/src/conf_serde/traits.rs new file mode 100644 index 0000000..1ce9085 --- /dev/null +++ b/src/conf_serde/traits.rs @@ -0,0 +1,169 @@ +use crate::{Conf, ConfContext, InnerError, Subcommands}; +use serde::de::{Deserialize, DeserializeSeed}; + +/// Extension to Conf trait with serde-integration implementation details. +/// +/// To derive this trait, use `#[derive(Conf)]` together with the `#[conf(serde)]` annotation +/// on your struct. +/// +/// Requires the `"serde"` feature to be enabled (it is on by default). +/// +/// Any flattened substructures must also have `#[conf(serde)]`. +/// May cause requirements on other fields to support `serde::Deserialize` -- +/// you can use `#[conf(serde(skip))]` to avoid that. +/// +/// Note that this is NOT based on deriving `serde::Deserialize` on your structure, because +/// we would not be able to properly implement layering or to collect errors comprehensively. +/// Annotations that you put like `#[serde(rename)]` on your conf structure will have no effect. +/// Only `#[conf(serde(...))]` options will affect the behavior. +/// +/// **Hand-written implementations of this trait are not supported**. +/// +/// You should think of this trait as a "non-exhaustive trait" with hidden required items. +/// `conf` is free to add, modify, or remove these implementation details without considering it +/// a semver breaking change, so the only stable way to get an impl is via the derive macro. +pub trait ConfSerde: Conf + Sized { + // This GAT points to the seed type that is generated by the derive macro. + // This is the hook for the builder and other types to invoke the generated code. + #[doc(hidden)] + type Seed<'a>: From> + + for<'de> DeserializeSeed<'de, Value = Result>>; +} + +/// Extension to Subcommands trait with serde-integration implementation details. +/// +/// To derive this trait, use `#[derive(Subcommands)]` together with the `#[conf(serde)]` annotation +/// on your enum. +/// +/// Requires the `"serde"` feature to be enabled (it is on by default). +/// +/// Each variant must have a `Conf` struct which is annotated `#[conf(serde)]`, or else +/// the variant must be marked `#[conf(serde(skip))]`. +/// +/// **Hand-written implementations of this trait are not supported**. +/// +/// You should think of this trait as a "non-exhaustive trait" with hidden required items. +/// `conf` is free to add, modify, or remove these implementation details without considering it +/// a semver breaking change, so the only stable way to get an impl is via the derive macro. +pub trait SubcommandsSerde: Subcommands + Sized { + /// List of (command_name, serde_name) pairs, for those subcommands that are not serde(skip) + #[doc(hidden)] + const SERDE_NAMES: &'static [(&'static str, &'static str)]; + + // Similar to Subcommands::from_conf_context but now with serde data as well. + // + // Arguments: + // * command_name: The command name that appeared on the command line + // * ctxt: ConfSerdeContext for the subcommand + // * map_access: Ready to provide the next_value corresponding to this subcommand. + // + // Callee is guaranteed that: + // * command_name is one of the command names listed in Self::SERDE_NAMES + // * command_name and conf_serde_context are the results of `ConfSerdeContext::for_subcommand`, + // so this command name appeared on the command-line. + // * the key from the most recent MapAccess::next_key() call matched to a serde_name that is + // paired with this command_name in Self::SERDE_NAMES. + // + // Note: + // MapAccess is provided to allow calling next_value_seed. + // We should really wrap that so that only that function can be called, + // and only once, in order to have a stronger API. + // For now this is all doc(hidden) anyways. + #[doc(hidden)] + fn from_conf_serde_context<'de, NVP>( + command_name: &str, + ctxt: ConfSerdeContext, + next_value_producer: NVP, + ) -> Result> + where + NVP: NextValueProducer<'de>; +} + +/// A handle to a subset of the [`serde::de::MapAccess`] functionality. +/// This handle allows one to call `next_value` or `next_value_seed` *once*, with +/// whatever type is desired. +/// +/// This is useful when the type should depend on the key, +/// and how exactly is decided by code that is generated elsewhere. +/// The `NextValueProducer` can be passed to that code without risk of corrupting +/// the `MapAccess`, since the caller has a static guarantee about how and how +/// many times the receiver can use the `MapAccess`. +#[doc(hidden)] +pub trait NextValueProducer<'de>: Sized { + type Error: serde::de::Error; + + fn next_value_seed(self, seed: S) -> Result + where + S: DeserializeSeed<'de>; + + fn next_value(self) -> Result + where + V: Deserialize<'de>, + { + self.next_value_seed(core::marker::PhantomData) + } +} + +impl<'de, MA> NextValueProducer<'de> for &mut MA +where + MA: serde::de::MapAccess<'de>, +{ + type Error = MA::Error; + + fn next_value_seed(self, seed: S) -> Result + where + S: DeserializeSeed<'de>, + { + MA::next_value_seed(self, seed) + } +} + +/// A regular [`ConfContext`], plus any additional context about the serde document we are parsing. +/// An instance of this should be contained in any [`ConfSerde::Seed`] type. +/// +/// Note: Depth parameter isn't really used anymore, but it is there anyways. +/// Note: This is non-exhaustive mainly so that the proc-macro code uses the functions +/// to construct it, and we add more functions as needed, which is more maintainable. +#[doc(hidden)] +#[non_exhaustive] +pub struct ConfSerdeContext<'a> { + pub conf_context: ConfContext<'a>, + pub document_name: &'a str, + pub depth: usize, +} + +impl<'a> ConfSerdeContext<'a> { + /// Create `ConfSerdeContext` from `ConfContext` and document name + pub(crate) fn new(conf_context: ConfContext<'a>, document_name: &'a str) -> Self { + Self { + conf_context, + document_name, + depth: 0, + } + } + + /// Same as [`ConfContext::for_flattened`] but now for a `ConfSerdeContext` + pub fn for_flattened(&self, id_prefix: &str) -> Self { + Self { + conf_context: self.conf_context.for_flattened(id_prefix), + document_name: self.document_name, + depth: self.depth + 1, + } + } + + /// Same as [`ConfContext::for_subcommand`], but now for a `ConfSerdeContext` + pub fn for_subcommand(&self) -> Option<(String, Self)> { + self.conf_context + .for_subcommand() + .map(|(subcommand_name, conf_context)| { + ( + subcommand_name, + Self { + conf_context, + document_name: self.document_name, + depth: self.depth, + }, + ) + }) + } +} diff --git a/src/error.rs b/src/error.rs index 77dae76..d49c4d1 100644 --- a/src/error.rs +++ b/src/error.rs @@ -101,6 +101,9 @@ pub enum InnerError { /// Missing required subcommand // struct name, field name, subcommands MissingRequiredSubcommand(String, String, Vec), + /// Parsing (document) + // document name, field name, error + Serde(String, String, String), } impl InnerError { @@ -230,6 +233,16 @@ impl InnerError { subcommand_names.iter().map(|x| (*x).to_owned()).collect(), ) } + + /// Helper which makes Serde + pub fn serde(document_name: &str, field_name: &str, err: impl fmt::Display) -> Self { + Self::Serde( + document_name.to_owned(), + field_name.to_owned(), + err.to_string(), + ) + } + // A short (one-line) description of the problem fn title(&self) -> &'static str { match self { @@ -240,6 +253,7 @@ impl InnerError { Self::ValidationFailed(..) => "Validation failed", Self::InvalidParameterValue(..) => "Invalid value", Self::MissingRequiredSubcommand(..) => "Missing required subcommand", + Self::Serde(..) => "Parsing document", } } @@ -253,6 +267,7 @@ impl InnerError { Self::ValidationFailed(..) => ErrorKind::ValueValidation, Self::InvalidParameterValue(..) => ErrorKind::InvalidValue, Self::MissingRequiredSubcommand(..) => ErrorKind::MissingSubcommand, + Self::Serde(..) => ErrorKind::InvalidValue, } } @@ -267,6 +282,7 @@ impl InnerError { Self::TooManyArguments(..) => None, Self::ValidationFailed(..) => None, Self::MissingRequiredSubcommand(..) => None, + Self::Serde(..) => None, } } @@ -420,6 +436,15 @@ impl InnerError { writeln!(stream, " {name}")?; } } + Self::Serde(document_name, field_name, err) => { + let context = format!(" Parsing {document_name} (@ {field_name})"); + let estimated_len = context.len(); + writeln!( + stream, + "{context}: {err_str}", + err_str = Self::format_err_str(err, estimated_len + 2) + )?; + } } Ok(()) } @@ -485,6 +510,9 @@ fn render_provided_opt(opt: &ProgramOption, value_source: &ConfValueSource { format!("env '{name}'") } + ConfValueSource::Document(name) => { + format!("document '{name}'") + } } } diff --git a/src/find_parameter.rs b/src/find_parameter.rs new file mode 100644 index 0000000..5faac3d --- /dev/null +++ b/src/find_parameter.rs @@ -0,0 +1,144 @@ +use clap_lex::RawArgs; +use std::ffi::OsString; + +/// In some cases, you may want to grab a config file path from CLI args before doing the +/// main parse, so that you can load the config file content and pass it to conf::conf_builder(), +/// if the config file path was present. +/// +/// This helper function uses `clap_lex` to do that, which is already indirectly a dependency of +/// `conf` anyways. It is only meant to be used for a one-off like this that is needed before the +/// main parse. +/// +/// See `examples/serde/basic.rs` for example usage. +/// +/// Limitations: +/// +/// * Only returns the first example that it finds on the command line. Don't use this with a multi +/// option, only with a parameter. +/// * No error reporting. If the parameter is found but has no argument, we just return `None`. That +/// is expected to be caught in the `Conf::parse` step. +/// * No auto-generated docs. +/// +/// If you use this, you should add a dummy parameter of the same name to your top-level config +/// struct, even if it is not used later in the program, so that you don't get an "unexpected +/// argument" error during the main parse, and so that this parameter will appear in the `--help`. +pub fn find_parameter( + name: &str, + args_os: impl IntoIterator>, +) -> Option { + let raw = RawArgs::new(args_os); + let mut cursor = raw.cursor(); + raw.next(&mut cursor); // Skip the bin + + while let Some(arg) = raw.next(&mut cursor) { + if arg.is_escape() { + return None; + } else if let Some((maybe_flag, maybe_value)) = arg.to_long() { + let Ok(flag) = maybe_flag else { + continue; + }; + + if flag != name { + continue; + } + + // We found what we are searching for, now the two possibilities are + // --flag=value + // and + // --flag value + if let Some(value) = maybe_value { + return Some(value.to_owned()); + } else { + // Note: If there is no next value, it should be thought of as an error, + // but we just return None, and let the main parse do the error reporting. + return raw.next_os(&mut cursor).map(|os_str| os_str.to_owned()); + } + } + } + + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_find_parameter() { + assert_eq!(None, find_parameter("foo", ["."])); + + assert_eq!( + "x", + find_parameter("foo", [".", "--foo=x"]) + .unwrap() + .to_string_lossy() + ); + assert_eq!(None, find_parameter("fo", [".", "--foo=x"])); + assert_eq!(None, find_parameter("fooo", [".", "--foo=x"])); + assert_eq!( + "x", + find_parameter("foo", [".", "-zed", "z", "--foo=x"]) + .unwrap() + .to_string_lossy() + ); + assert_eq!(None, find_parameter("fo", [".", "-zed", "z", "--foo=x"])); + assert_eq!(None, find_parameter("fooo", [".", "-zed", "z", "--foo=x"])); + + assert_eq!( + "x", + find_parameter("foo", [".", "--foo", "x"]) + .unwrap() + .to_string_lossy() + ); + assert_eq!(None, find_parameter("fo", [".", "--foo", "x"])); + assert_eq!(None, find_parameter("fooo", [".", "--foo", "x"])); + assert_eq!( + "x", + find_parameter("foo", [".", "-zed", "z", "--foo", "x"]) + .unwrap() + .to_string_lossy() + ); + assert_eq!(None, find_parameter("fo", [".", "-zed", "z", "--foo", "x"])); + assert_eq!( + None, + find_parameter("fooo", [".", "-zed", "z", "--foo", "x"]) + ); + + assert_eq!(None, find_parameter("foo", [".", "--foo"])); + assert_eq!(None, find_parameter("foo", [".", "--bar=9", "--foo"])); + + assert_eq!( + "x", + find_parameter("foo", [".", "--foo=x", "--foo", "y"]) + .unwrap() + .to_string_lossy() + ); + assert_eq!( + "x", + find_parameter("foo", [".", "--foo", "x", "--foo", "y"]) + .unwrap() + .to_string_lossy() + ); + assert_eq!( + "x", + find_parameter("foo", [".", "--bar=9", "--foo=x", "--foo", "y"]) + .unwrap() + .to_string_lossy() + ); + assert_eq!( + "x", + find_parameter("foo", [".", "--bar=9", "--foo", "x", "--foo", "y"]) + .unwrap() + .to_string_lossy() + ); + + assert_eq!( + None, + find_parameter("fo", [".", "--bar=9", "--foo=x", "--foo", "y"]) + ); + assert_eq!( + None, + find_parameter("fo", [".", "--bar=9", "--foo", "x", "--foo", "y"]) + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index c78852a..9a0eab6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,8 +9,10 @@ #![deny(unsafe_code)] #![deny(missing_docs)] +mod builder; mod conf_context; mod error; +mod find_parameter; mod parse_env; mod parser; mod program_option; @@ -23,11 +25,17 @@ use parse_env::parse_env; use parser::ParsedArgs; use str_to_bool::str_to_bool; -// Conf, Subcommands, and perhaps Error, is the only public API, but the derive macro needs these -// other types. +// These exports represent the public API. +pub use builder::ConfBuilder; pub use error::Error; +pub use find_parameter::find_parameter; pub use traits::{Conf, Subcommands}; +// Export conf_derive proc-macros unconditionally. Their docs are on the traits that they +// produce implementations for. +#[doc(hidden)] +pub use conf_derive::{self, *}; +// The derive macro needs these other types, so they are exported, but doc(hidden). #[doc(hidden)] pub use conf_context::{ConfContext, ConfValueSource}; #[doc(hidden)] @@ -39,9 +47,35 @@ pub use parser::{Parser, ParserConfig}; #[doc(hidden)] pub use program_option::{ParseType, ProgramOption}; +// The serde feature brings in some more types and traits +#[cfg(feature = "serde")] +mod conf_serde; +// These are publicly documented. Everything needed to understand how to use the builder +// should be well-documented. +#[cfg(feature = "serde")] +pub use conf_serde::{ConfSerde, ConfSerdeBuilder}; +// These are internals used by the derive macro. #[doc(hidden)] -pub use conf_derive::{self, *}; +#[cfg(feature = "serde")] +pub use conf_serde::{ConfSerdeContext, IdentString, NextValueProducer, SubcommandsSerde}; +// Re-export serde crate for the proc macro +#[doc(hidden)] +#[cfg(feature = "serde")] +pub use serde::{self, *}; // CowStr is used internally mainly because using it allows us to construct ProgramOption in a const // way from string literals, but also to modify them if they have to be flattened into something. type CowStr = std::borrow::Cow<'static, str>; + +// Helper for some of the proc-macro code-gen +// This lets you get the inner-type of a Vec no matter how Vec is spelled +// (Vec, std::vec::Vec, alloc::vec::Vec) or aliased by user code. +// This is normally difficult to do directly from a proc-macro, which can only see the tokens. +#[doc(hidden)] +pub trait InnerTypeHelper { + type Ty; +} + +impl InnerTypeHelper for Vec { + type Ty = T; +} diff --git a/src/parse_env.rs b/src/parse_env.rs index dbca8ce..ede52c7 100644 --- a/src/parse_env.rs +++ b/src/parse_env.rs @@ -29,8 +29,8 @@ impl ParsedEnv { /// and store it in a searchable container. pub fn parse_env(env_vars_os: impl IntoIterator) -> ParsedEnv where - K: Into + Clone, - V: Into + Clone, + K: Into, + V: Into, { // Drop any non-utf8 env keys, since there's no way the parser can read them anyways, since we // don't give the user a way to specify a non-utf8 env value that should be read. diff --git a/src/traits.rs b/src/traits.rs index 43f9413..a7ab74e 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -1,50 +1,49 @@ use crate::{ - parse_env, ConfContext, ConfValueSource, Error, InnerError, ParsedArgs, ParsedEnv, Parser, - ParserConfig, ProgramOption, + ConfBuilder, ConfContext, ConfValueSource, Error, InnerError, ParsedEnv, Parser, ParserConfig, + ProgramOption, }; use std::ffi::OsString; /// The Conf trait is implemented by types that represent a collection of config parsed on startup, -/// and is modeled on `clap::Parser`. Users usually call `parse` or another of these functions on -/// their config structure in `main()`. +/// and is modeled on `clap::Parser`. /// -/// Hand-written implementations of this trait are not supported. +/// To use it, put `#[derive(Conf)]` on your config structure, and then call `Conf::parse` or +/// another of these functions in `main()`. +/// +/// **Hand-written implementations of this trait are not supported**. +/// +/// You should think of this trait as a "non-exhaustive trait" with hidden required items. +/// `conf` is free to add, modify, or remove these implementation details without considering it +/// a semver breaking change, so the only stable way to get an impl is via the derive macro. #[doc = include_str!("../REFERENCE_derive_conf.md")] pub trait Conf: Sized { /// Parse self from the process CLI args and environment, and exit the program with a help /// message if we cannot. #[inline] fn parse() -> Self { - match Self::try_parse() { - Ok(result) => result, - Err(err) => err.exit(), - } + Self::conf_builder().parse() } /// Try to parse self from the process CLI args and environment, and return an error if we /// cannot. #[inline] fn try_parse() -> Result { - Self::try_parse_from(std::env::args_os(), std::env::vars_os()) + Self::conf_builder().try_parse() } /// Parse self from given containers which stand in for the process args and environment, and /// exit the program with a help message if we cannot. This function's behavior is isolated - /// from the values of `std::env::args_os` and `std::env::vars_os`. + /// from the values of [`std::env::args_os`] and [`std::env::vars_os`]. #[inline] fn parse_from( - args_os: impl IntoIterator, + args_os: impl IntoIterator>, env_vars_os: impl IntoIterator, ) -> Self where - T: Into + Clone, - K: Into + Clone, - V: Into + Clone, + K: Into, + V: Into, { - match Self::try_parse_from(args_os, env_vars_os) { - Ok(result) => result, - Err(err) => err.exit(), - } + Self::conf_builder().args(args_os).env(env_vars_os).parse() } /// Try to parse self from given containers which stand in for the process args and environment, @@ -58,13 +57,16 @@ pub trait Conf: Sized { K: Into + Clone, V: Into + Clone, { - let parsed_env = parse_env(env_vars_os); - let parser = Self::get_parser(&parsed_env)?; - let arg_matches = parser.parse(args_os)?; - let parsed_args = ParsedArgs::new(&arg_matches, &parser); - let conf_context = ConfContext::new(parsed_args, &parsed_env); - Self::from_conf_context(conf_context) - .map_err(|errs| InnerError::vec_to_clap_error(errs, parser.get_command())) + Self::conf_builder() + .args(args_os) + .env(env_vars_os) + .try_parse() + } + + /// Obtain a ConfBuilder, and use the builder API to parse. + /// The builder API is needed if you want to use advanced features. + fn conf_builder() -> ConfBuilder { + Default::default() } // Construct a conf::Parser object appropriate for this Conf. @@ -142,16 +144,25 @@ pub trait Conf: Sized { fn get_name() -> &'static str; } -/// The Subcommands trait represents one or more subcommands, and is derived on Enums. +/// The Subcommands trait represents one or more subcommands that can be added to a `Conf` +/// structure. To use it, put `#[derive(Subcommands)]` on your enum, and then add a +/// `#[conf(subcommands)]` field to your `Conf` structure whose type is your enum type, or +/// `Option`. +/// +/// Each variant of the enum corresponds to a subcommand, and must contain a single un-named `Conf` +/// structure. /// -/// Each subcommand is an enum variant containing a Conf structure. +/// The subcommand name is the name of the enum variant, but you can use attributes to change this +/// name. /// -/// The subcommand name is the name of the enum variant. -///in /// A Subcommands enum can then be added as a field to a top-level Conf structure and marked using /// the `#[conf(subcommands)]` attribute. /// -/// Hand-written implementations of this trait are not supported. +/// **Hand-written implementations of this trait are not supported**. +/// +/// You should think of this trait as a "non-exhaustive trait" with hidden required items. +/// `conf` is free to add, modify, or remove these implementation details without considering it +/// a semver breaking change, so the only stable way to get an impl is via the derive macro. #[doc = include_str!("../REFERENCE_derive_subcommands.md")] pub trait Subcommands: Sized { // Get the subcommands associated to this enum. diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 500706e..97eb0e3 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -2,6 +2,8 @@ use conf::Error; use core::cmp::min; +use escargot::{CargoBuild, CargoRun}; +use std::{process::Command, sync::OnceLock}; // Helper for making Vec concisely pub fn vec_str(list: impl IntoIterator) -> Vec { @@ -148,3 +150,23 @@ pub fn assert_error_contains_text_and_not_other_text( } } } + +// Helper for running examples in integration tests, so that we can test process behavior and output +// end to end. +pub trait Example { + const NAME: &'static str; + + fn get_built_example() -> &'static CargoRun { + static ONCE: OnceLock = OnceLock::new(); + ONCE.get_or_init(|| CargoBuild::new().example(Self::NAME).run().unwrap()) + } + + fn get_command() -> Command { + Self::get_built_example().command() + } +} + +// Helper for finding the example directory +pub fn examples_dir() -> &'static str { + concat!(env!("CARGO_MANIFEST_DIR"), "/examples") +} diff --git a/tests/test_basic_serde_example.rs b/tests/test_basic_serde_example.rs new file mode 100644 index 0000000..daed2b7 --- /dev/null +++ b/tests/test_basic_serde_example.rs @@ -0,0 +1,832 @@ +#![cfg(feature = "serde")] + +mod common; +use common::{assert_multiline_eq, examples_dir, Example}; +use std::str::from_utf8; + +struct SerdeBasicExample {} +impl Example for SerdeBasicExample { + const NAME: &'static str = "serde_basic"; +} + +#[test] +fn test_serde_basic_example_no_args() { + let mut command = SerdeBasicExample::get_command(); + let output = command.output().unwrap(); + + let expected = &" +error: A required value was not provided + env 'DB_RETRIES', or '--db-retries', must be provided + env 'DB_URL', or '--db-url', must be provided +"[1..]; + + assert_multiline_eq!(from_utf8(&output.stderr).unwrap(), &expected); + assert_eq!(output.status.code(), Some(2)); +} + +#[test] +fn test_serde_basic_example_some_invalid_args() { + let mut command = SerdeBasicExample::get_command(); + let output = command + .args(["--db-url=asdf:/"]) + .envs([("DB_RETRIES", "5")]) + .output() + .unwrap(); + + let expected = &" +error: Invalid value + when parsing '--db-url' value 'asdf:/': invalid format + +Help: + --db-url + Database: Base URL + [env: DB_URL] +"[1..]; + + assert_eq!(output.status.code(), Some(2)); + assert_multiline_eq!(from_utf8(&output.stderr).unwrap(), &expected); +} + +#[test] +fn test_serde_basic_example_help() { + let mut command = SerdeBasicExample::get_command(); + let output = command.args(["--help"]).output().unwrap(); + + let expected = &" +Configuration for model service + +Usage: serde_basic [OPTIONS] [COMMAND] + +Commands: + run-migrations + show-pending-migrations + help Print this message or the help of the given subcommand(s) + +Options: + --listen-addr Listen address to bind to + [env LISTEN_ADDR=] + [default: 127.0.0.1:9090] + --auth-url Auth service: Base URL + [env AUTH_URL=] + --auth-retries Auth service: Number of retries + [env AUTH_RETRIES=] + --db-url Database: Base URL + [env DB_URL=] + --db-retries Database: Number of retries + [env DB_RETRIES=] + --config Config file path + [env CONFIG=] + -h, --help Print help +"[1..]; + + assert_eq!( + output.status.code(), + Some(0), + "stderr: {}", + from_utf8(&output.stderr).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stdout).unwrap(), &expected); +} + +#[test] +fn test_serde_basic_example_success_args() { + let mut command = SerdeBasicExample::get_command(); + let output = command + .args([ + "--auth-url=https://example.com", + "--auth-retries=7", + "--db-url", + "postgres://localhost/dev", + "--db-retries", + "9", + ]) + .output() + .unwrap(); + + let expected = &r#" +ModelServiceConfig { + listen_addr: 127.0.0.1:9090, + auth: Some( + HttpClientConfig { + url: https://example.com/, + retries: 7, + }, + ), + db: HttpClientConfig { + url: postgres://localhost/dev, + retries: 9, + }, + config_file: None, + command: None, +} +"#[1..]; + + assert_eq!( + output.status.code(), + Some(0), + "stderr: {}", + from_utf8(&output.stderr).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stdout).unwrap(), &expected); +} + +#[test] +fn test_serde_basic_example_success_env() { + let mut command = SerdeBasicExample::get_command(); + let output = command + .envs([ + ("LISTEN_ADDR", "0.0.0.0:7777"), + ("DB_URL", "postgres://localhost/dev"), + ("DB_RETRIES", "3"), + ]) + .output() + .unwrap(); + + let expected = &r#" +ModelServiceConfig { + listen_addr: 0.0.0.0:7777, + auth: None, + db: HttpClientConfig { + url: postgres://localhost/dev, + retries: 3, + }, + config_file: None, + command: None, +} +"#[1..]; + + assert_eq!( + output.status.code(), + Some(0), + "stderr: {}", + from_utf8(&output.stderr).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stdout).unwrap(), &expected); +} + +#[test] +fn test_serde_basic_example_success_args_and_file() { + let mut command = SerdeBasicExample::get_command(); + let toml_file = examples_dir().to_string() + "/model_service.toml"; + let output = command + .args([ + "--config", + &toml_file, + "--db-url", + "postgres://localhost/dev", + ]) + .output() + .unwrap(); + + let expected = &format!( + r#" +ModelServiceConfig {{ + listen_addr: 0.0.0.0:80, + auth: None, + db: HttpClientConfig {{ + url: postgres://localhost/dev, + retries: 3, + }}, + config_file: Some( + "{toml_file}", + ), + command: None, +}} +"# + )[1..]; + + assert_eq!( + output.status.code(), + Some(0), + "stderr: {}", + from_utf8(&output.stderr).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stdout).unwrap(), &expected); +} + +#[test] +fn test_serde_basic_example_success_args_and_file_shadowing() { + let mut command = SerdeBasicExample::get_command(); + let toml_file = examples_dir().to_string() + "/model_service.toml"; + let output = command + .args([ + "--config", + &toml_file, + "--db-url", + "postgres://localhost/dev", + "--db-retries=14", + ]) + .output() + .unwrap(); + + let expected = &format!( + r#" +ModelServiceConfig {{ + listen_addr: 0.0.0.0:80, + auth: None, + db: HttpClientConfig {{ + url: postgres://localhost/dev, + retries: 14, + }}, + config_file: Some( + "{toml_file}", + ), + command: None, +}} +"# + )[1..]; + + assert_eq!( + output.status.code(), + Some(0), + "stderr: {}", + from_utf8(&output.stderr).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stdout).unwrap(), &expected); +} + +#[test] +fn test_serde_basic_example_success_and_args_env_and_file() { + let mut command = SerdeBasicExample::get_command(); + let toml_file = examples_dir().to_string() + "/model_service.toml"; + let output = command + .args(["--db-url", "postgres://localhost/dev"]) + .envs([ + ("CONFIG", toml_file.as_str()), + ("AUTH_URL", "http://auth.svc.cluster"), + ("AUTH_RETRIES", "3"), + ]) + .output() + .unwrap(); + + let expected = &format!( + r#" +ModelServiceConfig {{ + listen_addr: 0.0.0.0:80, + auth: Some( + HttpClientConfig {{ + url: http://auth.svc.cluster/, + retries: 3, + }}, + ), + db: HttpClientConfig {{ + url: postgres://localhost/dev, + retries: 3, + }}, + config_file: Some( + "{toml_file}", + ), + command: None, +}} +"# + )[1..]; + + assert_eq!( + output.status.code(), + Some(0), + "stderr: {}", + from_utf8(&output.stderr).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stdout).unwrap(), &expected); +} + +#[test] +fn test_serde_basic_example_success_and_args_env_and_file_shadowing() { + let mut command = SerdeBasicExample::get_command(); + let toml_file = examples_dir().to_string() + "/model_service.toml"; + let output = command + .args(["--db-url", "postgres://localhost/dev"]) + .envs([ + ("CONFIG", toml_file.as_str()), + ("DB_URL", "postgres://db.svc.cluster"), + ("DB_RETRIES", "82"), + ("AUTH_URL", "http://auth.svc.cluster"), + ("AUTH_RETRIES", "3"), + ]) + .output() + .unwrap(); + + let expected = &format!( + r#" +ModelServiceConfig {{ + listen_addr: 0.0.0.0:80, + auth: Some( + HttpClientConfig {{ + url: http://auth.svc.cluster/, + retries: 3, + }}, + ), + db: HttpClientConfig {{ + url: postgres://localhost/dev, + retries: 82, + }}, + config_file: Some( + "{toml_file}", + ), + command: None, +}} +"# + )[1..]; + + assert_eq!( + output.status.code(), + Some(0), + "stderr: {}", + from_utf8(&output.stderr).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stdout).unwrap(), &expected); +} + +#[test] +fn test_serde_basic_example_args_env_and_file_with_missing_and_invalid() { + let mut command = SerdeBasicExample::get_command(); + let toml_file = examples_dir().to_string() + "/model_service.toml"; + let output = command + .args(["--auth-url", "asdf:/"]) + .envs([ + ("CONFIG", toml_file.as_str()), + ("DB_RETRIES", "82"), + ("AUTH_RETRIES", "xxx"), + ]) + .output() + .unwrap(); + + let expected = &format!( + r#" +error: A required value was not provided + env 'DB_URL', or '--db-url', must be provided +error: Invalid value + when parsing '--auth-url' value 'asdf:/': invalid format + when parsing env 'AUTH_RETRIES' value 'xxx': invalid digit found in string +"# + )[1..]; + + assert_eq!( + output.status.code(), + Some(2), + "stdout: {}", + from_utf8(&output.stdout).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stderr).unwrap(), &expected); +} + +#[test] +fn test_serde_basic_example_args_env_and_file_with_missing_and_invalid2() { + let mut command = SerdeBasicExample::get_command(); + let toml_file = examples_dir().to_string() + "/model_service2.toml"; + let output = command + .args(["--auth-url", "asdf:/"]) + .envs([ + ("CONFIG", toml_file.as_str()), + ("DB_RETRIES", "82"), + ("AUTH_RETRIES", "xxx"), + ]) + .output() + .unwrap(); + + let expected = &format!( + r#" +error: A required value was not provided + env 'DB_URL', or '--db-url', must be provided +error: Invalid value + when parsing '--auth-url' value 'asdf:/': invalid format + when parsing env 'AUTH_RETRIES' value 'xxx': invalid digit found in string +error: Parsing document + Parsing {toml_file} (@ retries): + invalid type: string "xxx", expected u32 + in `retries` + +"# + )[1..]; + + assert_eq!( + output.status.code(), + Some(2), + "stderr: {}", + from_utf8(&output.stderr).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stderr).unwrap(), &expected); +} + +#[test] +fn test_serde_basic_example_subcommand_help() { + let mut command = SerdeBasicExample::get_command(); + let output = command.args(["run-migrations", "--help"]).output().unwrap(); + + let expected = &" +Usage: serde_basic run-migrations [OPTIONS] + +Options: + --sql-file Path to migrations file (instead of embedded migrations) + [env SQL_FILE=] + -h, --help Print help +"[1..]; + + assert_multiline_eq!(from_utf8(&output.stdout).unwrap(), &expected); + assert_eq!(output.status.code(), Some(0)); +} + +#[test] +fn test_serde_basic_example_subcommand_invalid_args() { + let mut command = SerdeBasicExample::get_command(); + let output = command + .args(["run-migrations", "--db-url=postgres://localhost/dev"]) + .output() + .unwrap(); + + let expected = &" +error: unexpected argument '--db-url' found + +Usage: serde_basic run-migrations [OPTIONS] + +For more information, try '--help'. +"[1..]; + + assert_multiline_eq!(from_utf8(&output.stderr).unwrap(), &expected); + assert_eq!(output.status.code(), Some(2)); +} + +#[test] +fn test_serde_basic_example_subcommand_missing_args() { + let mut command = SerdeBasicExample::get_command(); + let output = command + .args(["--db-url=postgres://localhost/dev", "run-migrations"]) + .output() + .unwrap(); + + let expected = &" +error: A required value was not provided + env 'DB_RETRIES', or '--db-retries', must be provided + +Help: + --db-retries + Database: Number of retries + [env: DB_RETRIES] +"[1..]; + + assert_multiline_eq!(from_utf8(&output.stderr).unwrap(), &expected); + assert_eq!(output.status.code(), Some(2)); +} + +#[test] +fn test_serde_basic_example_subcommand_args_and_env_success() { + let mut command = SerdeBasicExample::get_command(); + let output = command + .args(["--db-url=postgres://localhost/dev", "run-migrations"]) + .env("DB_RETRIES", "77") + .output() + .unwrap(); + + let expected = &" +ModelServiceConfig { + listen_addr: 127.0.0.1:9090, + auth: None, + db: HttpClientConfig { + url: postgres://localhost/dev, + retries: 77, + }, + config_file: None, + command: Some( + RunMigrations( + MigrationConfig { + sql_file: None, + }, + ), + ), +} +"[1..]; + + assert_eq!( + output.status.code(), + Some(0), + "stderr: {}", + from_utf8(&output.stderr).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stdout).unwrap(), &expected); +} + +#[test] +fn test_serde_basic_example_subcommand_args_and_env_success2() { + let mut command = SerdeBasicExample::get_command(); + let output = command + .args([ + "--db-url=postgres://localhost/dev", + "run-migrations", + "--sql-file=foobar.sql", + ]) + .env("DB_RETRIES", "77") + .output() + .unwrap(); + + let expected = &" +ModelServiceConfig { + listen_addr: 127.0.0.1:9090, + auth: None, + db: HttpClientConfig { + url: postgres://localhost/dev, + retries: 77, + }, + config_file: None, + command: Some( + RunMigrations( + MigrationConfig { + sql_file: Some( + \"foobar.sql\", + ), + }, + ), + ), +} +"[1..]; + + assert_eq!( + output.status.code(), + Some(0), + "stderr: {}", + from_utf8(&output.stderr).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stdout).unwrap(), &expected); +} + +#[test] +fn test_serde_basic_example_subcommand_args_and_file_success() { + let mut command = SerdeBasicExample::get_command(); + let toml_file = examples_dir().to_string() + "/model_service.toml"; + let output = command + .args([ + &format!("--config={toml_file}"), + "--db-url=postgres://localhost/dev", + "run-migrations", + ]) + .output() + .unwrap(); + + let expected = &format!( + " +ModelServiceConfig {{ + listen_addr: 0.0.0.0:80, + auth: None, + db: HttpClientConfig {{ + url: postgres://localhost/dev, + retries: 3, + }}, + config_file: Some( + \"{toml_file}\", + ), + command: Some( + RunMigrations( + MigrationConfig {{ + sql_file: Some( + \"xxx.sql\", + ), + }}, + ), + ), +}} +" + )[1..]; + + assert_eq!( + output.status.code(), + Some(0), + "stderr: {}", + from_utf8(&output.stderr).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stdout).unwrap(), &expected); +} + +#[test] +fn test_serde_basic_example_subcommand_args_and_file_success_shadowing() { + let mut command = SerdeBasicExample::get_command(); + let toml_file = examples_dir().to_string() + "/model_service.toml"; + let output = command + .args([ + "--db-url=postgres://localhost/dev", + &format!("--config={toml_file}"), + "run-migrations", + "--sql-file", + "foo.sql", + ]) + .output() + .unwrap(); + + let expected = &format!( + " +ModelServiceConfig {{ + listen_addr: 0.0.0.0:80, + auth: None, + db: HttpClientConfig {{ + url: postgres://localhost/dev, + retries: 3, + }}, + config_file: Some( + \"{toml_file}\", + ), + command: Some( + RunMigrations( + MigrationConfig {{ + sql_file: Some( + \"foo.sql\", + ), + }}, + ), + ), +}} +" + )[1..]; + + assert_eq!( + output.status.code(), + Some(0), + "stderr: {}", + from_utf8(&output.stderr).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stdout).unwrap(), &expected); +} + +#[test] +fn test_serde_basic_example_subcommand_args_and_env_file_success() { + let mut command = SerdeBasicExample::get_command(); + let toml_file = examples_dir().to_string() + "/model_service.toml"; + let output = command + .args(["--db-url=postgres://localhost/dev", "run-migrations"]) + .envs([ + ("CONFIG", toml_file.as_str()), + ("AUTH_URL", "http://auth.service.cluster"), + ("AUTH_RETRIES", "5"), + ]) + .output() + .unwrap(); + + let expected = &format!( + " +ModelServiceConfig {{ + listen_addr: 0.0.0.0:80, + auth: Some( + HttpClientConfig {{ + url: http://auth.service.cluster/, + retries: 5, + }}, + ), + db: HttpClientConfig {{ + url: postgres://localhost/dev, + retries: 3, + }}, + config_file: Some( + \"{toml_file}\", + ), + command: Some( + RunMigrations( + MigrationConfig {{ + sql_file: Some( + \"xxx.sql\", + ), + }}, + ), + ), +}} +" + )[1..]; + + assert_eq!( + output.status.code(), + Some(0), + "stderr: {}", + from_utf8(&output.stderr).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stdout).unwrap(), &expected); +} + +#[test] +fn test_serde_basic_example_subcommand_args_and_env_file_success_shadowing() { + let mut command = SerdeBasicExample::get_command(); + let toml_file = examples_dir().to_string() + "/model_service.toml"; + let output = command + .args([ + "--db-url=postgres://localhost/dev", + "run-migrations", + "--sql-file=foo.sql", + ]) + .envs([ + ("CONFIG", toml_file.as_str()), + ("SQL_FILE", "bar.sql"), + ("AUTH_URL", "http://auth.service.cluster"), + ("AUTH_RETRIES", "5"), + ]) + .output() + .unwrap(); + + let expected = &format!( + " +ModelServiceConfig {{ + listen_addr: 0.0.0.0:80, + auth: Some( + HttpClientConfig {{ + url: http://auth.service.cluster/, + retries: 5, + }}, + ), + db: HttpClientConfig {{ + url: postgres://localhost/dev, + retries: 3, + }}, + config_file: Some( + \"{toml_file}\", + ), + command: Some( + RunMigrations( + MigrationConfig {{ + sql_file: Some( + \"foo.sql\", + ), + }}, + ), + ), +}} +" + )[1..]; + + assert_eq!( + output.status.code(), + Some(0), + "stderr: {}", + from_utf8(&output.stderr).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stdout).unwrap(), &expected); +} + +#[test] +fn test_serde_basic_example_subcommand_args_and_env_file_missing_and_invalid() { + let mut command = SerdeBasicExample::get_command(); + let toml_file = examples_dir().to_string() + "/model_service.toml"; + let output = command + .args(["run-migrations", "--sql-file=foo.sql"]) + .envs([ + ("CONFIG", toml_file.as_str()), + ("SQL_FILE", "bar.sql"), + ("AUTH_RETRIES", "xxx"), + ]) + .output() + .unwrap(); + + let expected = &" +error: A required value was not provided + env 'AUTH_URL', or '--auth-url', must be provided + because env 'AUTH_RETRIES' was provided (enabling argument group HttpClientConfig @ .auth) + env 'DB_URL', or '--db-url', must be provided +error: Invalid value + when parsing env 'AUTH_RETRIES' value 'xxx': invalid digit found in string +"[1..]; + + assert_eq!( + output.status.code(), + Some(2), + "stdout: {}", + from_utf8(&output.stdout).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stderr).unwrap(), &expected); +} + +#[test] +fn test_serde_basic_example_subcommand_args_and_env_file_missing_and_invalid2() { + let mut command = SerdeBasicExample::get_command(); + let toml_file = examples_dir().to_string() + "/model_service2.toml"; + let output = command + .args(["run-migrations", "--sql-file=foo.sql"]) + .envs([ + ("CONFIG", toml_file.as_str()), + ("SQL_FILE", "bar.sql"), + ("AUTH_RETRIES", "yyy"), + ]) + .output() + .unwrap(); + + // This should NOT complain about unexpected arguments in model_service2.toml, + // because the subcommand section is skipped unless the subcommand is active. + let expected = &format!( + " +error: A required value was not provided + env 'AUTH_URL', or '--auth-url', must be provided + because env 'AUTH_RETRIES' was provided (enabling argument group HttpClientConfig @ .auth) + env 'DB_URL', or '--db-url', must be provided +error: Invalid value + when parsing env 'AUTH_RETRIES' value 'yyy': invalid digit found in string +error: Parsing document + Parsing {toml_file} (@ MigrationConfig): + unknown field `unexpected_arg`, expected `sql_file` + + Parsing {toml_file} (@ retries): + invalid type: string \"xxx\", expected u32 + in `retries` + +" + )[1..]; + + assert_eq!( + output.status.code(), + Some(2), + "stdout: {}", + from_utf8(&output.stdout).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stderr).unwrap(), &expected); +} diff --git a/tests/test_figment.rs b/tests/test_figment.rs new file mode 100644 index 0000000..8911c00 --- /dev/null +++ b/tests/test_figment.rs @@ -0,0 +1,217 @@ +#![cfg(feature = "serde")] + +mod common; +use common::{assert_multiline_eq, examples_dir, Example}; +use std::str::from_utf8; + +struct FigmentExample {} +impl Example for FigmentExample { + const NAME: &'static str = "figment"; +} + +#[test] +fn test_figment_example_no_args() { + let mut command = FigmentExample::get_command(); + let output = command.output().unwrap(); + + let expected = &" +error: A required value was not provided + env 'DB_RETRIES', or '--db-retries', must be provided + env 'DB_URL', or '--db-url', must be provided +"[1..]; + + assert_multiline_eq!(from_utf8(&output.stderr).unwrap(), &expected); + assert_eq!(output.status.code(), Some(2)); +} + +#[test] +fn test_figment_example_one_file_expected_failure() { + let mut command = FigmentExample::get_command(); + let toml_file = examples_dir().to_string() + "/model_service.toml"; + let output = command + .envs([("TOML", toml_file.as_str())]) + .output() + .unwrap(); + + let expected = &" +error: A required value was not provided + env 'DB_URL', or '--db-url', must be provided + +Help: + --db-url + Database: Base URL + [env: DB_URL] +"[1..]; + + assert_multiline_eq!(from_utf8(&output.stderr).unwrap(), &expected); + assert_eq!(output.status.code(), Some(2)); +} + +#[test] +fn test_figment_example_one_file_success() { + let mut command = FigmentExample::get_command(); + let toml_file = examples_dir().to_string() + "/model_service.toml"; + let output = command + .args(["--db-url", "postgres://localhost/dev"]) + .envs([("TOML", toml_file.as_str())]) + .output() + .unwrap(); + + let expected = &" +ModelServiceConfig { + listen_addr: 0.0.0.0:80, + auth: None, + db: HttpClientConfig { + url: postgres://localhost/dev, + retries: 3, + }, + config_file: None, + command: None, +} +"[1..]; + + assert_eq!( + output.status.code(), + Some(0), + "stderr: {}", + from_utf8(&output.stderr).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stdout).unwrap(), &expected); +} + +#[test] +fn test_figment_example_two_file_success() { + let mut command = FigmentExample::get_command(); + let toml_file = examples_dir().to_string() + "/model_service.toml"; + let toml_file2 = examples_dir().to_string() + "/model_service2.toml"; + let output = command + .args(["--db-url", "postgres://localhost/dev"]) + .envs([("TOML", toml_file2.as_str()), ("TOML2", toml_file.as_str())]) + .output() + .unwrap(); + + let expected = &" +ModelServiceConfig { + listen_addr: 0.0.0.0:80, + auth: None, + db: HttpClientConfig { + url: postgres://localhost/dev, + retries: 3, + }, + config_file: None, + command: None, +} +"[1..]; + + assert_eq!( + output.status.code(), + Some(0), + "stderr: {}", + from_utf8(&output.stderr).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stdout).unwrap(), &expected); +} + +#[test] +fn test_figment_example_two_file_expected_failure_other_order() { + let mut command = FigmentExample::get_command(); + let toml_file = examples_dir().to_string() + "/model_service.toml"; + let toml_file2 = examples_dir().to_string() + "/model_service2.toml"; + let output = command + .args(["--db-url", "postgres://localhost/dev"]) + .envs([("TOML", toml_file.as_str()), ("TOML2", toml_file2.as_str())]) + .output() + .unwrap(); + + let expected = &" +error: Parsing document + Parsing files (@ retries): invalid type: found string \"xxx\", expected u32 +"[1..]; + + assert_eq!( + output.status.code(), + Some(2), + "stdout: {}", + from_utf8(&output.stdout).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stderr).unwrap(), &expected); +} + +#[test] +fn test_figment_example_two_file_shadowing_failure_from_env() { + let mut command = FigmentExample::get_command(); + let toml_file = examples_dir().to_string() + "/model_service.toml"; + let toml_file2 = examples_dir().to_string() + "/model_service2.toml"; + let output = command + .args(["--db-url", "postgres://localhost/dev"]) + .envs([ + ("TOML", toml_file.as_str()), + ("TOML2", toml_file2.as_str()), + ("DB_RETRIES", "7"), + ]) + .output() + .unwrap(); + + // Trying to shadow DB_RETRIES doesn't prevent the error because the serde parser still fails, + // at least, that's the current behavior. + // + // Whether it's desirable, TBD. + // Note that we don't run the value_parser, but we do call serde next_value. + // + // The pros are, if your config file has junk in it, you find out now and not later + // when you remove an environment variable. + // The con is, a shadowed value is preventing the application from starting up. + let expected = &" +error: Parsing document + Parsing files (@ retries): invalid type: found string \"xxx\", expected u32 +"[1..]; + + assert_eq!( + output.status.code(), + Some(2), + "stdout: {}", + from_utf8(&output.stdout).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stderr).unwrap(), &expected); +} + +#[test] +fn test_figment_example_three_file_shadowing_success() { + let mut command = FigmentExample::get_command(); + let json_file = examples_dir().to_string() + "/model_service.json"; + let toml_file = examples_dir().to_string() + "/model_service.toml"; + let toml_file2 = examples_dir().to_string() + "/model_service2.toml"; + let output = command + .args(["--db-url", "postgres://localhost/dev"]) + .envs([ + ("TOML", toml_file.as_str()), + ("TOML2", toml_file2.as_str()), + ("JSON", json_file.as_str()), + ("DB_RETRIES", "7"), + ]) + .output() + .unwrap(); + + // This succeeds because the JSON overwrites the bad value in TOML2. + // This prevents us from trying to deserialize the bad value. + let expected = &" +ModelServiceConfig { + listen_addr: 0.0.0.0:81, + auth: None, + db: HttpClientConfig { + url: postgres://localhost/dev, + retries: 7, + }, + config_file: None, + command: None, +} +"[1..]; + + assert_eq!( + output.status.code(), + Some(0), + "stderr: {}", + from_utf8(&output.stderr).unwrap() + ); + assert_multiline_eq!(from_utf8(&output.stdout).unwrap(), &expected); +} diff --git a/tests/test_serde.rs b/tests/test_serde.rs new file mode 100644 index 0000000..23f1bdf --- /dev/null +++ b/tests/test_serde.rs @@ -0,0 +1,713 @@ +#![cfg(feature = "serde")] + +mod common; +use common::*; + +use conf::Conf; +use serde_json::json; + +#[derive(Conf, Debug)] +#[conf(serde)] +pub struct A { + #[arg(long, env)] + pub wiggle: i16, + #[arg(long, env)] + pub wobble: String, + #[arg(long, env)] + pub bobble: Option, +} + +#[test] +fn test_basic_serde() { + let result = A::conf_builder() + .args([".", "--wiggle=8"]) + .env([("WOBBLE", "xxx")]) + .doc( + "test_doc", + json! ({ + "bobble": 9 + }), + ) + .try_parse() + .unwrap(); + assert_eq!(result.wiggle, 8); + assert_eq!(result.wobble, "xxx"); + assert_eq!(result.bobble, Some(9)); + + let result = A::conf_builder() + .args([".", "--wiggle=8", "--bobble", "7"]) + .env([("WOBBLE", "xxx")]) + .doc( + "test_doc", + json! ({ + "bobble": 9 + }), + ) + .try_parse() + .unwrap(); + assert_eq!(result.wiggle, 8); + assert_eq!(result.wobble, "xxx"); + assert_eq!(result.bobble, Some(7)); + + assert_error_contains_text!( + A::conf_builder() + .args([".", "--bobble", "7"]) + .env([("WOBBLE", "xxx")]) + .doc( + "test_doc", + json! ({ + "bobble": 9 + }), + ) + .try_parse(), + ["env 'WIGGLE', or '--wiggle', must be provided"] + ); + + let result = A::conf_builder() + .args([".", "--bobble", "7"]) + .env([("WOBBLE", "xxx")]) + .doc( + "test_doc", + json! ({ + "bobble": 9, + "wiggle": -2 + }), + ) + .try_parse() + .unwrap(); + assert_eq!(result.wiggle, -2); + assert_eq!(result.wobble, "xxx"); + assert_eq!(result.bobble, Some(7)); + + let result = A::conf_builder() + .args([".", "--bobble", "7"]) + .env([("WOBBLE", "xxx")]) + .doc( + "test_doc", + json! ({ + "bobble": 9, + "wiggle": -2, + "wobble": "yyy" + }), + ) + .try_parse() + .unwrap(); + assert_eq!(result.wiggle, -2); + assert_eq!(result.wobble, "xxx"); + assert_eq!(result.bobble, Some(7)); + + let result = A::conf_builder() + .args([".", "--bobble", "7"]) + .env([("WOBBLY", "xxx")]) + .doc( + "test_doc", + json! ({ + "bobble": 9, + "wiggle": -2, + "wobble": "yyy" + }), + ) + .try_parse() + .unwrap(); + assert_eq!(result.wiggle, -2); + assert_eq!(result.wobble, "yyy"); + assert_eq!(result.bobble, Some(7)); + + assert_error_contains_text!( + A::conf_builder() + .args([".", "--bobble", "7", "--bobble=4"]) + .env([("WOBBLY", "xxx")]) + .doc( + "test_doc", + json! ({ + "bobble": 9, + "wiggle": -2, + "wobble": "yyy", + }), + ) + .try_parse(), + ["the argument '--bobble ' cannot be used multiple times"] + ); + + assert_error_contains_text!( + A::conf_builder() + .args([".", "--bobble", "7"]) + .env([("WOBBLY", "xxx")]) + .doc( + "test_doc", + json! ({ + "bobble": 9, + "wiggle": -2, + "wobble": "yyy", + "wobbly": "zzz" + }), + ) + .try_parse(), + ["Parsing test_doc (@ A): unknown field `wobbly`"] + ); + + assert_error_contains_text!( + A::conf_builder() + .args([".", "--bobble", "x", "--wiggle=o"]) + .env([("WOBBLY", "xxx")]) + .doc( + "test_doc", + json! ({ + "bobble": 9, + "wiggle": -2, + "wobble": "yyy", + "wobbly": "zzz", + "wubbly": "qqq", + }), + ) + .try_parse(), + [ + "when parsing '--wiggle' value 'o': invalid digit found in string", + "when parsing '--bobble' value 'x': invalid digit found in string", + "Parsing test_doc (@ A): unknown field `wobbly`", + "Parsing test_doc (@ A): unknown field `wubbly`", + ] + ); +} + +#[derive(Conf, Debug)] +#[conf(serde)] +pub struct B { + #[conf(flatten)] + a: A, + #[arg(short)] + f: bool, +} + +#[test] +fn test_serde_nested() { + assert_error_contains_text!( + B::conf_builder() + .args([".", "--wiggle=8"]) + .env([("WOBBLE", "xxx")]) + .doc( + "test_doc", + json! ({ + "bobble": 9 + }), + ) + .try_parse(), + ["Parsing test_doc (@ B): unknown field `bobble`"] + ); + + let result = B::conf_builder() + .args([".", "--wiggle=8"]) + .env([("WOBBLE", "xxx")]) + .doc( + "test_doc", + json! ({ + "a": { + "bobble": 9 + } + }), + ) + .try_parse() + .unwrap(); + assert!(!result.f); + assert_eq!(result.a.wiggle, 8); + assert_eq!(result.a.wobble, "xxx"); + assert_eq!(result.a.bobble, Some(9)); + + let result = B::conf_builder() + .args([".", "--wiggle=8"]) + .env([("WOBBLE", "xxx")]) + .doc( + "test_doc", + json! ({ + "a": { + "bobble": 9 + }, + "f": true + }), + ) + .try_parse() + .unwrap(); + assert!(result.f); + assert_eq!(result.a.wiggle, 8); + assert_eq!(result.a.wobble, "xxx"); + assert_eq!(result.a.bobble, Some(9)); + + assert_error_contains_text!( + B::conf_builder() + .args([".", "--wiggle=q"]) + .env([("WOBBLE", "xxx")]) + .doc( + "test_doc3", + json! ({ + "a": { + "bobble": "xxx" + }, + "f": 7, + "n": "q" + }), + ) + .try_parse(), + [ + "when parsing '--wiggle' value 'q': invalid digit found in string", + "Parsing test_doc3 (@ bobble): invalid type: string \"xxx\", expected i16", + "Parsing test_doc3 (@ f): invalid type: integer `7`, expected a boolean", + "Parsing test_doc3 (@ B): unknown field `n`, expected `a` or `f`" + ] + ); +} + +#[derive(Conf, Debug)] +#[conf(serde)] +pub struct C { + #[arg(repeat, long, env)] + out: Vec, + #[arg(repeat, long, env)] + p: Vec, +} + +#[test] +fn test_serde_repeat() { + let result = C::conf_builder() + .args([ + ".", "--out", "asdf", "--p", "1", "--out", "jkl", "--p", "-1", + ]) + .doc("test_doc", json!({})) + .try_parse() + .unwrap(); + assert_eq!(result.out, vec!["asdf", "jkl"]); + assert_eq!(result.p, vec![1, -1]); + + let result = C::conf_builder() + .args([".", "--out", "asdf", "--out", "jkl"]) + .doc("test_doc", json!({ "p": [1, -1]})) + .try_parse() + .unwrap(); + assert_eq!(result.out, vec!["asdf", "jkl"]); + assert_eq!(result.p, vec![1, -1]); + + let result = C::conf_builder() + .args([".", "--out", "asdf", "--p", "99", "--out", "jkl"]) + .doc("test_doc", json!({ "p": [1, -1]})) + .try_parse() + .unwrap(); + assert_eq!(result.out, vec!["asdf", "jkl"]); + assert_eq!(result.p, vec![99]); + + let result = C::conf_builder() + .args(["."]) + .doc("test_doc", json!({ "p": [1, -1], "out": ["asdf", "jkl"]})) + .try_parse() + .unwrap(); + assert_eq!(result.out, vec!["asdf", "jkl"]); + assert_eq!(result.p, vec![1, -1]); + + assert_error_contains_text!( + C::conf_builder() + .args(["."]) + .doc("test_doc", json!({ "out": [1, -1], "p": ["asdf", "jkl"]})) + .try_parse(), + [ + "Parsing test_doc (@ out): invalid type: integer `1`, expected a string", + "Parsing test_doc (@ p): invalid type: string \"asdf\", expected i64" + ] + ); +} + +#[derive(Conf, Debug)] +#[conf(serde)] +pub struct A2 { + #[arg(long, env)] + pub wiggle: i16, + #[arg(long, env)] + pub wobble: String, + #[arg(long, env, serde(use_value_parser))] + pub bobble: Option, + #[arg(repeat, long, env, serde(use_value_parser))] + pub out: Vec, +} + +#[test] +fn test_serde_use_value_parser() { + let result = A2::conf_builder() + .args([".", "--wiggle=8"]) + .env([("WOBBLE", "xxx")]) + .doc( + "test_doc", + json! ({ + "bobble": "9" + }), + ) + .try_parse() + .unwrap(); + assert_eq!(result.wiggle, 8); + assert_eq!(result.wobble, "xxx"); + assert_eq!(result.bobble, Some(9)); + assert!(result.out.is_empty()); + + assert_error_contains_text!( + A2::conf_builder() + .args([".", "--wiggle=8"]) + .env([("WOBBLE", "xxx")]) + .doc( + "test_doc", + json! ({ + "bobble": 9 + }), + ) + .try_parse(), + ["Parsing test_doc (@ bobble): invalid type: integer `9`, expected a string"] + ); + + let result = A2::conf_builder() + .args([".", "--wiggle=8"]) + .env([("WOBBLE", "xxx")]) + .doc( + "test_doc", + json! ({ + "bobble": "9", + "out": ["99", "44", "77"], + }), + ) + .try_parse() + .unwrap(); + assert_eq!(result.wiggle, 8); + assert_eq!(result.wobble, "xxx"); + assert_eq!(result.bobble, Some(9)); + assert_eq!(result.out, vec![99, 44, 77]); + + assert_error_contains_text!( + A2::conf_builder() + .args([".", "--wiggle=8"]) + .env([("WOBBLE", "xxx")]) + .doc( + "test_doc", + json! ({ + "bobble": "9", + "out": [99, 44, 77], + }), + ) + .try_parse(), + ["Parsing test_doc (@ out): invalid type: integer `99`, expected a string"] + ); +} + +// Custom data that implements FromStr but not serde::Deserialize +#[derive(Debug)] +pub struct CustomData { + val1: i64, + val2: i64, +} + +use std::str::FromStr; +impl FromStr for CustomData { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + let pieces = s.split(':').collect::>(); + if pieces.len() != 2 { + return Err("Expected one ':'"); + } + Ok(Self { + val1: FromStr::from_str(pieces[0]).map_err(|_| "Bad first number")?, + val2: FromStr::from_str(pieces[1]).map_err(|_| "Bad second number")?, + }) + } +} + +// A struct that implements Conf but not ConfSerde +#[derive(Conf, Debug)] +pub struct NotSerde { + #[arg(long, env)] + pub my_param: String, +} + +#[derive(Conf, Debug)] +#[conf(serde)] +pub struct TestSerdeSkip { + #[arg(short, env, serde(skip))] + pub f: bool, + #[arg(long, env, serde(skip))] + pub pair: CustomData, + #[arg(repeat, long, env, serde(skip))] + pub pairs: Vec, + #[conf(flatten, serde(skip))] + pub not_serde: NotSerde, + #[arg(long, env)] + pub val: u64, +} + +#[test] +fn test_serde_skip() { + let result = TestSerdeSkip::conf_builder() + .args([".", "--pair=2:3"]) + .env([("MY_PARAM", "Asdf"), ("VAL", "2")]) + .try_parse() + .unwrap(); + + assert!(!result.f); + assert_eq!(result.pair.val1, 2); + assert_eq!(result.pair.val2, 3); + assert!(result.pairs.is_empty()); + assert_eq!(result.not_serde.my_param, "Asdf"); + assert_eq!(result.val, 2); + + let result = TestSerdeSkip::conf_builder() + .args([".", "--pair=2:3"]) + .env([("MY_PARAM", "Asdf")]) + .doc("test", json!({"val": 2})) + .try_parse() + .unwrap(); + + assert!(!result.f); + assert_eq!(result.pair.val1, 2); + assert_eq!(result.pair.val2, 3); + assert!(result.pairs.is_empty()); + assert_eq!(result.not_serde.my_param, "Asdf"); + assert_eq!(result.val, 2); + + assert_error_contains_text!( + TestSerdeSkip::conf_builder() + .args([".", "--pair=2:3"]) + .env([("MY_PARAM", "Asdf")]) + .doc("test", json!({"f": true, "val": 2})) + .try_parse(), + ["Parsing test (@ TestSerdeSkip): unknown field `f`, expected `val`"] + ); + + assert_error_contains_text!( + TestSerdeSkip::conf_builder() + .args([".", "--pair=2:3"]) + .env([("MY_PARAM", "Asdf")]) + .doc( + "test", + json!({"not_serde": { "my_param": "Foo" }, "val": 2}) + ) + .try_parse(), + ["Parsing test (@ TestSerdeSkip): unknown field `not_serde`, expected `val`"] + ); + + assert_error_contains_text!( + TestSerdeSkip::conf_builder() + .args([".", "--pair=2:3"]) + .env([("MY_PARAM", "Asdf")]) + .doc("test", json!({"pair": "2:3", "val": 2})) + .try_parse(), + ["Parsing test (@ TestSerdeSkip): unknown field `pair`, expected `val`"] + ); + + assert_error_contains_text!( + TestSerdeSkip::conf_builder() + .args([".", "--pair=2:3"]) + .env([("MY_PARAM", "Asdf")]) + .doc("test", json!({"pairs": ["2:3"], "val": 2})) + .try_parse(), + ["Parsing test (@ TestSerdeSkip): unknown field `pairs`, expected `val`"] + ); + + let result = TestSerdeSkip::conf_builder() + .args([".", "--pair=2:3"]) + .env([("MY_PARAM", "Asdf"), ("PAIRS", "1:2,3:4,5:6")]) + .doc("test", json!({"val": 2})) + .try_parse() + .unwrap(); + + assert!(!result.f); + assert_eq!(result.pair.val1, 2); + assert_eq!(result.pair.val2, 3); + assert_eq!(result.pairs.len(), 3); + assert_eq!(result.pairs[0].val1, 1); + assert_eq!(result.pairs[0].val2, 2); + assert_eq!(result.pairs[1].val1, 3); + assert_eq!(result.pairs[1].val2, 4); + assert_eq!(result.pairs[2].val1, 5); + assert_eq!(result.pairs[2].val2, 6); + assert_eq!(result.not_serde.my_param, "Asdf"); + assert_eq!(result.val, 2); +} + +#[derive(Conf)] +#[conf(serde)] +pub struct E { + #[arg(long, default_value = "def", serde(rename = "p"))] + pub param: String, +} + +#[derive(Conf)] +#[conf(serde)] +pub struct D { + #[arg(long, serde(rename = "f"))] + pub force: bool, + #[conf(flatten, serde(rename = "a"))] + pub a2: A2, + #[conf(flatten, prefix, serde(rename = "a2"))] + pub b: B, + #[conf(long, serde(rename = "p"))] + pub p: String, + #[conf(repeat, long, env = "qs", serde(rename = "qs"))] + pub q: Vec, + #[conf(flatten)] + pub e: E, +} + +#[test] +fn test_serde_rename() { + let result = D::conf_builder() + .args([ + ".", + "--wiggle=4", + "--wobble=9", + "--b-wiggle=10", + "--b-wobble=14", + "--p=xyz", + ]) + .env::<&str, &str>([]) + .try_parse() + .unwrap(); + + assert!(!result.force); + assert_eq!(result.a2.wiggle, 4); + assert_eq!(result.a2.wobble, "9"); + assert_eq!(result.a2.bobble, None); + assert!(!result.b.f); + assert_eq!(result.b.a.wiggle, 10); + assert_eq!(result.b.a.wobble, "14"); + assert_eq!(result.b.a.bobble, None); + assert_eq!(result.p, "xyz"); + assert!(result.q.is_empty()); + assert_eq!(result.e.param, "def"); + + let result = D::conf_builder() + .args([ + ".", + "--wiggle=4", + "--wobble=9", + "--b-wiggle=10", + "--b-wobble=14", + "--p=xyz", + ]) + .env::<&str, &str>([]) + .doc("t.json", json!({ "f": true, "a2": { "f": true }})) + .try_parse() + .unwrap(); + + assert!(result.force); + assert_eq!(result.a2.wiggle, 4); + assert_eq!(result.a2.wobble, "9"); + assert_eq!(result.a2.bobble, None); + assert!(result.b.f); + assert_eq!(result.b.a.wiggle, 10); + assert_eq!(result.b.a.wobble, "14"); + assert_eq!(result.b.a.bobble, None); + assert_eq!(result.p, "xyz"); + assert!(result.q.is_empty()); + assert_eq!(result.e.param, "def"); + + let result = D::conf_builder() + .args([".", "--wiggle=4", "--wobble=9", "--b-wiggle=10", "--b-wobble=14", "--p=xyz"]) + .env::<&str, &str>([]) + .doc("t.json", json!({ "f": true, "a2": { "f": true, "a": {"wiggle": 7, "bobble": -8 }}, "e": { "p": "shadow" }})) + .try_parse() + .unwrap(); + + assert!(result.force); + assert_eq!(result.a2.wiggle, 4); + assert_eq!(result.a2.wobble, "9"); + assert_eq!(result.a2.bobble, None); + assert!(result.b.f); + assert_eq!(result.b.a.wiggle, 10); + assert_eq!(result.b.a.wobble, "14"); + assert_eq!(result.b.a.bobble, Some(-8)); + assert_eq!(result.p, "xyz"); + assert!(result.q.is_empty()); + assert_eq!(result.e.param, "shadow"); +} + +use conf::Subcommands; +#[derive(Subcommands, Debug)] +#[conf(serde)] +pub enum Commands { + A(A2), + B(B), +} + +#[derive(Conf, Debug)] +#[conf(serde)] +pub struct S { + #[arg(short)] + f: bool, + #[conf(subcommands)] + commands: Commands, +} + +#[test] +fn test_subcommands_serde() { + let result = S::conf_builder() + .args([".", "a", "--wiggle=4"]) + .env([("WOBBLE", "x")]) + .doc("t.json", json!({})) + .try_parse() + .unwrap(); + + assert!(!result.f); + let Commands::A(a2) = result.commands else { + panic!("unexpected enum value") + }; + assert_eq!(a2.wiggle, 4); + assert_eq!(a2.wobble, "x"); + assert_eq!(a2.bobble, None); + + assert_error_contains_text!( + S::conf_builder() + .args(["."]) + .env([("WOBBLE", "x")]) + .doc("t.json", json!({})) + .try_parse(), + ["Missing required subcommand"] + ); + + let result = S::conf_builder() + .args([".", "a"]) + .env([("LANG", "C")]) + .doc("t.json", json!({"a": { "wiggle": 4, "wobble": "x"}})) + .try_parse() + .unwrap(); + + assert!(!result.f); + let Commands::A(a2) = result.commands else { + panic!("unexpected enum value") + }; + assert_eq!(a2.wiggle, 4); + assert_eq!(a2.wobble, "x"); + assert_eq!(a2.bobble, None); + + let result = S::conf_builder() + .args([".", "a"]) + .env([("LANG", "C")]) + .doc("t.json", json!({"a": { "wiggle": 4, "wobble": "x"}, "b": {"f": true, "a": {"wiggle": 7, "wobble": "y"}}})) + .try_parse() + .unwrap(); + + assert!(!result.f); + let Commands::A(a2) = result.commands else { + panic!("unexpected enum value") + }; + assert_eq!(a2.wiggle, 4); + assert_eq!(a2.wobble, "x"); + assert_eq!(a2.bobble, None); + + let result = S::conf_builder() + .args([".", "b"]) + .env([("LANG", "C")]) + .doc("t.json", json!({"a": { "wiggle": 4, "wobble": "x"}, "b": {"f": true, "a": {"wiggle": 7, "wobble": "y"}}})) + .try_parse() + .unwrap(); + + assert!(!result.f); + let Commands::B(b) = result.commands else { + panic!("unexpected enum value") + }; + assert!(b.f); + assert_eq!(b.a.wiggle, 7); + assert_eq!(b.a.wobble, "y"); + assert_eq!(b.a.bobble, None); +} diff --git a/tests/test_showcase.rs b/tests/test_showcase_example.rs similarity index 90% rename from tests/test_showcase.rs rename to tests/test_showcase_example.rs index 6307433..98f897d 100644 --- a/tests/test_showcase.rs +++ b/tests/test_showcase_example.rs @@ -1,17 +1,15 @@ mod common; -use common::assert_multiline_eq; -use escargot::{CargoBuild, CargoRun}; -use std::{process::Command, str::from_utf8, sync::OnceLock}; +use common::{assert_multiline_eq, Example}; +use std::str::from_utf8; -fn get_built_showcase_example() -> Command { - static ONCE: OnceLock = OnceLock::new(); - ONCE.get_or_init(|| CargoBuild::new().example("showcase").run().unwrap()) - .command() +struct ShowcaseExample {} +impl Example for ShowcaseExample { + const NAME: &'static str = "showcase"; } #[test] fn test_showcase_example_no_args() { - let mut command = get_built_showcase_example(); + let mut command = ShowcaseExample::get_command(); let output = command.output().unwrap(); let expected = &" @@ -36,7 +34,7 @@ error: A required value was not provided #[test] fn test_showcase_example_some_invalid_args() { - let mut command = get_built_showcase_example(); + let mut command = ShowcaseExample::get_command(); let output = command .args(["--auth-retries=-3"]) .envs([("MYCO_TELEMETRY_RETRIES", "-2")]) @@ -66,7 +64,7 @@ error: Invalid value #[test] fn test_showcase_example_help() { - let mut command = get_built_showcase_example(); + let mut command = ShowcaseExample::get_command(); let output = command.args(["--help"]).output().unwrap(); let expected = &" @@ -144,7 +142,7 @@ Options: #[test] fn test_showcase_example_success_args() { - let mut command = get_built_showcase_example(); + let mut command = ShowcaseExample::get_command(); let output = command .args([ "--auth-url=https://example.com", @@ -176,11 +174,11 @@ Config { solver_service: SolveServiceConfig { listen_addr: 127.0.0.1:4040, auth: HttpClientConfig { - url: "https://example.com/", + url: https://example.com/, retries: 7, }, artifact: HttpClientConfig { - url: "https://what.com/", + url: https://what.com/, retries: 9, }, }, @@ -199,12 +197,12 @@ Config { round_limit: None, }, peer_urls: [ - "http://replica1.service.local/", - "http://replica2.service.local/", + http://replica1.service.local/, + http://replica2.service.local/, ], admin_listen_addr: 127.0.0.1:9090, telemetry: HttpClientConfig { - url: "https://far.scape/", + url: https://far.scape/, retries: 2, }, } @@ -216,7 +214,7 @@ Config { #[test] fn test_showcase_example_success_env() { - let mut command = get_built_showcase_example(); + let mut command = ShowcaseExample::get_command(); let output = command .envs([ ("MYCO_AUTH_URL", "https://example.com"), @@ -244,11 +242,11 @@ Config { solver_service: SolveServiceConfig { listen_addr: 127.0.0.1:4040, auth: HttpClientConfig { - url: "https://example.com/", + url: https://example.com/, retries: 7, }, artifact: HttpClientConfig { - url: "https://what.com/", + url: https://what.com/, retries: 9, }, }, @@ -267,12 +265,12 @@ Config { round_limit: None, }, peer_urls: [ - "http://replica1.service.local/", - "http://replica2.service.local/", + http://replica1.service.local/, + http://replica2.service.local/, ], admin_listen_addr: 127.0.0.1:9090, telemetry: HttpClientConfig { - url: "https://far.scape/", + url: https://far.scape/, retries: 2, }, } diff --git a/tests/test_showcase_subcommands.rs b/tests/test_subcommands_example.rs similarity index 80% rename from tests/test_showcase_subcommands.rs rename to tests/test_subcommands_example.rs index 1960848..78704e9 100644 --- a/tests/test_showcase_subcommands.rs +++ b/tests/test_subcommands_example.rs @@ -1,22 +1,15 @@ mod common; -use common::assert_multiline_eq; -use escargot::{CargoBuild, CargoRun}; -use std::{process::Command, str::from_utf8, sync::OnceLock}; - -fn get_built_showcase_subcommands_example() -> Command { - static ONCE: OnceLock = OnceLock::new(); - ONCE.get_or_init(|| { - CargoBuild::new() - .example("showcase_subcommands") - .run() - .unwrap() - }) - .command() +use common::{assert_multiline_eq, Example}; +use std::str::from_utf8; + +struct SubcommandsExample {} +impl Example for SubcommandsExample { + const NAME: &'static str = "subcommands_example"; } #[test] fn test_showcase_example_no_args() { - let mut command = get_built_showcase_subcommands_example(); + let mut command = SubcommandsExample::get_command(); let output = command.output().unwrap(); let expected = &" @@ -31,16 +24,17 @@ error: A required value was not provided #[test] fn test_showcase_example_some_invalid_args() { - let mut command = get_built_showcase_subcommands_example(); + let mut command = SubcommandsExample::get_command(); let output = command - .args(["--db-url=asdf"]) + .args(["--db-url=asdf:/"]) .envs([("DB_RETRIES", "5")]) .output() .unwrap(); + println!("{}", from_utf8(&output.stdout).unwrap()); let expected = &" error: Invalid value - when parsing '--db-url' value 'asdf': relative URL without a base + when parsing '--db-url' value 'asdf:/': invalid format Help: --db-url @@ -54,13 +48,13 @@ Help: #[test] fn test_showcase_example_help() { - let mut command = get_built_showcase_subcommands_example(); + let mut command = SubcommandsExample::get_command(); let output = command.args(["--help"]).output().unwrap(); let expected = &" Configuration for model service -Usage: showcase_subcommands [OPTIONS] [COMMAND] +Usage: subcommands_example [OPTIONS] [COMMAND] Commands: run-migrations @@ -88,7 +82,7 @@ Options: #[test] fn test_showcase_example_success_args() { - let mut command = get_built_showcase_subcommands_example(); + let mut command = SubcommandsExample::get_command(); let output = command .args([ "--auth-url=https://example.com", @@ -106,12 +100,12 @@ ModelServiceConfig { listen_addr: 127.0.0.1:9090, auth: Some( HttpClientConfig { - url: "https://example.com/", + url: https://example.com/, retries: 7, }, ), db: HttpClientConfig { - url: "postgres://localhost/dev", + url: postgres://localhost/dev, retries: 9, }, command: None, @@ -124,7 +118,7 @@ ModelServiceConfig { #[test] fn test_showcase_example_success_env() { - let mut command = get_built_showcase_subcommands_example(); + let mut command = SubcommandsExample::get_command(); let output = command .envs([ ("LISTEN_ADDR", "0.0.0.0:7777"), @@ -139,7 +133,7 @@ ModelServiceConfig { listen_addr: 0.0.0.0:7777, auth: None, db: HttpClientConfig { - url: "postgres://localhost/dev", + url: postgres://localhost/dev, retries: 3, }, command: None, @@ -152,11 +146,11 @@ ModelServiceConfig { #[test] fn test_showcase_example_subcommand_help() { - let mut command = get_built_showcase_subcommands_example(); + let mut command = SubcommandsExample::get_command(); let output = command.args(["run-migrations", "--help"]).output().unwrap(); let expected = &" -Usage: showcase_subcommands run-migrations [OPTIONS] +Usage: subcommands_example run-migrations [OPTIONS] Options: --migrations Path to migrations file (instead of embedded migrations) @@ -170,7 +164,7 @@ Options: #[test] fn test_showcase_example_subcommand_invalid_args() { - let mut command = get_built_showcase_subcommands_example(); + let mut command = SubcommandsExample::get_command(); let output = command .args(["run-migrations", "--db-url=postgres://localhost/dev"]) .output() @@ -179,7 +173,7 @@ fn test_showcase_example_subcommand_invalid_args() { let expected = &" error: unexpected argument '--db-url' found -Usage: showcase_subcommands run-migrations [OPTIONS] +Usage: subcommands_example run-migrations [OPTIONS] For more information, try '--help'. "[1..]; @@ -190,7 +184,7 @@ For more information, try '--help'. #[test] fn test_showcase_example_subcommand_missing_args() { - let mut command = get_built_showcase_subcommands_example(); + let mut command = SubcommandsExample::get_command(); let output = command .args(["--db-url=postgres://localhost/dev", "run-migrations"]) .output() @@ -212,7 +206,7 @@ Help: #[test] fn test_showcase_example_subcommand_success() { - let mut command = get_built_showcase_subcommands_example(); + let mut command = SubcommandsExample::get_command(); let output = command .args(["--db-url=postgres://localhost/dev", "run-migrations"]) .env("DB_RETRIES", "77") @@ -224,7 +218,7 @@ ModelServiceConfig { listen_addr: 127.0.0.1:9090, auth: None, db: HttpClientConfig { - url: \"postgres://localhost/dev\", + url: postgres://localhost/dev, retries: 77, }, command: Some( @@ -243,7 +237,7 @@ ModelServiceConfig { #[test] fn test_showcase_example_subcommand_success2() { - let mut command = get_built_showcase_subcommands_example(); + let mut command = SubcommandsExample::get_command(); let output = command .args([ "--db-url=postgres://localhost/dev", @@ -259,7 +253,7 @@ ModelServiceConfig { listen_addr: 127.0.0.1:9090, auth: None, db: HttpClientConfig { - url: \"postgres://localhost/dev\", + url: postgres://localhost/dev, retries: 77, }, command: Some(