From bc762dc98d1fc84b373741136988ffa1fbc8d867 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Thu, 12 Dec 2024 11:45:46 +0200 Subject: [PATCH 01/42] add upsert signed docs query --- catalyst-gateway/bin/Cargo.toml | 1 + catalyst-gateway/bin/src/db/event/mod.rs | 3 ++- .../bin/src/db/event/signed_docs/mod.rs | 24 +++++++++++++++++++ .../sql/upsert_signed_documents.sql | 22 +++++++++++++++++ 4 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 catalyst-gateway/bin/src/db/event/signed_docs/mod.rs create mode 100644 catalyst-gateway/bin/src/db/event/signed_docs/sql/upsert_signed_documents.sql diff --git a/catalyst-gateway/bin/Cargo.toml b/catalyst-gateway/bin/Cargo.toml index fbba8109c1b..50c84b17540 100644 --- a/catalyst-gateway/bin/Cargo.toml +++ b/catalyst-gateway/bin/Cargo.toml @@ -43,6 +43,7 @@ tokio-postgres = { version = "0.7.12", features = [ "with-chrono-0_4", "with-serde_json-1", "with-time-0_3", + "with-uuid-1" ] } tokio = { version = "1.41.0", features = ["rt", "macros", "rt-multi-thread"] } dotenvy = "0.15.7" diff --git a/catalyst-gateway/bin/src/db/event/mod.rs b/catalyst-gateway/bin/src/db/event/mod.rs index e0e28b1b923..1bc4bd1d476 100644 --- a/catalyst-gateway/bin/src/db/event/mod.rs +++ b/catalyst-gateway/bin/src/db/event/mod.rs @@ -18,6 +18,7 @@ pub(crate) mod config; pub(crate) mod error; pub(crate) mod legacy; pub(crate) mod schema_check; +pub(crate) mod signed_docs; /// Database version this crate matches. /// Must equal the last Migrations Version Number from `event-db/migrations`. @@ -212,7 +213,7 @@ impl EventDB { /// /// The env var "`DATABASE_URL`" can be set directly as an anv var, or in a /// `.env` file. -pub(crate) fn establish_connection() { +pub fn establish_connection() { let (url, user, pass) = Settings::event_db_settings(); // This was pre-validated and can't fail, but provide default in the impossible case it diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs new file mode 100644 index 00000000000..333a9eb33c3 --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -0,0 +1,24 @@ +//! Signed docs queries + +use super::EventDB; + +/// Upsert sql query +const UPSERT_SIGNED_DOCS: &str = include_str!("./sql/upsert_signed_documents.sql"); + +/// Make an upsert query into the `event-db` by adding data into the `signed_docs` table +/// +/// * IF the record primary key (id,ver) does not exist, then add the new record. Return +/// success. +/// * IF the record does exist, but all values are the same as stored, return Success. +/// * Otherwise return an error. (Can not over-write an existing record with new data). +#[allow(dead_code)] +pub(crate) async fn upsert_signed_docs( + id: &uuid::Uuid, ver: &uuid::Uuid, doc_type: &uuid::Uuid, author: &String, + metadata: &serde_json::Value, payload: &serde_json::Value, raw: &Vec, +) -> anyhow::Result<()> { + EventDB::modify(UPSERT_SIGNED_DOCS, &[ + id, ver, doc_type, author, metadata, payload, raw, + ]) + .await?; + Ok(()) +} diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/sql/upsert_signed_documents.sql b/catalyst-gateway/bin/src/db/event/signed_docs/sql/upsert_signed_documents.sql new file mode 100644 index 00000000000..b5d79222259 --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/signed_docs/sql/upsert_signed_documents.sql @@ -0,0 +1,22 @@ +INSERT INTO signed_docs +( + id, + ver, + type, + author, + metadata, + payload, + raw +) +VALUES +($1, $2, $3, $4, $5, $6, $7) +ON CONFLICT (id, ver) DO UPDATE +SET +type = signed_docs.type + +WHERE +signed_docs.type = excluded.type +AND signed_docs.author = excluded.author +AND signed_docs.metadata = excluded.metadata +AND signed_docs.payload = excluded.payload +AND signed_docs.raw = excluded.raw From 86860383c850559e1ef4d7d96776c14539de8998 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Fri, 13 Dec 2024 09:40:07 +0200 Subject: [PATCH 02/42] add integration tests for signed docs query --- catalyst-gateway/Earthfile | 16 ++++++++ catalyst-gateway/bin/Cargo.toml | 2 +- .../bin/src/db/event/signed_docs/mod.rs | 3 ++ .../bin/src/db/event/signed_docs/tests/mod.rs | 37 +++++++++++++++++++ catalyst-gateway/docker-compose.yml | 12 ++++-- 5 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs diff --git a/catalyst-gateway/Earthfile b/catalyst-gateway/Earthfile index 1cde02848f7..bde5adb213f 100644 --- a/catalyst-gateway/Earthfile +++ b/catalyst-gateway/Earthfile @@ -124,3 +124,19 @@ check-builder-src-cache: RUN diff ../src_fingerprint.txt ../src_fingerprint_uncached.txt \ || (echo "ERROR: Source fingerprints do not match. Caching Error Detected!!" && exit 1) \ && echo "Source fingerprints match. Caching OK." + +test: + FROM +builder-src + + COPY docker-compose.yml . + + ENV EVENT_DB_URL "postgres://catalyst-event-dev:CHANGE_ME@localhost/CatalystEventDev" + + WITH DOCKER \ + --compose "./docker-compose.yml" \ + --load ./event-db+build \ + --pull alpine:3.20.3 \ + --service event-db-is-running + RUN --mount=$EARTHLY_RUST_CARGO_HOME_CACHE --mount=$EARTHLY_RUST_TARGET_CACHE \ + cargo nextest run --release --run-ignored=only signed_docs + END \ No newline at end of file diff --git a/catalyst-gateway/bin/Cargo.toml b/catalyst-gateway/bin/Cargo.toml index 50c84b17540..4f664ec101d 100644 --- a/catalyst-gateway/bin/Cargo.toml +++ b/catalyst-gateway/bin/Cargo.toml @@ -78,7 +78,7 @@ poem-openapi = { version = "5.1.2", features = [ "url", "chrono", ] } -uuid = { version = "1.11.0", features = ["v4", "serde"] } +uuid = { version = "1.11.0", features = ["v4", "v7", "serde"] } ulid = { version = "1.1.3", features = ["serde", "uuid"] } blake2b_simd = "1.0.2" url = "2.5.3" diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index 333a9eb33c3..3891c9c360c 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -1,5 +1,8 @@ //! Signed docs queries +#[cfg(test)] +mod tests; + use super::EventDB; /// Upsert sql query diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs new file mode 100644 index 00000000000..c91a69c380a --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -0,0 +1,37 @@ +//! Integration tests of the `signed docs` queries + +use super::*; +use crate::db::event::establish_connection; + +#[ignore = "An integration test which requires a running EventDB instance, disabled from `testunit` CI run"] +#[tokio::test] +async fn some_test() { + establish_connection(); + + let docs = [ + ( + uuid::Uuid::now_v7(), + uuid::Uuid::now_v7(), + uuid::Uuid::now_v7(), + "Alex".to_string(), + serde_json::Value::Null, + serde_json::Value::Null, + vec![1, 2, 3, 4], + ), + ( + uuid::Uuid::now_v7(), + uuid::Uuid::now_v7(), + uuid::Uuid::now_v7(), + "Steven".to_string(), + serde_json::Value::Null, + serde_json::Value::Null, + vec![5, 6, 7, 8], + ), + ]; + + for (id, ver, doc_type, author, metadata, payload, raw) in &docs { + upsert_signed_docs(id, ver, doc_type, author, metadata, payload, raw) + .await + .unwrap(); + } +} diff --git a/catalyst-gateway/docker-compose.yml b/catalyst-gateway/docker-compose.yml index 4b2d833735b..00b2a348764 100644 --- a/catalyst-gateway/docker-compose.yml +++ b/catalyst-gateway/docker-compose.yml @@ -1,10 +1,7 @@ -version: "3" - services: event-db: image: event-db:latest environment: - # Required environment variables for migrations - DB_HOST=localhost - DB_PORT=5432 - DB_NAME=CatalystEventDev @@ -16,7 +13,6 @@ services: - INIT_AND_DROP_DB=true - WITH_MIGRATIONS=true - - WITH_SEED_DATA=true ports: - 5432:5432 healthcheck: @@ -25,6 +21,14 @@ services: timeout: 5s retries: 10 +# it is a helper service to wait until the event-db will be ready +# mainly its a trick for Earthly how to wait until service will be fully functional + event-db-is-running: + image: alpine:3.20.3 + depends_on: + event-db: + condition: service_healthy + cat-gateway: image: cat-gateway:latest environment: From c966f61707e511c1afdb7aec7c83a3b5e35b5082 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Fri, 13 Dec 2024 10:54:42 +0200 Subject: [PATCH 03/42] wip --- catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index c91a69c380a..31017f995d0 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -33,5 +33,9 @@ async fn some_test() { upsert_signed_docs(id, ver, doc_type, author, metadata, payload, raw) .await .unwrap(); + // try to insert the same data again + upsert_signed_docs(id, ver, doc_type, author, metadata, payload, raw) + .await + .unwrap(); } } From 40d039ee854c67f42b1afd6f16a4c627efedba91 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Fri, 13 Dec 2024 11:03:45 +0200 Subject: [PATCH 04/42] fix --- .../bin/src/db/event/signed_docs/mod.rs | 13 +++++++++++++ .../bin/src/db/event/signed_docs/tests/mod.rs | 4 ++-- catalyst-gateway/blueprint.cue | 5 +++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index 3891c9c360c..4d9346715fa 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -19,6 +19,19 @@ pub(crate) async fn upsert_signed_docs( id: &uuid::Uuid, ver: &uuid::Uuid, doc_type: &uuid::Uuid, author: &String, metadata: &serde_json::Value, payload: &serde_json::Value, raw: &Vec, ) -> anyhow::Result<()> { + anyhow::ensure!( + id.get_version() == Some(uuid::Version::SortRand), + "`id` must be a UUID v7" + ); + anyhow::ensure!( + ver.get_version() == Some(uuid::Version::SortRand), + "`ver` must be a UUID v7" + ); + anyhow::ensure!( + doc_type.get_version() == Some(uuid::Version::Random), + "`doc_type` must be a UUID v4" + ); + EventDB::modify(UPSERT_SIGNED_DOCS, &[ id, ver, doc_type, author, metadata, payload, raw, ]) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index 31017f995d0..39c585df5f3 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -12,7 +12,7 @@ async fn some_test() { ( uuid::Uuid::now_v7(), uuid::Uuid::now_v7(), - uuid::Uuid::now_v7(), + uuid::Uuid::new_v4(), "Alex".to_string(), serde_json::Value::Null, serde_json::Value::Null, @@ -21,7 +21,7 @@ async fn some_test() { ( uuid::Uuid::now_v7(), uuid::Uuid::now_v7(), - uuid::Uuid::now_v7(), + uuid::Uuid::new_v4(), "Steven".to_string(), serde_json::Value::Null, serde_json::Value::Null, diff --git a/catalyst-gateway/blueprint.cue b/catalyst-gateway/blueprint.cue index c81c19d56c6..9e36433f3f4 100644 --- a/catalyst-gateway/blueprint.cue +++ b/catalyst-gateway/blueprint.cue @@ -12,4 +12,9 @@ project: { } } } + ci: { + targets: { + test: privileged: true + } + } } From 7e11ced3bea18d2c7855fef99cecf1807fabc9da Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Fri, 13 Dec 2024 11:13:42 +0200 Subject: [PATCH 05/42] wip --- .../bin/src/db/event/signed_docs/mod.rs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index 4d9346715fa..3891c9c360c 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -19,19 +19,6 @@ pub(crate) async fn upsert_signed_docs( id: &uuid::Uuid, ver: &uuid::Uuid, doc_type: &uuid::Uuid, author: &String, metadata: &serde_json::Value, payload: &serde_json::Value, raw: &Vec, ) -> anyhow::Result<()> { - anyhow::ensure!( - id.get_version() == Some(uuid::Version::SortRand), - "`id` must be a UUID v7" - ); - anyhow::ensure!( - ver.get_version() == Some(uuid::Version::SortRand), - "`ver` must be a UUID v7" - ); - anyhow::ensure!( - doc_type.get_version() == Some(uuid::Version::Random), - "`doc_type` must be a UUID v4" - ); - EventDB::modify(UPSERT_SIGNED_DOCS, &[ id, ver, doc_type, author, metadata, payload, raw, ]) From dfee5640a3793c9108d54375fb3a3bba2a2e77a1 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Fri, 13 Dec 2024 11:30:19 +0200 Subject: [PATCH 06/42] wip --- .../bin/src/db/event/signed_docs/mod.rs | 2 +- .../bin/src/db/event/signed_docs/tests/mod.rs | 24 +++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index 3891c9c360c..75bedf1f073 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -17,7 +17,7 @@ const UPSERT_SIGNED_DOCS: &str = include_str!("./sql/upsert_signed_documents.sql #[allow(dead_code)] pub(crate) async fn upsert_signed_docs( id: &uuid::Uuid, ver: &uuid::Uuid, doc_type: &uuid::Uuid, author: &String, - metadata: &serde_json::Value, payload: &serde_json::Value, raw: &Vec, + metadata: &Option, payload: &Option, raw: &Vec, ) -> anyhow::Result<()> { EventDB::modify(UPSERT_SIGNED_DOCS, &[ id, ver, doc_type, author, metadata, payload, raw, diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index 39c585df5f3..35577451949 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -14,8 +14,8 @@ async fn some_test() { uuid::Uuid::now_v7(), uuid::Uuid::new_v4(), "Alex".to_string(), - serde_json::Value::Null, - serde_json::Value::Null, + &Some(serde_json::Value::Null), + &Some(serde_json::Value::Null), vec![1, 2, 3, 4], ), ( @@ -23,10 +23,19 @@ async fn some_test() { uuid::Uuid::now_v7(), uuid::Uuid::new_v4(), "Steven".to_string(), - serde_json::Value::Null, - serde_json::Value::Null, + &Some(serde_json::Value::Null), + &Some(serde_json::Value::Null), vec![5, 6, 7, 8], ), + ( + uuid::Uuid::now_v7(), + uuid::Uuid::now_v7(), + uuid::Uuid::new_v4(), + "Sasha".to_string(), + &None, + &None, + vec![9, 10, 11, 12], + ), ]; for (id, ver, doc_type, author, metadata, payload, raw) in &docs { @@ -37,5 +46,12 @@ async fn some_test() { upsert_signed_docs(id, ver, doc_type, author, metadata, payload, raw) .await .unwrap(); + + let another_author = "Neil".to_string(); + assert!( + upsert_signed_docs(id, ver, doc_type, &another_author, metadata, payload, raw) + .await + .is_err() + ); } } From 0d2dc8030d0637ae29c1fa346e0a10b8e02b9946 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Fri, 13 Dec 2024 12:51:25 +0200 Subject: [PATCH 07/42] wip --- catalyst-gateway/bin/src/db/event/signed_docs/mod.rs | 5 +++++ .../bin/src/db/event/signed_docs/tests/mod.rs | 7 ------- .../event-db/migrations/V2__signed_documents.sql | 8 ++++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index 75bedf1f073..61cf58551cc 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -14,6 +14,11 @@ const UPSERT_SIGNED_DOCS: &str = include_str!("./sql/upsert_signed_documents.sql /// success. /// * IF the record does exist, but all values are the same as stored, return Success. /// * Otherwise return an error. (Can not over-write an existing record with new data). +/// +/// # Arguments: +/// - `id` is a UUID v7 +/// - `ver` is a UUID v7 +/// - `doc_type` is a UUID v4 #[allow(dead_code)] pub(crate) async fn upsert_signed_docs( id: &uuid::Uuid, ver: &uuid::Uuid, doc_type: &uuid::Uuid, author: &String, diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index 35577451949..bec0b04d179 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -46,12 +46,5 @@ async fn some_test() { upsert_signed_docs(id, ver, doc_type, author, metadata, payload, raw) .await .unwrap(); - - let another_author = "Neil".to_string(); - assert!( - upsert_signed_docs(id, ver, doc_type, &another_author, metadata, payload, raw) - .await - .is_err() - ); } } diff --git a/catalyst-gateway/event-db/migrations/V2__signed_documents.sql b/catalyst-gateway/event-db/migrations/V2__signed_documents.sql index 523680ef9f2..28ff361d4a3 100644 --- a/catalyst-gateway/event-db/migrations/V2__signed_documents.sql +++ b/catalyst-gateway/event-db/migrations/V2__signed_documents.sql @@ -12,11 +12,11 @@ -- Signed Documents Storage Repository defintion. CREATE TABLE IF NOT EXISTS signed_docs ( - id UUID NOT NULL, -- Actually a ULID - ver UUID NOT NULL, -- Actually a ULID - type UUID NOT NULL, -- Yes its a UUID this time + id UUID NOT NULL, -- UUID v7 + ver UUID NOT NULL, -- UUID v7 + type UUID NOT NULL, -- UUID v4 author TEXT NOT NULL, - metadata JSONB NOT NULL, + metadata JSONB NULL, payload JSONB NULL, raw BYTEA NOT NULL, From d14c42829dcbc2309dab1ccc6bf631aa77d5f2bf Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Fri, 13 Dec 2024 13:10:48 +0200 Subject: [PATCH 08/42] remake query from upsert to insert --- .../bin/src/db/event/signed_docs/mod.rs | 10 ++++----- .../sql/insert_signed_documents.sql | 12 ++++++++++ .../sql/upsert_signed_documents.sql | 22 ------------------- .../bin/src/db/event/signed_docs/tests/mod.rs | 10 ++++----- 4 files changed, 22 insertions(+), 32 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/event/signed_docs/sql/insert_signed_documents.sql delete mode 100644 catalyst-gateway/bin/src/db/event/signed_docs/sql/upsert_signed_documents.sql diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index 61cf58551cc..19ae3a854b3 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -5,10 +5,10 @@ mod tests; use super::EventDB; -/// Upsert sql query -const UPSERT_SIGNED_DOCS: &str = include_str!("./sql/upsert_signed_documents.sql"); +/// Insert sql query +const INSERT_SIGNED_DOCS: &str = include_str!("./sql/insert_signed_documents.sql"); -/// Make an upsert query into the `event-db` by adding data into the `signed_docs` table +/// Make an insert query into the `event-db` by adding data into the `signed_docs` table /// /// * IF the record primary key (id,ver) does not exist, then add the new record. Return /// success. @@ -20,11 +20,11 @@ const UPSERT_SIGNED_DOCS: &str = include_str!("./sql/upsert_signed_documents.sql /// - `ver` is a UUID v7 /// - `doc_type` is a UUID v4 #[allow(dead_code)] -pub(crate) async fn upsert_signed_docs( +pub(crate) async fn insert_signed_docs( id: &uuid::Uuid, ver: &uuid::Uuid, doc_type: &uuid::Uuid, author: &String, metadata: &Option, payload: &Option, raw: &Vec, ) -> anyhow::Result<()> { - EventDB::modify(UPSERT_SIGNED_DOCS, &[ + EventDB::modify(INSERT_SIGNED_DOCS, &[ id, ver, doc_type, author, metadata, payload, raw, ]) .await?; diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/sql/insert_signed_documents.sql b/catalyst-gateway/bin/src/db/event/signed_docs/sql/insert_signed_documents.sql new file mode 100644 index 00000000000..61183ea693a --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/signed_docs/sql/insert_signed_documents.sql @@ -0,0 +1,12 @@ +INSERT INTO signed_docs +( + id, + ver, + type, + author, + metadata, + payload, + raw +) +VALUES +($1, $2, $3, $4, $5, $6, $7) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/sql/upsert_signed_documents.sql b/catalyst-gateway/bin/src/db/event/signed_docs/sql/upsert_signed_documents.sql deleted file mode 100644 index b5d79222259..00000000000 --- a/catalyst-gateway/bin/src/db/event/signed_docs/sql/upsert_signed_documents.sql +++ /dev/null @@ -1,22 +0,0 @@ -INSERT INTO signed_docs -( - id, - ver, - type, - author, - metadata, - payload, - raw -) -VALUES -($1, $2, $3, $4, $5, $6, $7) -ON CONFLICT (id, ver) DO UPDATE -SET -type = signed_docs.type - -WHERE -signed_docs.type = excluded.type -AND signed_docs.author = excluded.author -AND signed_docs.metadata = excluded.metadata -AND signed_docs.payload = excluded.payload -AND signed_docs.raw = excluded.raw diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index bec0b04d179..a65c9206adc 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -39,12 +39,12 @@ async fn some_test() { ]; for (id, ver, doc_type, author, metadata, payload, raw) in &docs { - upsert_signed_docs(id, ver, doc_type, author, metadata, payload, raw) - .await - .unwrap(); - // try to insert the same data again - upsert_signed_docs(id, ver, doc_type, author, metadata, payload, raw) + insert_signed_docs(id, ver, doc_type, author, metadata, payload, raw) .await .unwrap(); + // // try to insert the same data again + // insert_signed_docs(id, ver, doc_type, author, metadata, payload, raw) + // .await + // .unwrap(); } } From c872de4357bc557984e6b5fc9e26b7b6e81ceaef Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Fri, 13 Dec 2024 16:59:33 +0200 Subject: [PATCH 09/42] add simple select signed docs query --- catalyst-gateway/bin/Cargo.toml | 1 + .../bin/src/db/event/signed_docs/mod.rs | 83 +++++++++++++++++-- .../sql/select_signed_documents.sql.jinja | 13 +++ .../bin/src/db/event/signed_docs/tests/mod.rs | 66 ++++++++------- catalyst-gateway/bin/src/jinja.rs | 30 +++++++ catalyst-gateway/bin/src/main.rs | 1 + 6 files changed, 158 insertions(+), 36 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/event/signed_docs/sql/select_signed_documents.sql.jinja create mode 100644 catalyst-gateway/bin/src/jinja.rs diff --git a/catalyst-gateway/bin/Cargo.toml b/catalyst-gateway/bin/Cargo.toml index 4f664ec101d..2d52f701cf4 100644 --- a/catalyst-gateway/bin/Cargo.toml +++ b/catalyst-gateway/bin/Cargo.toml @@ -95,6 +95,7 @@ jsonschema = "0.26.1" bech32 = "0.11.0" const_format = "0.2.33" regex = "1.11.1" +minijinja = "2.5.0" [dev-dependencies] proptest = "1.5.0" diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index 19ae3a854b3..10dd59f61ed 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -4,10 +4,36 @@ mod tests; use super::EventDB; +use crate::jinja::{JinjaTemplateSource, JINJA_ENV}; /// Insert sql query const INSERT_SIGNED_DOCS: &str = include_str!("./sql/insert_signed_documents.sql"); +/// Select sql query jinja template +pub(crate) const SELECT_SIGNED_DOCS_TEMPLATE: JinjaTemplateSource = JinjaTemplateSource { + name: "select_signed_documents.jinja.template", + source: include_str!("./sql/select_signed_documents.sql.jinja"), +}; + +/// signed doc event db struct +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct SignedDoc { + /// `signed_doc` table `id` field + id: uuid::Uuid, + /// `signed_doc` table `ver` field + ver: uuid::Uuid, + /// `signed_doc` table `type` field + doc_type: uuid::Uuid, + /// `signed_doc` table `author` field + author: String, + /// `signed_doc` table `metadata` field + metadata: Option, + /// `signed_doc` table `payload` field + payload: Option, + /// `signed_doc` table `raw` field + raw: Vec, +} + /// Make an insert query into the `event-db` by adding data into the `signed_docs` table /// /// * IF the record primary key (id,ver) does not exist, then add the new record. Return @@ -20,13 +46,60 @@ const INSERT_SIGNED_DOCS: &str = include_str!("./sql/insert_signed_documents.sql /// - `ver` is a UUID v7 /// - `doc_type` is a UUID v4 #[allow(dead_code)] -pub(crate) async fn insert_signed_docs( - id: &uuid::Uuid, ver: &uuid::Uuid, doc_type: &uuid::Uuid, author: &String, - metadata: &Option, payload: &Option, raw: &Vec, -) -> anyhow::Result<()> { +pub(crate) async fn insert_signed_docs(doc: &SignedDoc) -> anyhow::Result<()> { EventDB::modify(INSERT_SIGNED_DOCS, &[ - id, ver, doc_type, author, metadata, payload, raw, + &doc.id, + &doc.ver, + &doc.doc_type, + &doc.author, + &doc.metadata, + &doc.payload, + &doc.raw, ]) .await?; Ok(()) } + +/// Make an select query into the `event-db` by getting data from the `signed_docs` table +/// +/// * This returns a single document. All data from the document is returned, including +/// the `payload` and `raw` fields. +/// * `ver` should be able to be optional, in which case get the latest ver of the given +/// `id`. +/// +/// # Arguments: +/// - `id` is a UUID v7 +/// - `ver` is a UUID v7 +#[allow(dead_code)] +pub(crate) async fn select_signed_docs( + id: &uuid::Uuid, ver: &Option, +) -> anyhow::Result { + let query_template = JINJA_ENV.get_template(SELECT_SIGNED_DOCS_TEMPLATE.name)?; + let query = query_template.render(serde_json::json!({ + "id": id, + "ver": ver, + }))?; + let res = EventDB::query_one(&query, &[]).await?; + + let ver = if let Some(ver) = ver { + *ver + } else { + res.try_get("ver")? + }; + + let doc_type = res.try_get("type")?; + let author = res.try_get("author")?; + let metadata = res.try_get("metadata")?; + let payload = res.try_get("payload")?; + let raw = res.try_get("raw")?; + + Ok(SignedDoc { + id: *id, + ver, + doc_type, + author, + metadata, + payload, + raw, + }) +} diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/sql/select_signed_documents.sql.jinja b/catalyst-gateway/bin/src/db/event/signed_docs/sql/select_signed_documents.sql.jinja new file mode 100644 index 00000000000..1ed2397a831 --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/signed_docs/sql/select_signed_documents.sql.jinja @@ -0,0 +1,13 @@ +SELECT + {% if not ver %} signed_docs.ver, {% endif %} + signed_docs.type, + signed_docs.author, + signed_docs.metadata, + signed_docs.payload, + signed_docs.raw +FROM signed_docs +WHERE + signed_docs.id = '{{ id }}' + {% if ver %} AND signed_docs.ver = '{{ ver }}' {% endif %} +ORDER BY signed_docs.ver DESC +LIMIT 1 diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index a65c9206adc..1bd3e679d35 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -9,39 +9,43 @@ async fn some_test() { establish_connection(); let docs = [ - ( - uuid::Uuid::now_v7(), - uuid::Uuid::now_v7(), - uuid::Uuid::new_v4(), - "Alex".to_string(), - &Some(serde_json::Value::Null), - &Some(serde_json::Value::Null), - vec![1, 2, 3, 4], - ), - ( - uuid::Uuid::now_v7(), - uuid::Uuid::now_v7(), - uuid::Uuid::new_v4(), - "Steven".to_string(), - &Some(serde_json::Value::Null), - &Some(serde_json::Value::Null), - vec![5, 6, 7, 8], - ), - ( - uuid::Uuid::now_v7(), - uuid::Uuid::now_v7(), - uuid::Uuid::new_v4(), - "Sasha".to_string(), - &None, - &None, - vec![9, 10, 11, 12], - ), + SignedDoc { + id: uuid::Uuid::now_v7(), + ver: uuid::Uuid::now_v7(), + doc_type: uuid::Uuid::new_v4(), + author: "Alex".to_string(), + metadata: Some(serde_json::Value::Null), + payload: Some(serde_json::Value::Null), + raw: vec![1, 2, 3, 4], + }, + SignedDoc { + id: uuid::Uuid::now_v7(), + ver: uuid::Uuid::now_v7(), + doc_type: uuid::Uuid::new_v4(), + author: "Steven".to_string(), + metadata: Some(serde_json::Value::Null), + payload: Some(serde_json::Value::Null), + raw: vec![5, 6, 7, 8], + }, + SignedDoc { + id: uuid::Uuid::now_v7(), + ver: uuid::Uuid::now_v7(), + doc_type: uuid::Uuid::new_v4(), + author: "Sasha".to_string(), + metadata: None, + payload: None, + raw: vec![9, 10, 11, 12], + }, ]; - for (id, ver, doc_type, author, metadata, payload, raw) in &docs { - insert_signed_docs(id, ver, doc_type, author, metadata, payload, raw) - .await - .unwrap(); + for doc in &docs { + insert_signed_docs(doc).await.unwrap(); + + let res_doc = select_signed_docs(&doc.id, &None).await.unwrap(); + assert_eq!(doc, &res_doc); + let res_doc = select_signed_docs(&doc.id, &Some(doc.ver)).await.unwrap(); + assert_eq!(doc, &res_doc); + // // try to insert the same data again // insert_signed_docs(id, ver, doc_type, author, metadata, payload, raw) // .await diff --git a/catalyst-gateway/bin/src/jinja.rs b/catalyst-gateway/bin/src/jinja.rs new file mode 100644 index 00000000000..cbd8f810674 --- /dev/null +++ b/catalyst-gateway/bin/src/jinja.rs @@ -0,0 +1,30 @@ +//! `minijinja` static global variables + +use std::sync::LazyLock; + +use minijinja::Environment; + +use crate::db::event::signed_docs::SELECT_SIGNED_DOCS_TEMPLATE; + +/// Jinja template source struct. +pub(crate) struct JinjaTemplateSource { + /// Jinja template name + pub(crate) name: &'static str, + /// Jinja template source + pub(crate) source: &'static str, +} + +/// Global static `minijinja::Environment` with all preloaded templates +#[allow(clippy::unwrap_used, dead_code)] +pub(crate) static JINJA_ENV: LazyLock = LazyLock::new(|| { + let mut env = minijinja::Environment::new(); + + // Preload templates + env.add_template( + SELECT_SIGNED_DOCS_TEMPLATE.name, + SELECT_SIGNED_DOCS_TEMPLATE.source, + ) + .unwrap(); + + env +}); diff --git a/catalyst-gateway/bin/src/main.rs b/catalyst-gateway/bin/src/main.rs index b2b591e0f3a..7faf44550bc 100644 --- a/catalyst-gateway/bin/src/main.rs +++ b/catalyst-gateway/bin/src/main.rs @@ -5,6 +5,7 @@ mod build_info; mod cardano; mod cli; mod db; +mod jinja; mod logger; mod service; mod settings; From facdcb5f510151d128396d48812c3561417fa267 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Fri, 13 Dec 2024 18:18:06 +0200 Subject: [PATCH 10/42] cleanup jinja, add jinit initialization --- catalyst-gateway/bin/src/cli.rs | 2 ++ .../bin/src/db/event/signed_docs/mod.rs | 4 ++-- .../bin/src/db/event/signed_docs/tests/mod.rs | 7 ++----- catalyst-gateway/bin/src/jinja.rs | 15 +++++++++++++-- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/catalyst-gateway/bin/src/cli.rs b/catalyst-gateway/bin/src/cli.rs index e24f938befb..c8c35ba4c35 100644 --- a/catalyst-gateway/bin/src/cli.rs +++ b/catalyst-gateway/bin/src/cli.rs @@ -7,6 +7,7 @@ use tracing::{error, info}; use crate::{ cardano::start_followers, db::{self, index::session::CassandraSession}, + jinja, service::{self, started}, settings::{DocsSettings, ServiceSettings, Settings}, }; @@ -37,6 +38,7 @@ impl Cli { pub(crate) async fn exec(self) -> anyhow::Result<()> { match self { Self::Run(settings) => { + jinja::init_jinja(); Settings::init(settings)?; let mut tasks = Vec::new(); diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index 737c6b22b07..42c9e7ee6e1 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -4,7 +4,7 @@ mod tests; use super::{EventDB, NotFoundError}; -use crate::jinja::{JinjaTemplateSource, JINJA_ENV}; +use crate::jinja::{get_template, JinjaTemplateSource}; /// Insert sql query const INSERT_SIGNED_DOCS: &str = include_str!("./sql/insert_signed_documents.sql"); @@ -86,7 +86,7 @@ pub(crate) async fn insert_signed_docs(doc: &SignedDoc) -> anyhow::Result<()> { pub(crate) async fn select_signed_docs( id: &uuid::Uuid, ver: &Option, ) -> anyhow::Result { - let query_template = JINJA_ENV.get_template(SELECT_SIGNED_DOCS_TEMPLATE.name)?; + let query_template = get_template(&SELECT_SIGNED_DOCS_TEMPLATE)?; let query = query_template.render(serde_json::json!({ "id": id, "ver": ver, diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index 1bd3e679d35..d67797007ab 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -40,15 +40,12 @@ async fn some_test() { for doc in &docs { insert_signed_docs(doc).await.unwrap(); + // try to insert the same data again + insert_signed_docs(doc).await.unwrap(); let res_doc = select_signed_docs(&doc.id, &None).await.unwrap(); assert_eq!(doc, &res_doc); let res_doc = select_signed_docs(&doc.id, &Some(doc.ver)).await.unwrap(); assert_eq!(doc, &res_doc); - - // // try to insert the same data again - // insert_signed_docs(id, ver, doc_type, author, metadata, payload, raw) - // .await - // .unwrap(); } } diff --git a/catalyst-gateway/bin/src/jinja.rs b/catalyst-gateway/bin/src/jinja.rs index cbd8f810674..e182d09f423 100644 --- a/catalyst-gateway/bin/src/jinja.rs +++ b/catalyst-gateway/bin/src/jinja.rs @@ -2,7 +2,7 @@ use std::sync::LazyLock; -use minijinja::Environment; +use minijinja::{Environment, Template}; use crate::db::event::signed_docs::SELECT_SIGNED_DOCS_TEMPLATE; @@ -16,7 +16,7 @@ pub(crate) struct JinjaTemplateSource { /// Global static `minijinja::Environment` with all preloaded templates #[allow(clippy::unwrap_used, dead_code)] -pub(crate) static JINJA_ENV: LazyLock = LazyLock::new(|| { +static JINJA_ENV: LazyLock = LazyLock::new(|| { let mut env = minijinja::Environment::new(); // Preload templates @@ -28,3 +28,14 @@ pub(crate) static JINJA_ENV: LazyLock = LazyLock::new(|| { env }); + +/// Initializes `JINJA_ENV` +pub(crate) fn init_jinja() { + let _unused = &*JINJA_ENV; +} + +/// Returns a template from the jinja environment, returns error if it does not exit. +pub(crate) fn get_template(temp: &JinjaTemplateSource) -> anyhow::Result> { + let template = JINJA_ENV.get_template(temp.name)?; + Ok(template) +} From 48442468ce5fa75e653ed4c94a2abb81df595925 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Fri, 13 Dec 2024 18:34:56 +0200 Subject: [PATCH 11/42] update tests --- .../bin/src/db/event/signed_docs/tests/mod.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index d67797007ab..72e8714395d 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -47,5 +47,15 @@ async fn some_test() { assert_eq!(doc, &res_doc); let res_doc = select_signed_docs(&doc.id, &Some(doc.ver)).await.unwrap(); assert_eq!(doc, &res_doc); + + let another_doc = SignedDoc { + author: "Neil".to_string(), + ..doc.clone() + }; + assert!(insert_signed_docs(&another_doc).await.is_err()); + assert!(select_signed_docs(&another_doc.id, &None).await.is_err()); + assert!(select_signed_docs(&another_doc.id, &Some(another_doc.ver)) + .await + .is_err()); } } From 7a4c3c2f3b18ae424edd3015a035f95069aafa11 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Fri, 13 Dec 2024 18:43:56 +0200 Subject: [PATCH 12/42] fix --- .../bin/src/db/event/signed_docs/tests/mod.rs | 16 ++++++---------- catalyst-gateway/bin/src/jinja.rs | 2 +- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index 72e8714395d..398815bc9e6 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -42,20 +42,16 @@ async fn some_test() { insert_signed_docs(doc).await.unwrap(); // try to insert the same data again insert_signed_docs(doc).await.unwrap(); - - let res_doc = select_signed_docs(&doc.id, &None).await.unwrap(); - assert_eq!(doc, &res_doc); - let res_doc = select_signed_docs(&doc.id, &Some(doc.ver)).await.unwrap(); - assert_eq!(doc, &res_doc); - + // try another doc with different `author` and same other fields let another_doc = SignedDoc { author: "Neil".to_string(), ..doc.clone() }; assert!(insert_signed_docs(&another_doc).await.is_err()); - assert!(select_signed_docs(&another_doc.id, &None).await.is_err()); - assert!(select_signed_docs(&another_doc.id, &Some(another_doc.ver)) - .await - .is_err()); + + let res_doc = select_signed_docs(&doc.id, &None).await.unwrap(); + assert_eq!(doc, &res_doc); + let res_doc = select_signed_docs(&doc.id, &Some(doc.ver)).await.unwrap(); + assert_eq!(doc, &res_doc); } } diff --git a/catalyst-gateway/bin/src/jinja.rs b/catalyst-gateway/bin/src/jinja.rs index e182d09f423..84d13205fef 100644 --- a/catalyst-gateway/bin/src/jinja.rs +++ b/catalyst-gateway/bin/src/jinja.rs @@ -15,7 +15,7 @@ pub(crate) struct JinjaTemplateSource { } /// Global static `minijinja::Environment` with all preloaded templates -#[allow(clippy::unwrap_used, dead_code)] +#[allow(clippy::unwrap_used)] static JINJA_ENV: LazyLock = LazyLock::new(|| { let mut env = minijinja::Environment::new(); From 2f83960a225c997000aeb8d8b8f8ae345c471c63 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Fri, 13 Dec 2024 18:54:57 +0200 Subject: [PATCH 13/42] fix spelling --- .config/dictionaries/project.dic | 1 + 1 file changed, 1 insertion(+) diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic index 3c271a5313c..93d3f54d220 100644 --- a/.config/dictionaries/project.dic +++ b/.config/dictionaries/project.dic @@ -172,6 +172,7 @@ metamap mgrybyk miniaturizable minicbor +minijinja mithril mitigations mocktail From a7587c75626abaebf60e92859dab6ce9fb8a92f2 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Fri, 13 Dec 2024 19:20:40 +0200 Subject: [PATCH 14/42] add advanced select query --- .../bin/src/db/event/signed_docs/mod.rs | 52 ++++++++++++++++++- ...advanced_select_signed_documents.sql.jinja | 14 +++++ .../bin/src/db/event/signed_docs/tests/mod.rs | 13 +++++ catalyst-gateway/bin/src/jinja.rs | 9 +++- 4 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/event/signed_docs/sql/advanced_select_signed_documents.sql.jinja diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index 42c9e7ee6e1..46c280fffbf 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -15,6 +15,12 @@ pub(crate) const SELECT_SIGNED_DOCS_TEMPLATE: JinjaTemplateSource = JinjaTemplat source: include_str!("./sql/select_signed_documents.sql.jinja"), }; +/// Advanced select sql query jinja template +pub(crate) const ADVANCED_SELECT_SIGNED_DOCS_TEMPLATE: JinjaTemplateSource = JinjaTemplateSource { + name: "advanced_select_signed_documents.jinja.template", + source: include_str!("./sql/advanced_select_signed_documents.sql.jinja"), +}; + /// signed doc event db struct #[derive(Debug, Clone, PartialEq)] pub(crate) struct SignedDoc { @@ -34,7 +40,7 @@ pub(crate) struct SignedDoc { raw: Vec, } -/// Make an insert query into the `event-db` by adding data into the `signed_docs` table +/// Make an insert query into the `event-db` by adding data into the `signed_docs` table. /// /// * IF the record primary key (id,ver) does not exist, then add the new record. Return /// success. @@ -72,7 +78,7 @@ pub(crate) async fn insert_signed_docs(doc: &SignedDoc) -> anyhow::Result<()> { Ok(()) } -/// Make an select query into the `event-db` by getting data from the `signed_docs` table +/// Make a select query into the `event-db` by getting data from the `signed_docs` table. /// /// * This returns a single document. All data from the document is returned, including /// the `payload` and `raw` fields. @@ -115,3 +121,45 @@ pub(crate) async fn select_signed_docs( raw, }) } + +/// Make an advanced select query into the `event-db` by getting data from the +/// `signed_docs` table. +/// +/// * This returns a single document. All data from the document is returned, including +/// the `payload` and `raw` fields. +/// * `ver` should be able to be optional, in which case get the latest ver of the given +/// `id`. +/// +/// # Arguments: +/// - `id` is a UUID v7 +/// - `ver` is a UUID v7 +#[allow(dead_code)] +pub(crate) async fn advanced_select_signed_docs( + conditions: &str, limit: Option, offset: Option, +) -> anyhow::Result { + let query_template = get_template(&ADVANCED_SELECT_SIGNED_DOCS_TEMPLATE)?; + let query = query_template.render(serde_json::json!({ + "conditions": conditions, + "limit": limit, + "offset": offset, + }))?; + let res = EventDB::query_one(&query, &[]).await?; + + let id = res.try_get("id")?; + let ver = res.try_get("ver")?; + let doc_type = res.try_get("type")?; + let author = res.try_get("author")?; + let metadata = res.try_get("metadata")?; + let payload = res.try_get("payload")?; + let raw = res.try_get("raw")?; + + Ok(SignedDoc { + id, + ver, + doc_type, + author, + metadata, + payload, + raw, + }) +} diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/sql/advanced_select_signed_documents.sql.jinja b/catalyst-gateway/bin/src/db/event/signed_docs/sql/advanced_select_signed_documents.sql.jinja new file mode 100644 index 00000000000..fc386d0b7fb --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/signed_docs/sql/advanced_select_signed_documents.sql.jinja @@ -0,0 +1,14 @@ +SELECT + signed_docs.id, + signed_docs.ver, + signed_docs.type, + signed_docs.author, + signed_docs.metadata, + signed_docs.payload, + signed_docs.raw +FROM signed_docs +WHERE + {{ conditions }} +ORDER BY signed_docs.type DESC, signed_docs.id DESC, signed_docs.ver DESC +{% if limit %} LIMIT {{ limit }} {% endif %} +{% if offset %} OFFSET {{ offset }} {% endif %} diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index 398815bc9e6..df5a0ff8a22 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -53,5 +53,18 @@ async fn some_test() { assert_eq!(doc, &res_doc); let res_doc = select_signed_docs(&doc.id, &Some(doc.ver)).await.unwrap(); assert_eq!(doc, &res_doc); + + let res_doc = advanced_select_signed_docs( + format!( + "signed_docs.id = '{}' AND signed_docs.ver = '{}'", + doc.id, doc.ver + ) + .as_str(), + None, + None, + ) + .await + .unwrap(); + assert_eq!(doc, &res_doc); } } diff --git a/catalyst-gateway/bin/src/jinja.rs b/catalyst-gateway/bin/src/jinja.rs index 84d13205fef..0a81937b807 100644 --- a/catalyst-gateway/bin/src/jinja.rs +++ b/catalyst-gateway/bin/src/jinja.rs @@ -4,7 +4,9 @@ use std::sync::LazyLock; use minijinja::{Environment, Template}; -use crate::db::event::signed_docs::SELECT_SIGNED_DOCS_TEMPLATE; +use crate::db::event::signed_docs::{ + ADVANCED_SELECT_SIGNED_DOCS_TEMPLATE, SELECT_SIGNED_DOCS_TEMPLATE, +}; /// Jinja template source struct. pub(crate) struct JinjaTemplateSource { @@ -25,6 +27,11 @@ static JINJA_ENV: LazyLock = LazyLock::new(|| { SELECT_SIGNED_DOCS_TEMPLATE.source, ) .unwrap(); + env.add_template( + ADVANCED_SELECT_SIGNED_DOCS_TEMPLATE.name, + ADVANCED_SELECT_SIGNED_DOCS_TEMPLATE.source, + ) + .unwrap(); env }); From 6bdb60bf325302cfdb976acd107fde1b76170094 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Sat, 14 Dec 2024 14:11:50 +0200 Subject: [PATCH 15/42] fix docs --- catalyst-gateway/bin/src/db/event/signed_docs/mod.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index 46c280fffbf..7ffd1927cf4 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -125,14 +125,8 @@ pub(crate) async fn select_signed_docs( /// Make an advanced select query into the `event-db` by getting data from the /// `signed_docs` table. /// -/// * This returns a single document. All data from the document is returned, including -/// the `payload` and `raw` fields. -/// * `ver` should be able to be optional, in which case get the latest ver of the given -/// `id`. -/// /// # Arguments: -/// - `id` is a UUID v7 -/// - `ver` is a UUID v7 +/// - `conditions` an SQL `WHERE` statements #[allow(dead_code)] pub(crate) async fn advanced_select_signed_docs( conditions: &str, limit: Option, offset: Option, From d0fd29507d7035ceb2e3ef67725ac1c0ea6c36ea Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Mon, 16 Dec 2024 16:13:10 +0200 Subject: [PATCH 16/42] add QueryLimits --- catalyst-gateway/bin/src/db/event/common.rs | 24 +++++++++++++++++++ catalyst-gateway/bin/src/db/event/mod.rs | 1 + .../bin/src/db/event/signed_docs/mod.rs | 14 +++++++---- ...advanced_select_signed_documents.sql.jinja | 3 +-- .../bin/src/db/event/signed_docs/tests/mod.rs | 3 +-- 5 files changed, 37 insertions(+), 8 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/event/common.rs diff --git a/catalyst-gateway/bin/src/db/event/common.rs b/catalyst-gateway/bin/src/db/event/common.rs new file mode 100644 index 00000000000..0252fc3d870 --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/common.rs @@ -0,0 +1,24 @@ +//! Reusable common database objects + +#![allow(dead_code)] + +/// A query `LIMIT` and `OFFSET` limits. +pub(crate) enum QueryLimits { + /// Specifies `LIMIT` parameter equals to `1` + One, + /// Specifies `LIMIT` parameter + Limit(u64), + /// Specifies `LIMIT` and `OFFSET` parameters + LimitAndOffset(u64, u64), +} + +impl QueryLimits { + /// Returns a string with the corresponding query limit statement + pub(crate) fn query_limit_stmt(&self) -> String { + match self { + Self::One => "LIMIT 1".to_string(), + Self::Limit(limit) => format!("LIMIT {limit}"), + Self::LimitAndOffset(limit, offset) => format!("LIMIT {limit} OFFSET {offset}"), + } + } +} diff --git a/catalyst-gateway/bin/src/db/event/mod.rs b/catalyst-gateway/bin/src/db/event/mod.rs index d86c8ee584f..0d737876505 100644 --- a/catalyst-gateway/bin/src/db/event/mod.rs +++ b/catalyst-gateway/bin/src/db/event/mod.rs @@ -15,6 +15,7 @@ use tracing::{debug, debug_span, error, Instrument}; use crate::settings::Settings; +pub(crate) mod common; pub(crate) mod config; pub(crate) mod error; pub(crate) mod legacy; diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index 7ffd1927cf4..81c6d02425a 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -3,7 +3,7 @@ #[cfg(test)] mod tests; -use super::{EventDB, NotFoundError}; +use super::{common::QueryLimits, EventDB, NotFoundError}; use crate::jinja::{get_template, JinjaTemplateSource}; /// Insert sql query @@ -122,6 +122,13 @@ pub(crate) async fn select_signed_docs( }) } +/// A `select_signed_docs` query filtering argument. +#[allow(dead_code)] +pub(crate) enum DocQueryFilter { + /// All entries + All, +} + /// Make an advanced select query into the `event-db` by getting data from the /// `signed_docs` table. /// @@ -129,13 +136,12 @@ pub(crate) async fn select_signed_docs( /// - `conditions` an SQL `WHERE` statements #[allow(dead_code)] pub(crate) async fn advanced_select_signed_docs( - conditions: &str, limit: Option, offset: Option, + conditions: &str, query_limits: &QueryLimits, ) -> anyhow::Result { let query_template = get_template(&ADVANCED_SELECT_SIGNED_DOCS_TEMPLATE)?; let query = query_template.render(serde_json::json!({ "conditions": conditions, - "limit": limit, - "offset": offset, + "query_limits": query_limits.query_limit_stmt(), }))?; let res = EventDB::query_one(&query, &[]).await?; diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/sql/advanced_select_signed_documents.sql.jinja b/catalyst-gateway/bin/src/db/event/signed_docs/sql/advanced_select_signed_documents.sql.jinja index fc386d0b7fb..1fa3646300b 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/sql/advanced_select_signed_documents.sql.jinja +++ b/catalyst-gateway/bin/src/db/event/signed_docs/sql/advanced_select_signed_documents.sql.jinja @@ -10,5 +10,4 @@ FROM signed_docs WHERE {{ conditions }} ORDER BY signed_docs.type DESC, signed_docs.id DESC, signed_docs.ver DESC -{% if limit %} LIMIT {{ limit }} {% endif %} -{% if offset %} OFFSET {{ offset }} {% endif %} + {{ query_limits }} diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index df5a0ff8a22..df7ac9e7de9 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -60,8 +60,7 @@ async fn some_test() { doc.id, doc.ver ) .as_str(), - None, - None, + &QueryLimits::One, ) .await .unwrap(); From c52828e74a4b00e26fc386f679b639d16aea9049 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Mon, 16 Dec 2024 18:46:56 +0200 Subject: [PATCH 17/42] wip --- catalyst-gateway/bin/src/db/event/common.rs | 5 +- .../bin/src/db/event/signed_docs/mod.rs | 103 ++++++++++-------- ...advanced_select_signed_documents.sql.jinja | 4 +- .../sql/select_signed_documents.sql.jinja | 3 +- .../bin/src/db/event/signed_docs/tests/mod.rs | 54 ++++++--- 5 files changed, 101 insertions(+), 68 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/common.rs b/catalyst-gateway/bin/src/db/event/common.rs index 0252fc3d870..0b9410b440f 100644 --- a/catalyst-gateway/bin/src/db/event/common.rs +++ b/catalyst-gateway/bin/src/db/event/common.rs @@ -4,6 +4,8 @@ /// A query `LIMIT` and `OFFSET` limits. pub(crate) enum QueryLimits { + /// Return all entries without any `LIMIT` and `OFFSET` parametrs + All, /// Specifies `LIMIT` parameter equals to `1` One, /// Specifies `LIMIT` parameter @@ -14,8 +16,9 @@ pub(crate) enum QueryLimits { impl QueryLimits { /// Returns a string with the corresponding query limit statement - pub(crate) fn query_limit_stmt(&self) -> String { + pub(crate) fn query_stmt(&self) -> String { match self { + Self::All => String::new(), Self::One => "LIMIT 1".to_string(), Self::Limit(limit) => format!("LIMIT {limit}"), Self::LimitAndOffset(limit, offset) => format!("LIMIT {limit} OFFSET {offset}"), diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index 81c6d02425a..c3c9be112af 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -53,7 +53,7 @@ pub(crate) struct SignedDoc { /// - `doc_type` is a UUID v4 #[allow(dead_code)] pub(crate) async fn insert_signed_docs(doc: &SignedDoc) -> anyhow::Result<()> { - match select_signed_docs(&doc.id, &Some(doc.ver)).await { + match select_signed_docs(&doc.id, &doc.ver).await { Ok(res_doc) => { anyhow::ensure!( &res_doc == doc, @@ -79,19 +79,7 @@ pub(crate) async fn insert_signed_docs(doc: &SignedDoc) -> anyhow::Result<()> { } /// Make a select query into the `event-db` by getting data from the `signed_docs` table. -/// -/// * This returns a single document. All data from the document is returned, including -/// the `payload` and `raw` fields. -/// * `ver` should be able to be optional, in which case get the latest ver of the given -/// `id`. -/// -/// # Arguments: -/// - `id` is a UUID v7 -/// - `ver` is a UUID v7 -#[allow(dead_code)] -pub(crate) async fn select_signed_docs( - id: &uuid::Uuid, ver: &Option, -) -> anyhow::Result { +async fn select_signed_docs(id: &uuid::Uuid, ver: &uuid::Uuid) -> anyhow::Result { let query_template = get_template(&SELECT_SIGNED_DOCS_TEMPLATE)?; let query = query_template.render(serde_json::json!({ "id": id, @@ -99,12 +87,6 @@ pub(crate) async fn select_signed_docs( }))?; let res = EventDB::query_one(&query, &[]).await?; - let ver = if let Some(ver) = ver { - *ver - } else { - res.try_get("ver")? - }; - let doc_type = res.try_get("type")?; let author = res.try_get("author")?; let metadata = res.try_get("metadata")?; @@ -113,7 +95,7 @@ pub(crate) async fn select_signed_docs( Ok(SignedDoc { id: *id, - ver, + ver: *ver, doc_type, author, metadata, @@ -124,9 +106,32 @@ pub(crate) async fn select_signed_docs( /// A `select_signed_docs` query filtering argument. #[allow(dead_code)] -pub(crate) enum DocQueryFilter { +pub(crate) enum DocsQueryFilter { /// All entries All, + /// Select docs with the specific `type` field + DocType(uuid::Uuid), + /// Select docs with the specific `id` field + DocId(uuid::Uuid), + /// Select docs with the specific `id` and `ver` field + DocVer(uuid::Uuid, uuid::Uuid), + /// Select docs with the specific `author` field + Author(String), +} + +impl DocsQueryFilter { + /// Returns a string with the corresponding query docs filter statement + pub(crate) fn query_stmt(&self) -> String { + match self { + Self::All => "TRUE".to_string(), + Self::DocType(doc_type) => format!("signed_docs.type = '{doc_type}'"), + Self::DocId(id) => format!("signed_docs.id = '{id}'"), + Self::DocVer(id, ver) => { + format!("signed_docs.id = '{id}' AND signed_docs.ver = '{ver}'") + }, + Self::Author(author) => format!("signed_docs.author = '{author}'"), + } + } } /// Make an advanced select query into the `event-db` by getting data from the @@ -135,31 +140,37 @@ pub(crate) enum DocQueryFilter { /// # Arguments: /// - `conditions` an SQL `WHERE` statements #[allow(dead_code)] -pub(crate) async fn advanced_select_signed_docs( - conditions: &str, query_limits: &QueryLimits, -) -> anyhow::Result { +pub(crate) async fn filtered_select_signed_docs( + conditions: &DocsQueryFilter, query_limits: &QueryLimits, +) -> anyhow::Result> { let query_template = get_template(&ADVANCED_SELECT_SIGNED_DOCS_TEMPLATE)?; let query = query_template.render(serde_json::json!({ - "conditions": conditions, - "query_limits": query_limits.query_limit_stmt(), + "conditions": conditions.query_stmt(), + "query_limits": query_limits.query_stmt(), }))?; - let res = EventDB::query_one(&query, &[]).await?; - - let id = res.try_get("id")?; - let ver = res.try_get("ver")?; - let doc_type = res.try_get("type")?; - let author = res.try_get("author")?; - let metadata = res.try_get("metadata")?; - let payload = res.try_get("payload")?; - let raw = res.try_get("raw")?; - - Ok(SignedDoc { - id, - ver, - doc_type, - author, - metadata, - payload, - raw, - }) + let rows = EventDB::query(&query, &[]).await?; + + let docs = rows + .into_iter() + .map(|row| { + let id = row.try_get("id")?; + let ver = row.try_get("ver")?; + let doc_type = row.try_get("type")?; + let author = row.try_get("author")?; + let metadata = row.try_get("metadata")?; + let payload = row.try_get("payload")?; + let raw = row.try_get("raw")?; + Ok(SignedDoc { + id, + ver, + doc_type, + author, + metadata, + payload, + raw, + }) + }) + .collect::>()?; + + Ok(docs) } diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/sql/advanced_select_signed_documents.sql.jinja b/catalyst-gateway/bin/src/db/event/signed_docs/sql/advanced_select_signed_documents.sql.jinja index 1fa3646300b..3210e1c6dff 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/sql/advanced_select_signed_documents.sql.jinja +++ b/catalyst-gateway/bin/src/db/event/signed_docs/sql/advanced_select_signed_documents.sql.jinja @@ -3,9 +3,7 @@ SELECT signed_docs.ver, signed_docs.type, signed_docs.author, - signed_docs.metadata, - signed_docs.payload, - signed_docs.raw + signed_docs.metadata FROM signed_docs WHERE {{ conditions }} diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/sql/select_signed_documents.sql.jinja b/catalyst-gateway/bin/src/db/event/signed_docs/sql/select_signed_documents.sql.jinja index 1ed2397a831..cf4ca4920b5 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/sql/select_signed_documents.sql.jinja +++ b/catalyst-gateway/bin/src/db/event/signed_docs/sql/select_signed_documents.sql.jinja @@ -1,5 +1,4 @@ SELECT - {% if not ver %} signed_docs.ver, {% endif %} signed_docs.type, signed_docs.author, signed_docs.metadata, @@ -8,6 +7,6 @@ SELECT FROM signed_docs WHERE signed_docs.id = '{{ id }}' - {% if ver %} AND signed_docs.ver = '{{ ver }}' {% endif %} + AND signed_docs.ver = '{{ ver }}' ORDER BY signed_docs.ver DESC LIMIT 1 diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index df7ac9e7de9..fa1a0930981 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -8,11 +8,13 @@ use crate::db::event::establish_connection; async fn some_test() { establish_connection(); - let docs = [ + let doc_type = uuid::Uuid::new_v4(); + + let docs = vec![ SignedDoc { id: uuid::Uuid::now_v7(), ver: uuid::Uuid::now_v7(), - doc_type: uuid::Uuid::new_v4(), + doc_type, author: "Alex".to_string(), metadata: Some(serde_json::Value::Null), payload: Some(serde_json::Value::Null), @@ -21,7 +23,7 @@ async fn some_test() { SignedDoc { id: uuid::Uuid::now_v7(), ver: uuid::Uuid::now_v7(), - doc_type: uuid::Uuid::new_v4(), + doc_type, author: "Steven".to_string(), metadata: Some(serde_json::Value::Null), payload: Some(serde_json::Value::Null), @@ -30,7 +32,7 @@ async fn some_test() { SignedDoc { id: uuid::Uuid::now_v7(), ver: uuid::Uuid::now_v7(), - doc_type: uuid::Uuid::new_v4(), + doc_type, author: "Sasha".to_string(), metadata: None, payload: None, @@ -49,21 +51,41 @@ async fn some_test() { }; assert!(insert_signed_docs(&another_doc).await.is_err()); - let res_doc = select_signed_docs(&doc.id, &None).await.unwrap(); - assert_eq!(doc, &res_doc); - let res_doc = select_signed_docs(&doc.id, &Some(doc.ver)).await.unwrap(); - assert_eq!(doc, &res_doc); + let res_docs = + filtered_select_signed_docs(&DocsQueryFilter::DocId(doc.id), &QueryLimits::All) + .await + .unwrap(); + assert_eq!(res_docs.len(), 1); + assert_eq!(doc, res_docs.first().unwrap()); - let res_doc = advanced_select_signed_docs( - format!( - "signed_docs.id = '{}' AND signed_docs.ver = '{}'", - doc.id, doc.ver - ) - .as_str(), - &QueryLimits::One, + let res_docs = filtered_select_signed_docs( + &DocsQueryFilter::DocVer(doc.id, doc.ver), + &QueryLimits::All, ) .await .unwrap(); - assert_eq!(doc, &res_doc); + assert_eq!(res_docs.len(), 1); + assert_eq!(doc, res_docs.first().unwrap()); + + let res_docs = + filtered_select_signed_docs(&DocsQueryFilter::DocType(doc.doc_type), &QueryLimits::All) + .await + .unwrap(); + assert_eq!(res_docs.len(), 1); + assert_eq!(doc, res_docs.first().unwrap()); + + let res_docs = filtered_select_signed_docs( + &DocsQueryFilter::Author(doc.author.clone()), + &QueryLimits::All, + ) + .await + .unwrap(); + assert_eq!(res_docs.len(), 1); + assert_eq!(doc, res_docs.first().unwrap()); } + + let res_docs = filtered_select_signed_docs(&DocsQueryFilter::All, &QueryLimits::All) + .await + .unwrap(); + assert_eq!(docs, res_docs.into_iter().rev().collect::>()); } From d866f715f9a652789ad29d83d27b5460a0d48c71 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Mon, 16 Dec 2024 18:57:51 +0200 Subject: [PATCH 18/42] wip --- .../bin/src/db/event/signed_docs/mod.rs | 49 ++++++------ .../bin/src/db/event/signed_docs/tests/mod.rs | 78 +++++++++++-------- 2 files changed, 73 insertions(+), 54 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index c3c9be112af..f24112f9cac 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -21,9 +21,9 @@ pub(crate) const ADVANCED_SELECT_SIGNED_DOCS_TEMPLATE: JinjaTemplateSource = Jin source: include_str!("./sql/advanced_select_signed_documents.sql.jinja"), }; -/// signed doc event db struct +/// Signed doc body event db struct #[derive(Debug, Clone, PartialEq)] -pub(crate) struct SignedDoc { +pub(crate) struct SignedDocBody { /// `signed_doc` table `id` field id: uuid::Uuid, /// `signed_doc` table `ver` field @@ -34,6 +34,13 @@ pub(crate) struct SignedDoc { author: String, /// `signed_doc` table `metadata` field metadata: Option, +} + +/// Full signed doc event db struct +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct FullSignedDoc { + /// Signed doc body + body: SignedDocBody, /// `signed_doc` table `payload` field payload: Option, /// `signed_doc` table `raw` field @@ -52,8 +59,8 @@ pub(crate) struct SignedDoc { /// - `ver` is a UUID v7 /// - `doc_type` is a UUID v4 #[allow(dead_code)] -pub(crate) async fn insert_signed_docs(doc: &SignedDoc) -> anyhow::Result<()> { - match select_signed_docs(&doc.id, &doc.ver).await { +pub(crate) async fn insert_signed_docs(doc: &FullSignedDoc) -> anyhow::Result<()> { + match select_signed_docs(&doc.body.id, &doc.body.ver).await { Ok(res_doc) => { anyhow::ensure!( &res_doc == doc, @@ -66,11 +73,11 @@ pub(crate) async fn insert_signed_docs(doc: &SignedDoc) -> anyhow::Result<()> { } EventDB::modify(INSERT_SIGNED_DOCS, &[ - &doc.id, - &doc.ver, - &doc.doc_type, - &doc.author, - &doc.metadata, + &doc.body.id, + &doc.body.ver, + &doc.body.doc_type, + &doc.body.author, + &doc.body.metadata, &doc.payload, &doc.raw, ]) @@ -79,7 +86,7 @@ pub(crate) async fn insert_signed_docs(doc: &SignedDoc) -> anyhow::Result<()> { } /// Make a select query into the `event-db` by getting data from the `signed_docs` table. -async fn select_signed_docs(id: &uuid::Uuid, ver: &uuid::Uuid) -> anyhow::Result { +async fn select_signed_docs(id: &uuid::Uuid, ver: &uuid::Uuid) -> anyhow::Result { let query_template = get_template(&SELECT_SIGNED_DOCS_TEMPLATE)?; let query = query_template.render(serde_json::json!({ "id": id, @@ -93,12 +100,14 @@ async fn select_signed_docs(id: &uuid::Uuid, ver: &uuid::Uuid) -> anyhow::Result let payload = res.try_get("payload")?; let raw = res.try_get("raw")?; - Ok(SignedDoc { - id: *id, - ver: *ver, - doc_type, - author, - metadata, + Ok(FullSignedDoc { + body: SignedDocBody { + id: *id, + ver: *ver, + doc_type, + author, + metadata, + }, payload, raw, }) @@ -142,7 +151,7 @@ impl DocsQueryFilter { #[allow(dead_code)] pub(crate) async fn filtered_select_signed_docs( conditions: &DocsQueryFilter, query_limits: &QueryLimits, -) -> anyhow::Result> { +) -> anyhow::Result> { let query_template = get_template(&ADVANCED_SELECT_SIGNED_DOCS_TEMPLATE)?; let query = query_template.render(serde_json::json!({ "conditions": conditions.query_stmt(), @@ -158,16 +167,12 @@ pub(crate) async fn filtered_select_signed_docs( let doc_type = row.try_get("type")?; let author = row.try_get("author")?; let metadata = row.try_get("metadata")?; - let payload = row.try_get("payload")?; - let raw = row.try_get("raw")?; - Ok(SignedDoc { + Ok(SignedDocBody { id, ver, doc_type, author, metadata, - payload, - raw, }) }) .collect::>()?; diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index fa1a0930981..571d79796aa 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -11,30 +11,36 @@ async fn some_test() { let doc_type = uuid::Uuid::new_v4(); let docs = vec![ - SignedDoc { - id: uuid::Uuid::now_v7(), - ver: uuid::Uuid::now_v7(), - doc_type, - author: "Alex".to_string(), - metadata: Some(serde_json::Value::Null), + FullSignedDoc { + body: SignedDocBody { + id: uuid::Uuid::now_v7(), + ver: uuid::Uuid::now_v7(), + doc_type, + author: "Alex".to_string(), + metadata: Some(serde_json::Value::Null), + }, payload: Some(serde_json::Value::Null), raw: vec![1, 2, 3, 4], }, - SignedDoc { - id: uuid::Uuid::now_v7(), - ver: uuid::Uuid::now_v7(), - doc_type, - author: "Steven".to_string(), - metadata: Some(serde_json::Value::Null), + FullSignedDoc { + body: SignedDocBody { + id: uuid::Uuid::now_v7(), + ver: uuid::Uuid::now_v7(), + doc_type, + author: "Steven".to_string(), + metadata: Some(serde_json::Value::Null), + }, payload: Some(serde_json::Value::Null), raw: vec![5, 6, 7, 8], }, - SignedDoc { - id: uuid::Uuid::now_v7(), - ver: uuid::Uuid::now_v7(), - doc_type, - author: "Sasha".to_string(), - metadata: None, + FullSignedDoc { + body: SignedDocBody { + id: uuid::Uuid::now_v7(), + ver: uuid::Uuid::now_v7(), + doc_type, + author: "Sasha".to_string(), + metadata: None, + }, payload: None, raw: vec![9, 10, 11, 12], }, @@ -45,47 +51,55 @@ async fn some_test() { // try to insert the same data again insert_signed_docs(doc).await.unwrap(); // try another doc with different `author` and same other fields - let another_doc = SignedDoc { - author: "Neil".to_string(), + let another_doc = FullSignedDoc { + body: SignedDocBody { + author: "Neil".to_string(), + ..doc.body.clone() + }, ..doc.clone() }; assert!(insert_signed_docs(&another_doc).await.is_err()); let res_docs = - filtered_select_signed_docs(&DocsQueryFilter::DocId(doc.id), &QueryLimits::All) + filtered_select_signed_docs(&DocsQueryFilter::DocId(doc.body.id), &QueryLimits::All) .await .unwrap(); assert_eq!(res_docs.len(), 1); - assert_eq!(doc, res_docs.first().unwrap()); + assert_eq!(&doc.body, res_docs.first().unwrap()); let res_docs = filtered_select_signed_docs( - &DocsQueryFilter::DocVer(doc.id, doc.ver), + &DocsQueryFilter::DocVer(doc.body.id, doc.body.ver), &QueryLimits::All, ) .await .unwrap(); assert_eq!(res_docs.len(), 1); - assert_eq!(doc, res_docs.first().unwrap()); + assert_eq!(&doc.body, res_docs.first().unwrap()); - let res_docs = - filtered_select_signed_docs(&DocsQueryFilter::DocType(doc.doc_type), &QueryLimits::All) - .await - .unwrap(); + let res_docs = filtered_select_signed_docs( + &DocsQueryFilter::DocType(doc.body.doc_type), + &QueryLimits::All, + ) + .await + .unwrap(); assert_eq!(res_docs.len(), 1); - assert_eq!(doc, res_docs.first().unwrap()); + assert_eq!(&doc.body, res_docs.first().unwrap()); let res_docs = filtered_select_signed_docs( - &DocsQueryFilter::Author(doc.author.clone()), + &DocsQueryFilter::Author(doc.body.author.clone()), &QueryLimits::All, ) .await .unwrap(); assert_eq!(res_docs.len(), 1); - assert_eq!(doc, res_docs.first().unwrap()); + assert_eq!(&doc.body, res_docs.first().unwrap()); } let res_docs = filtered_select_signed_docs(&DocsQueryFilter::All, &QueryLimits::All) .await .unwrap(); - assert_eq!(docs, res_docs.into_iter().rev().collect::>()); + assert_eq!( + docs.into_iter().map(|doc| doc.body).collect::>(), + res_docs.into_iter().rev().collect::>() + ); } From 9fc43b4ed4ef2e3ae1e3d0346b4dda0b43af00cf Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Mon, 16 Dec 2024 19:10:41 +0200 Subject: [PATCH 19/42] update tests --- catalyst-gateway/bin/src/db/event/common.rs | 2 +- .../bin/src/db/event/signed_docs/tests/mod.rs | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/common.rs b/catalyst-gateway/bin/src/db/event/common.rs index 0b9410b440f..5e6428b0ba8 100644 --- a/catalyst-gateway/bin/src/db/event/common.rs +++ b/catalyst-gateway/bin/src/db/event/common.rs @@ -4,7 +4,7 @@ /// A query `LIMIT` and `OFFSET` limits. pub(crate) enum QueryLimits { - /// Return all entries without any `LIMIT` and `OFFSET` parametrs + /// Return all entries without any `LIMIT` and `OFFSET` parameters All, /// Specifies `LIMIT` parameter equals to `1` One, diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index 571d79796aa..4fd66e8c3a1 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -76,15 +76,6 @@ async fn some_test() { assert_eq!(res_docs.len(), 1); assert_eq!(&doc.body, res_docs.first().unwrap()); - let res_docs = filtered_select_signed_docs( - &DocsQueryFilter::DocType(doc.body.doc_type), - &QueryLimits::All, - ) - .await - .unwrap(); - assert_eq!(res_docs.len(), 1); - assert_eq!(&doc.body, res_docs.first().unwrap()); - let res_docs = filtered_select_signed_docs( &DocsQueryFilter::Author(doc.body.author.clone()), &QueryLimits::All, @@ -95,11 +86,20 @@ async fn some_test() { assert_eq!(&doc.body, res_docs.first().unwrap()); } + let res_docs = + filtered_select_signed_docs(&DocsQueryFilter::DocType(doc_type), &QueryLimits::All) + .await + .unwrap(); + assert_eq!( + docs.iter().map(|doc| &doc.body).collect::>(), + res_docs.iter().rev().collect::>() + ); + let res_docs = filtered_select_signed_docs(&DocsQueryFilter::All, &QueryLimits::All) .await .unwrap(); assert_eq!( - docs.into_iter().map(|doc| doc.body).collect::>(), - res_docs.into_iter().rev().collect::>() + docs.iter().map(|doc| &doc.body).collect::>(), + res_docs.iter().rev().collect::>() ); } From b088b06310308aec73bcd91e9c784c3d4b1a9217 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Mon, 16 Dec 2024 20:51:01 +0200 Subject: [PATCH 20/42] fix docs --- catalyst-gateway/bin/src/db/event/signed_docs/mod.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index f24112f9cac..ac560735c16 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -143,11 +143,8 @@ impl DocsQueryFilter { } } -/// Make an advanced select query into the `event-db` by getting data from the +/// Make an filtered select query into the `event-db` by getting data from the /// `signed_docs` table. -/// -/// # Arguments: -/// - `conditions` an SQL `WHERE` statements #[allow(dead_code)] pub(crate) async fn filtered_select_signed_docs( conditions: &DocsQueryFilter, query_limits: &QueryLimits, From 3f3eccf1fbb4e8cd696ec1839b255a5b8fb5f07e Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Wed, 18 Dec 2024 12:23:33 +0200 Subject: [PATCH 21/42] fix cardano-chain-follower dep, remove init of jinja --- catalyst-gateway/bin/Cargo.toml | 2 +- catalyst-gateway/bin/src/cli.rs | 2 -- catalyst-gateway/bin/src/jinja.rs | 5 ----- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/catalyst-gateway/bin/Cargo.toml b/catalyst-gateway/bin/Cargo.toml index 2d52f701cf4..c19c4976f28 100644 --- a/catalyst-gateway/bin/Cargo.toml +++ b/catalyst-gateway/bin/Cargo.toml @@ -15,7 +15,7 @@ repository.workspace = true workspace = true [dependencies] -cardano-chain-follower = { version = "0.0.5", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "v0.0.9" } +cardano-chain-follower = { version = "0.0.6", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "v0.0.10" } c509-certificate = { version = "0.0.3", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "v0.0.3" } rbac-registration = { version = "0.0.2", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "v0.0.8" } diff --git a/catalyst-gateway/bin/src/cli.rs b/catalyst-gateway/bin/src/cli.rs index c8c35ba4c35..e24f938befb 100644 --- a/catalyst-gateway/bin/src/cli.rs +++ b/catalyst-gateway/bin/src/cli.rs @@ -7,7 +7,6 @@ use tracing::{error, info}; use crate::{ cardano::start_followers, db::{self, index::session::CassandraSession}, - jinja, service::{self, started}, settings::{DocsSettings, ServiceSettings, Settings}, }; @@ -38,7 +37,6 @@ impl Cli { pub(crate) async fn exec(self) -> anyhow::Result<()> { match self { Self::Run(settings) => { - jinja::init_jinja(); Settings::init(settings)?; let mut tasks = Vec::new(); diff --git a/catalyst-gateway/bin/src/jinja.rs b/catalyst-gateway/bin/src/jinja.rs index 0a81937b807..f8c1534e7bf 100644 --- a/catalyst-gateway/bin/src/jinja.rs +++ b/catalyst-gateway/bin/src/jinja.rs @@ -36,11 +36,6 @@ static JINJA_ENV: LazyLock = LazyLock::new(|| { env }); -/// Initializes `JINJA_ENV` -pub(crate) fn init_jinja() { - let _unused = &*JINJA_ENV; -} - /// Returns a template from the jinja environment, returns error if it does not exit. pub(crate) fn get_template(temp: &JinjaTemplateSource) -> anyhow::Result> { let template = JINJA_ENV.get_template(temp.name)?; From e1181dd9b3f0dc7b4834dce7e361626510548bd7 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Wed, 18 Dec 2024 15:02:04 +0200 Subject: [PATCH 22/42] refactor QueryLimits struct --- catalyst-gateway/bin/src/db/event/common.rs | 27 ------- .../bin/src/db/event/common/mod.rs | 3 + .../bin/src/db/event/common/query_limits.rs | 73 +++++++++++++++++++ .../bin/src/db/event/signed_docs/mod.rs | 2 +- .../bin/src/db/event/signed_docs/tests/mod.rs | 18 +++-- .../common/types/generic/query/pagination.rs | 18 +++++ catalyst-gateway/bin/src/service/mod.rs | 2 +- 7 files changed, 106 insertions(+), 37 deletions(-) delete mode 100644 catalyst-gateway/bin/src/db/event/common.rs create mode 100644 catalyst-gateway/bin/src/db/event/common/mod.rs create mode 100644 catalyst-gateway/bin/src/db/event/common/query_limits.rs diff --git a/catalyst-gateway/bin/src/db/event/common.rs b/catalyst-gateway/bin/src/db/event/common.rs deleted file mode 100644 index 5e6428b0ba8..00000000000 --- a/catalyst-gateway/bin/src/db/event/common.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! Reusable common database objects - -#![allow(dead_code)] - -/// A query `LIMIT` and `OFFSET` limits. -pub(crate) enum QueryLimits { - /// Return all entries without any `LIMIT` and `OFFSET` parameters - All, - /// Specifies `LIMIT` parameter equals to `1` - One, - /// Specifies `LIMIT` parameter - Limit(u64), - /// Specifies `LIMIT` and `OFFSET` parameters - LimitAndOffset(u64, u64), -} - -impl QueryLimits { - /// Returns a string with the corresponding query limit statement - pub(crate) fn query_stmt(&self) -> String { - match self { - Self::All => String::new(), - Self::One => "LIMIT 1".to_string(), - Self::Limit(limit) => format!("LIMIT {limit}"), - Self::LimitAndOffset(limit, offset) => format!("LIMIT {limit} OFFSET {offset}"), - } - } -} diff --git a/catalyst-gateway/bin/src/db/event/common/mod.rs b/catalyst-gateway/bin/src/db/event/common/mod.rs new file mode 100644 index 00000000000..612900debc7 --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/common/mod.rs @@ -0,0 +1,3 @@ +//! Reusable common database objects + +pub(crate) mod query_limits; diff --git a/catalyst-gateway/bin/src/db/event/common/query_limits.rs b/catalyst-gateway/bin/src/db/event/common/query_limits.rs new file mode 100644 index 00000000000..69d63355081 --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/common/query_limits.rs @@ -0,0 +1,73 @@ +//! `QueryLimits` query argument object. + +#![allow(dead_code)] + +use anyhow::ensure; + +use crate::service::common::types::generic::query::pagination::{Limit, Page}; + +/// A query limits struct. +pub(crate) struct QueryLimits(QueryLimitsInner); + +/// `QueryLimits` inner enum representation. +enum QueryLimitsInner { + /// Return all entries without any `LIMIT` and `OFFSET` parameters + All, + /// Specifies `LIMIT` parameter equals to `1` + One, + /// Specifies `LIMIT` parameter + Limit(u64), + /// Specifies `LIMIT` and `OFFSET` parameters + LimitAndOffset(u64, u64), +} + +impl QueryLimits { + /// Create a `QueryLimits` object from the service `Limit` and `Page` values. + /// + /// # Errors + /// - Invalid `limit` value, must be more than `0`. + /// - Invalid arguments, `limit` must be provided when `page` is not None. + pub(crate) fn new(limit: Option, page: Option) -> anyhow::Result { + if let Some(limit) = limit { + let limit = limit.into(); + ensure!( + limit != 0_u64, + "Invalid `limit` value, must be more than `0`" + ); + + if let Some(page) = page { + Ok(Self(QueryLimitsInner::LimitAndOffset(limit, page.into()))) + } else if limit == 1_u64 { + Ok(Self(QueryLimitsInner::One)) + } else { + Ok(Self(QueryLimitsInner::Limit(limit))) + } + } else if page.is_none() { + Ok(Self(QueryLimitsInner::All)) + } else { + anyhow::bail!("Invalid arguments, `limit` must be provided when `page` is not None") + } + } + + /// Create a `QueryLimits` object with the limit equals to `1`. + pub(crate) fn new_one() -> Self { + Self(QueryLimitsInner::One) + } + + /// Create a `QueryLimits` object without the any limits. + pub(crate) fn new_all() -> Self { + Self(QueryLimitsInner::All) + } + + /// Returns a string with the corresponding query limit statement + pub(crate) fn query_stmt(&self) -> String { + match self.0 { + QueryLimitsInner::All => String::new(), + QueryLimitsInner::One => "LIMIT 1".to_string(), + QueryLimitsInner::Limit(limit) => format!("LIMIT {limit}"), + QueryLimitsInner::LimitAndOffset(limit, offset) => { + format!("LIMIT {limit} OFFSET {offset}") + }, + } + } +} diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index ac560735c16..4258f3d056a 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -3,7 +3,7 @@ #[cfg(test)] mod tests; -use super::{common::QueryLimits, EventDB, NotFoundError}; +use super::{common::query_limits::QueryLimits, EventDB, NotFoundError}; use crate::jinja::{get_template, JinjaTemplateSource}; /// Insert sql query diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index 4fd66e8c3a1..0493b937e0b 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -60,16 +60,18 @@ async fn some_test() { }; assert!(insert_signed_docs(&another_doc).await.is_err()); - let res_docs = - filtered_select_signed_docs(&DocsQueryFilter::DocId(doc.body.id), &QueryLimits::All) - .await - .unwrap(); + let res_docs = filtered_select_signed_docs( + &DocsQueryFilter::DocId(doc.body.id), + &QueryLimits::new_all(), + ) + .await + .unwrap(); assert_eq!(res_docs.len(), 1); assert_eq!(&doc.body, res_docs.first().unwrap()); let res_docs = filtered_select_signed_docs( &DocsQueryFilter::DocVer(doc.body.id, doc.body.ver), - &QueryLimits::All, + &QueryLimits::new_all(), ) .await .unwrap(); @@ -78,7 +80,7 @@ async fn some_test() { let res_docs = filtered_select_signed_docs( &DocsQueryFilter::Author(doc.body.author.clone()), - &QueryLimits::All, + &QueryLimits::new_all(), ) .await .unwrap(); @@ -87,7 +89,7 @@ async fn some_test() { } let res_docs = - filtered_select_signed_docs(&DocsQueryFilter::DocType(doc_type), &QueryLimits::All) + filtered_select_signed_docs(&DocsQueryFilter::DocType(doc_type), &QueryLimits::new_all()) .await .unwrap(); assert_eq!( @@ -95,7 +97,7 @@ async fn some_test() { res_docs.iter().rev().collect::>() ); - let res_docs = filtered_select_signed_docs(&DocsQueryFilter::All, &QueryLimits::All) + let res_docs = filtered_select_signed_docs(&DocsQueryFilter::All, &QueryLimits::new_all()) .await .unwrap(); assert_eq!( diff --git a/catalyst-gateway/bin/src/service/common/types/generic/query/pagination.rs b/catalyst-gateway/bin/src/service/common/types/generic/query/pagination.rs index 85824bd4a74..03c413e0d83 100644 --- a/catalyst-gateway/bin/src/service/common/types/generic/query/pagination.rs +++ b/catalyst-gateway/bin/src/service/common/types/generic/query/pagination.rs @@ -125,6 +125,12 @@ impl From for Page { } } +impl From for u64 { + fn from(value: Page) -> Self { + value.0 + } +} + impl Example for Page { fn example() -> Self { Self(PAGE_EXAMPLE) @@ -250,6 +256,12 @@ impl From for Limit { } } +impl From for u64 { + fn from(value: Limit) -> Self { + value.0 + } +} + impl Example for Limit { fn example() -> Self { Self(LIMIT_EXAMPLE) @@ -353,6 +365,12 @@ impl From for Remaining { } } +impl From for u64 { + fn from(value: Remaining) -> Self { + value.0 + } +} + impl Example for Remaining { fn example() -> Self { Self(REMAINING_EXAMPLE) diff --git a/catalyst-gateway/bin/src/service/mod.rs b/catalyst-gateway/bin/src/service/mod.rs index f7571d3fdba..158a631c96d 100644 --- a/catalyst-gateway/bin/src/service/mod.rs +++ b/catalyst-gateway/bin/src/service/mod.rs @@ -5,7 +5,7 @@ mod api; mod docs; // These modules are utility or common types/functions -mod common; +pub(crate) mod common; mod poem_service; pub(crate) mod utilities; From c6bb226578225b01aab7944765e0cc26bb0c53c8 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Wed, 18 Dec 2024 18:45:14 +0200 Subject: [PATCH 23/42] revert select_signed_docs query --- .../bin/src/db/event/signed_docs/mod.rs | 33 ++++++++++++++----- ...iltered_select_signed_documents.sql.jinja} | 0 .../sql/select_signed_documents.sql.jinja | 5 +-- .../bin/src/db/event/signed_docs/tests/mod.rs | 8 +++++ catalyst-gateway/bin/src/jinja.rs | 6 ++-- 5 files changed, 39 insertions(+), 13 deletions(-) rename catalyst-gateway/bin/src/db/event/signed_docs/sql/{advanced_select_signed_documents.sql.jinja => filtered_select_signed_documents.sql.jinja} (100%) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index 4258f3d056a..058e2b007f0 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -15,10 +15,10 @@ pub(crate) const SELECT_SIGNED_DOCS_TEMPLATE: JinjaTemplateSource = JinjaTemplat source: include_str!("./sql/select_signed_documents.sql.jinja"), }; -/// Advanced select sql query jinja template -pub(crate) const ADVANCED_SELECT_SIGNED_DOCS_TEMPLATE: JinjaTemplateSource = JinjaTemplateSource { - name: "advanced_select_signed_documents.jinja.template", - source: include_str!("./sql/advanced_select_signed_documents.sql.jinja"), +/// Filtered select sql query jinja template +pub(crate) const FILTERED_SELECT_SIGNED_DOCS_TEMPLATE: JinjaTemplateSource = JinjaTemplateSource { + name: "filtered_select_signed_documents.jinja.template", + source: include_str!("./sql/filtered_select_signed_documents.sql.jinja"), }; /// Signed doc body event db struct @@ -60,7 +60,7 @@ pub(crate) struct FullSignedDoc { /// - `doc_type` is a UUID v4 #[allow(dead_code)] pub(crate) async fn insert_signed_docs(doc: &FullSignedDoc) -> anyhow::Result<()> { - match select_signed_docs(&doc.body.id, &doc.body.ver).await { + match select_signed_docs(&doc.body.id, &Some(doc.body.ver)).await { Ok(res_doc) => { anyhow::ensure!( &res_doc == doc, @@ -86,7 +86,18 @@ pub(crate) async fn insert_signed_docs(doc: &FullSignedDoc) -> anyhow::Result<() } /// Make a select query into the `event-db` by getting data from the `signed_docs` table. -async fn select_signed_docs(id: &uuid::Uuid, ver: &uuid::Uuid) -> anyhow::Result { +/// +/// * This returns a single document. All data from the document is returned, including +/// the `payload` and `raw` fields. +/// * `ver` should be able to be optional, in which case get the latest ver of the given +/// `id`. +/// +/// # Arguments: +/// - `id` is a UUID v7 +/// - `ver` is a UUID v7 +pub(crate) async fn select_signed_docs( + id: &uuid::Uuid, ver: &Option, +) -> anyhow::Result { let query_template = get_template(&SELECT_SIGNED_DOCS_TEMPLATE)?; let query = query_template.render(serde_json::json!({ "id": id, @@ -94,6 +105,12 @@ async fn select_signed_docs(id: &uuid::Uuid, ver: &uuid::Uuid) -> anyhow::Result }))?; let res = EventDB::query_one(&query, &[]).await?; + let ver = if let Some(ver) = ver { + *ver + } else { + res.try_get("ver")? + }; + let doc_type = res.try_get("type")?; let author = res.try_get("author")?; let metadata = res.try_get("metadata")?; @@ -103,7 +120,7 @@ async fn select_signed_docs(id: &uuid::Uuid, ver: &uuid::Uuid) -> anyhow::Result Ok(FullSignedDoc { body: SignedDocBody { id: *id, - ver: *ver, + ver, doc_type, author, metadata, @@ -149,7 +166,7 @@ impl DocsQueryFilter { pub(crate) async fn filtered_select_signed_docs( conditions: &DocsQueryFilter, query_limits: &QueryLimits, ) -> anyhow::Result> { - let query_template = get_template(&ADVANCED_SELECT_SIGNED_DOCS_TEMPLATE)?; + let query_template = get_template(&FILTERED_SELECT_SIGNED_DOCS_TEMPLATE)?; let query = query_template.render(serde_json::json!({ "conditions": conditions.query_stmt(), "query_limits": query_limits.query_stmt(), diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/sql/advanced_select_signed_documents.sql.jinja b/catalyst-gateway/bin/src/db/event/signed_docs/sql/filtered_select_signed_documents.sql.jinja similarity index 100% rename from catalyst-gateway/bin/src/db/event/signed_docs/sql/advanced_select_signed_documents.sql.jinja rename to catalyst-gateway/bin/src/db/event/signed_docs/sql/filtered_select_signed_documents.sql.jinja diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/sql/select_signed_documents.sql.jinja b/catalyst-gateway/bin/src/db/event/signed_docs/sql/select_signed_documents.sql.jinja index cf4ca4920b5..a1a16a7bcb9 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/sql/select_signed_documents.sql.jinja +++ b/catalyst-gateway/bin/src/db/event/signed_docs/sql/select_signed_documents.sql.jinja @@ -1,4 +1,5 @@ SELECT + {% if not ver %} signed_docs.ver, {% endif %} signed_docs.type, signed_docs.author, signed_docs.metadata, @@ -7,6 +8,6 @@ SELECT FROM signed_docs WHERE signed_docs.id = '{{ id }}' - AND signed_docs.ver = '{{ ver }}' + {% if ver %} AND signed_docs.ver = '{{ ver }}' {% endif %} ORDER BY signed_docs.ver DESC -LIMIT 1 +LIMIT 1 \ No newline at end of file diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index 0493b937e0b..194dd9a386c 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -60,6 +60,14 @@ async fn some_test() { }; assert!(insert_signed_docs(&another_doc).await.is_err()); + let res_doc = select_signed_docs(&doc.body.id, &Some(doc.body.ver)) + .await + .unwrap(); + assert_eq!(doc, &res_doc); + + let res_doc = select_signed_docs(&doc.body.id, &None).await.unwrap(); + assert_eq!(doc, &res_doc); + let res_docs = filtered_select_signed_docs( &DocsQueryFilter::DocId(doc.body.id), &QueryLimits::new_all(), diff --git a/catalyst-gateway/bin/src/jinja.rs b/catalyst-gateway/bin/src/jinja.rs index f8c1534e7bf..0af9032f5aa 100644 --- a/catalyst-gateway/bin/src/jinja.rs +++ b/catalyst-gateway/bin/src/jinja.rs @@ -5,7 +5,7 @@ use std::sync::LazyLock; use minijinja::{Environment, Template}; use crate::db::event::signed_docs::{ - ADVANCED_SELECT_SIGNED_DOCS_TEMPLATE, SELECT_SIGNED_DOCS_TEMPLATE, + FILTERED_SELECT_SIGNED_DOCS_TEMPLATE, SELECT_SIGNED_DOCS_TEMPLATE, }; /// Jinja template source struct. @@ -28,8 +28,8 @@ static JINJA_ENV: LazyLock = LazyLock::new(|| { ) .unwrap(); env.add_template( - ADVANCED_SELECT_SIGNED_DOCS_TEMPLATE.name, - ADVANCED_SELECT_SIGNED_DOCS_TEMPLATE.source, + FILTERED_SELECT_SIGNED_DOCS_TEMPLATE.name, + FILTERED_SELECT_SIGNED_DOCS_TEMPLATE.source, ) .unwrap(); From 5315e5933b5037626434af07eb51c9f76b8598ab Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Wed, 18 Dec 2024 19:02:46 +0200 Subject: [PATCH 24/42] refactor QueryLimits --- .../bin/src/db/event/common/query_limits.rs | 37 ++++++--------- .../common/objects/generic/pagination.rs | 15 +++--- .../common/types/generic/query/pagination.rs | 46 +++++++++++-------- 3 files changed, 50 insertions(+), 48 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/common/query_limits.rs b/catalyst-gateway/bin/src/db/event/common/query_limits.rs index 69d63355081..adb8901a6cc 100644 --- a/catalyst-gateway/bin/src/db/event/common/query_limits.rs +++ b/catalyst-gateway/bin/src/db/event/common/query_limits.rs @@ -2,8 +2,6 @@ #![allow(dead_code)] -use anyhow::ensure; - use crate::service::common::types::generic::query::pagination::{Limit, Page}; /// A query limits struct. @@ -13,8 +11,6 @@ pub(crate) struct QueryLimits(QueryLimitsInner); enum QueryLimitsInner { /// Return all entries without any `LIMIT` and `OFFSET` parameters All, - /// Specifies `LIMIT` parameter equals to `1` - One, /// Specifies `LIMIT` parameter Limit(u64), /// Specifies `LIMIT` and `OFFSET` parameters @@ -28,30 +24,24 @@ impl QueryLimits { /// - Invalid `limit` value, must be more than `0`. /// - Invalid arguments, `limit` must be provided when `page` is not None. pub(crate) fn new(limit: Option, page: Option) -> anyhow::Result { - if let Some(limit) = limit { - let limit = limit.into(); - ensure!( - limit != 0_u64, - "Invalid `limit` value, must be more than `0`" - ); - - if let Some(page) = page { - Ok(Self(QueryLimitsInner::LimitAndOffset(limit, page.into()))) - } else if limit == 1_u64 { - Ok(Self(QueryLimitsInner::One)) - } else { - Ok(Self(QueryLimitsInner::Limit(limit))) - } - } else if page.is_none() { - Ok(Self(QueryLimitsInner::All)) - } else { - anyhow::bail!("Invalid arguments, `limit` must be provided when `page` is not None") + match (limit, page) { + (Some(limit), Some(page)) => { + Ok(Self(QueryLimitsInner::LimitAndOffset( + limit.into(), + page.into(), + ))) + }, + (Some(limit), None) => Ok(Self(QueryLimitsInner::Limit(limit.into()))), + (None, None) => Ok(Self(QueryLimitsInner::All)), + (None, Some(_)) => { + anyhow::bail!("Invalid arguments, `limit` must be provided when `page` is not None") + }, } } /// Create a `QueryLimits` object with the limit equals to `1`. pub(crate) fn new_one() -> Self { - Self(QueryLimitsInner::One) + Self(QueryLimitsInner::Limit(1)) } /// Create a `QueryLimits` object without the any limits. @@ -63,7 +53,6 @@ impl QueryLimits { pub(crate) fn query_stmt(&self) -> String { match self.0 { QueryLimitsInner::All => String::new(), - QueryLimitsInner::One => "LIMIT 1".to_string(), QueryLimitsInner::Limit(limit) => format!("LIMIT {limit}"), QueryLimitsInner::LimitAndOffset(limit, offset) => { format!("LIMIT {limit} OFFSET {offset}") diff --git a/catalyst-gateway/bin/src/service/common/objects/generic/pagination.rs b/catalyst-gateway/bin/src/service/common/objects/generic/pagination.rs index 278fbc65d5d..43d52dcd7e8 100644 --- a/catalyst-gateway/bin/src/service/common/objects/generic/pagination.rs +++ b/catalyst-gateway/bin/src/service/common/objects/generic/pagination.rs @@ -37,12 +37,15 @@ impl Example for CurrentPage { impl CurrentPage { /// Create a new `CurrentPage` object. + /// + /// # Errors + /// - Invalid `limit` value, must be more than `0` #[allow(dead_code)] - fn new(page: u64, limit: u64, remaining: u64) -> Self { - Self { - page: page.into(), - limit: limit.into(), - remaining: remaining.into(), - } + fn new(page: u64, limit: u64, remaining: u64) -> anyhow::Result { + Ok(Self { + page: common::types::generic::query::pagination::Page::new(page), + limit: common::types::generic::query::pagination::Limit::new(limit)?, + remaining: common::types::generic::query::pagination::Remaining::new(remaining), + }) } } diff --git a/catalyst-gateway/bin/src/service/common/types/generic/query/pagination.rs b/catalyst-gateway/bin/src/service/common/types/generic/query/pagination.rs index 03c413e0d83..bd27e05ce60 100644 --- a/catalyst-gateway/bin/src/service/common/types/generic/query/pagination.rs +++ b/catalyst-gateway/bin/src/service/common/types/generic/query/pagination.rs @@ -53,6 +53,13 @@ static PAGE_SCHEMA: LazyLock = LazyLock::new(|| { #[derive(Debug, Eq, PartialEq, Hash)] pub(crate) struct Page(u64); +impl Page { + /// Creates a new `Page` instance. + pub(crate) fn new(page: u64) -> Self { + Self(page) + } +} + impl Default for Page { fn default() -> Self { Self(PAGE_DEFAULT) @@ -119,12 +126,6 @@ impl ToJSON for Page { } } -impl From for Page { - fn from(value: u64) -> Self { - Self(value) - } -} - impl From for u64 { fn from(value: Page) -> Self { value.0 @@ -184,6 +185,20 @@ static LIMIT_SCHEMA: LazyLock = LazyLock::new(|| { #[derive(Debug, Eq, PartialEq, Hash)] pub(crate) struct Limit(u64); +impl Limit { + /// Creates a new `Limit` instance. + /// + /// # Errors: + /// - Invalid `limit` value, must be more than `0` + pub(crate) fn new(limit: u64) -> anyhow::Result { + anyhow::ensure!( + limit != 0_u64, + "Invalid `limit` value, must be more than `0`" + ); + Ok(Self(limit)) + } +} + impl Default for Limit { fn default() -> Self { Self(LIMIT_DEFAULT) @@ -250,12 +265,6 @@ impl ToJSON for Limit { } } -impl From for Limit { - fn from(value: u64) -> Self { - Self(value) - } -} - impl From for u64 { fn from(value: Limit) -> Self { value.0 @@ -306,6 +315,13 @@ static REMAINING_SCHEMA: LazyLock = LazyLock::new(|| { #[derive(Debug, Eq, PartialEq, Hash)] pub(crate) struct Remaining(u64); +impl Remaining { + /// Creates a new `Remaining` instance. + pub(crate) fn new(remaining: u64) -> Self { + Self(remaining) + } +} + /// Is the `Page` valid? fn is_valid_remaining(value: u64) -> bool { (REMAINING_MINIMUM..=REMAINING_MAXIMUM).contains(&value) @@ -359,12 +375,6 @@ impl ToJSON for Remaining { } } -impl From for Remaining { - fn from(value: u64) -> Self { - Self(value) - } -} - impl From for u64 { fn from(value: Remaining) -> Self { value.0 From eaa0e36f32a6d2b6de8ecaeb82135d4053fc5524 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Wed, 18 Dec 2024 19:11:19 +0200 Subject: [PATCH 25/42] wip --- .../common/objects/generic/pagination.rs | 8 ++++-- .../common/types/generic/query/pagination.rs | 28 ++++++++++++++----- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/catalyst-gateway/bin/src/service/common/objects/generic/pagination.rs b/catalyst-gateway/bin/src/service/common/objects/generic/pagination.rs index 43d52dcd7e8..b3313b43aa5 100644 --- a/catalyst-gateway/bin/src/service/common/objects/generic/pagination.rs +++ b/catalyst-gateway/bin/src/service/common/objects/generic/pagination.rs @@ -39,13 +39,15 @@ impl CurrentPage { /// Create a new `CurrentPage` object. /// /// # Errors - /// - Invalid `limit` value, must be more than `0` + /// - Invalid `page` value, must be in range + /// - Invalid `limit` value, must be in range + /// - Invalid `remaining` value, must be in range #[allow(dead_code)] fn new(page: u64, limit: u64, remaining: u64) -> anyhow::Result { Ok(Self { - page: common::types::generic::query::pagination::Page::new(page), + page: common::types::generic::query::pagination::Page::new(page)?, limit: common::types::generic::query::pagination::Limit::new(limit)?, - remaining: common::types::generic::query::pagination::Remaining::new(remaining), + remaining: common::types::generic::query::pagination::Remaining::new(remaining)?, }) } } diff --git a/catalyst-gateway/bin/src/service/common/types/generic/query/pagination.rs b/catalyst-gateway/bin/src/service/common/types/generic/query/pagination.rs index bd27e05ce60..7410aa951b6 100644 --- a/catalyst-gateway/bin/src/service/common/types/generic/query/pagination.rs +++ b/catalyst-gateway/bin/src/service/common/types/generic/query/pagination.rs @@ -55,8 +55,15 @@ pub(crate) struct Page(u64); impl Page { /// Creates a new `Page` instance. - pub(crate) fn new(page: u64) -> Self { - Self(page) + /// + /// # Errors: + /// - Invalid `page` value, must be in range. + pub(crate) fn new(page: u64) -> anyhow::Result { + anyhow::ensure!( + is_valid_page(page), + "Invalid `page` value, must be in range {PAGE_MINIMUM}..{PAGE_MAXIMUM}" + ); + Ok(Self(page)) } } @@ -189,11 +196,11 @@ impl Limit { /// Creates a new `Limit` instance. /// /// # Errors: - /// - Invalid `limit` value, must be more than `0` + /// - Invalid `limit` value, must be in range pub(crate) fn new(limit: u64) -> anyhow::Result { anyhow::ensure!( - limit != 0_u64, - "Invalid `limit` value, must be more than `0`" + is_valid_limit(limit), + "Invalid `limit` value, must be in range {LIMIT_MINIMUM}..{LIMIT_MAXIMUM}" ); Ok(Self(limit)) } @@ -317,8 +324,15 @@ pub(crate) struct Remaining(u64); impl Remaining { /// Creates a new `Remaining` instance. - pub(crate) fn new(remaining: u64) -> Self { - Self(remaining) + /// + /// # Errors: + /// - Invalid `remaining` value, must be in range + pub(crate) fn new(remaining: u64) -> anyhow::Result { + anyhow::ensure!( + is_valid_remaining(remaining), + "Invalid `remaining` value, must be in range {REMAINING_MINIMUM}..{REMAINING_MAXIMUM}" + ); + Ok(Self(remaining)) } } From e14fe9680795a5d60065b1e0719658ee075d8e23 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Thu, 19 Dec 2024 10:02:14 +0200 Subject: [PATCH 26/42] rename QueryLimits methods --- .../bin/src/db/event/common/query_limits.rs | 4 ++-- .../bin/src/db/event/signed_docs/tests/mod.rs | 18 ++++++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/common/query_limits.rs b/catalyst-gateway/bin/src/db/event/common/query_limits.rs index adb8901a6cc..48075a04714 100644 --- a/catalyst-gateway/bin/src/db/event/common/query_limits.rs +++ b/catalyst-gateway/bin/src/db/event/common/query_limits.rs @@ -40,12 +40,12 @@ impl QueryLimits { } /// Create a `QueryLimits` object with the limit equals to `1`. - pub(crate) fn new_one() -> Self { + pub(crate) const fn one() -> Self { Self(QueryLimitsInner::Limit(1)) } /// Create a `QueryLimits` object without the any limits. - pub(crate) fn new_all() -> Self { + pub(crate) const fn all() -> Self { Self(QueryLimitsInner::All) } diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index 194dd9a386c..ff93b24e7b2 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -68,18 +68,16 @@ async fn some_test() { let res_doc = select_signed_docs(&doc.body.id, &None).await.unwrap(); assert_eq!(doc, &res_doc); - let res_docs = filtered_select_signed_docs( - &DocsQueryFilter::DocId(doc.body.id), - &QueryLimits::new_all(), - ) - .await - .unwrap(); + let res_docs = + filtered_select_signed_docs(&DocsQueryFilter::DocId(doc.body.id), &QueryLimits::all()) + .await + .unwrap(); assert_eq!(res_docs.len(), 1); assert_eq!(&doc.body, res_docs.first().unwrap()); let res_docs = filtered_select_signed_docs( &DocsQueryFilter::DocVer(doc.body.id, doc.body.ver), - &QueryLimits::new_all(), + &QueryLimits::all(), ) .await .unwrap(); @@ -88,7 +86,7 @@ async fn some_test() { let res_docs = filtered_select_signed_docs( &DocsQueryFilter::Author(doc.body.author.clone()), - &QueryLimits::new_all(), + &QueryLimits::all(), ) .await .unwrap(); @@ -97,7 +95,7 @@ async fn some_test() { } let res_docs = - filtered_select_signed_docs(&DocsQueryFilter::DocType(doc_type), &QueryLimits::new_all()) + filtered_select_signed_docs(&DocsQueryFilter::DocType(doc_type), &QueryLimits::all()) .await .unwrap(); assert_eq!( @@ -105,7 +103,7 @@ async fn some_test() { res_docs.iter().rev().collect::>() ); - let res_docs = filtered_select_signed_docs(&DocsQueryFilter::All, &QueryLimits::new_all()) + let res_docs = filtered_select_signed_docs(&DocsQueryFilter::All, &QueryLimits::all()) .await .unwrap(); assert_eq!( From b46043ace9a1430f49c26870c4481f59507a87cf Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Thu, 19 Dec 2024 10:09:09 +0200 Subject: [PATCH 27/42] replace `new` with TryFrom impl --- .../common/objects/generic/pagination.rs | 6 +- .../common/types/generic/query/pagination.rs | 78 +++++++++---------- 2 files changed, 39 insertions(+), 45 deletions(-) diff --git a/catalyst-gateway/bin/src/service/common/objects/generic/pagination.rs b/catalyst-gateway/bin/src/service/common/objects/generic/pagination.rs index b3313b43aa5..ee92db00ae2 100644 --- a/catalyst-gateway/bin/src/service/common/objects/generic/pagination.rs +++ b/catalyst-gateway/bin/src/service/common/objects/generic/pagination.rs @@ -45,9 +45,9 @@ impl CurrentPage { #[allow(dead_code)] fn new(page: u64, limit: u64, remaining: u64) -> anyhow::Result { Ok(Self { - page: common::types::generic::query::pagination::Page::new(page)?, - limit: common::types::generic::query::pagination::Limit::new(limit)?, - remaining: common::types::generic::query::pagination::Remaining::new(remaining)?, + page: page.try_into()?, + limit: limit.try_into()?, + remaining: remaining.try_into()?, }) } } diff --git a/catalyst-gateway/bin/src/service/common/types/generic/query/pagination.rs b/catalyst-gateway/bin/src/service/common/types/generic/query/pagination.rs index 7410aa951b6..6c521c54c7e 100644 --- a/catalyst-gateway/bin/src/service/common/types/generic/query/pagination.rs +++ b/catalyst-gateway/bin/src/service/common/types/generic/query/pagination.rs @@ -53,20 +53,6 @@ static PAGE_SCHEMA: LazyLock = LazyLock::new(|| { #[derive(Debug, Eq, PartialEq, Hash)] pub(crate) struct Page(u64); -impl Page { - /// Creates a new `Page` instance. - /// - /// # Errors: - /// - Invalid `page` value, must be in range. - pub(crate) fn new(page: u64) -> anyhow::Result { - anyhow::ensure!( - is_valid_page(page), - "Invalid `page` value, must be in range {PAGE_MINIMUM}..{PAGE_MAXIMUM}" - ); - Ok(Self(page)) - } -} - impl Default for Page { fn default() -> Self { Self(PAGE_DEFAULT) @@ -133,6 +119,18 @@ impl ToJSON for Page { } } +impl TryFrom for Page { + type Error = anyhow::Error; + + fn try_from(page: u64) -> Result { + anyhow::ensure!( + is_valid_page(page), + "Invalid `page` value, must be in range {PAGE_MINIMUM}..{PAGE_MAXIMUM}" + ); + Ok(Self(page)) + } +} + impl From for u64 { fn from(value: Page) -> Self { value.0 @@ -192,20 +190,6 @@ static LIMIT_SCHEMA: LazyLock = LazyLock::new(|| { #[derive(Debug, Eq, PartialEq, Hash)] pub(crate) struct Limit(u64); -impl Limit { - /// Creates a new `Limit` instance. - /// - /// # Errors: - /// - Invalid `limit` value, must be in range - pub(crate) fn new(limit: u64) -> anyhow::Result { - anyhow::ensure!( - is_valid_limit(limit), - "Invalid `limit` value, must be in range {LIMIT_MINIMUM}..{LIMIT_MAXIMUM}" - ); - Ok(Self(limit)) - } -} - impl Default for Limit { fn default() -> Self { Self(LIMIT_DEFAULT) @@ -272,6 +256,18 @@ impl ToJSON for Limit { } } +impl TryFrom for Limit { + type Error = anyhow::Error; + + fn try_from(limit: u64) -> Result { + anyhow::ensure!( + is_valid_limit(limit), + "Invalid `limit` value, must be in range {LIMIT_MINIMUM}..{LIMIT_MAXIMUM}" + ); + Ok(Self(limit)) + } +} + impl From for u64 { fn from(value: Limit) -> Self { value.0 @@ -322,20 +318,6 @@ static REMAINING_SCHEMA: LazyLock = LazyLock::new(|| { #[derive(Debug, Eq, PartialEq, Hash)] pub(crate) struct Remaining(u64); -impl Remaining { - /// Creates a new `Remaining` instance. - /// - /// # Errors: - /// - Invalid `remaining` value, must be in range - pub(crate) fn new(remaining: u64) -> anyhow::Result { - anyhow::ensure!( - is_valid_remaining(remaining), - "Invalid `remaining` value, must be in range {REMAINING_MINIMUM}..{REMAINING_MAXIMUM}" - ); - Ok(Self(remaining)) - } -} - /// Is the `Page` valid? fn is_valid_remaining(value: u64) -> bool { (REMAINING_MINIMUM..=REMAINING_MAXIMUM).contains(&value) @@ -389,6 +371,18 @@ impl ToJSON for Remaining { } } +impl TryFrom for Remaining { + type Error = anyhow::Error; + + fn try_from(remaining: u64) -> Result { + anyhow::ensure!( + is_valid_remaining(remaining), + "Invalid `remaining` value, must be in range {REMAINING_MINIMUM}..{REMAINING_MAXIMUM}" + ); + Ok(Self(remaining)) + } +} + impl From for u64 { fn from(value: Remaining) -> Self { value.0 From 1dbdd509747962c598e581b4071468200c03d63b Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Thu, 19 Dec 2024 10:15:21 +0200 Subject: [PATCH 28/42] wip --- .../bin/src/db/event/common/query_limits.rs | 15 +++++---------- .../bin/src/db/event/signed_docs/tests/mod.rs | 10 +++++----- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/common/query_limits.rs b/catalyst-gateway/bin/src/db/event/common/query_limits.rs index 48075a04714..5d450894714 100644 --- a/catalyst-gateway/bin/src/db/event/common/query_limits.rs +++ b/catalyst-gateway/bin/src/db/event/common/query_limits.rs @@ -18,6 +18,11 @@ enum QueryLimitsInner { } impl QueryLimits { + /// Create a `QueryLimits` object without the any limits. + pub(crate) const ALL: QueryLimits = Self(QueryLimitsInner::All); + /// Create a `QueryLimits` object with the limit equals to `1`. + pub(crate) const ONE: QueryLimits = Self(QueryLimitsInner::Limit(1)); + /// Create a `QueryLimits` object from the service `Limit` and `Page` values. /// /// # Errors @@ -39,16 +44,6 @@ impl QueryLimits { } } - /// Create a `QueryLimits` object with the limit equals to `1`. - pub(crate) const fn one() -> Self { - Self(QueryLimitsInner::Limit(1)) - } - - /// Create a `QueryLimits` object without the any limits. - pub(crate) const fn all() -> Self { - Self(QueryLimitsInner::All) - } - /// Returns a string with the corresponding query limit statement pub(crate) fn query_stmt(&self) -> String { match self.0 { diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index ff93b24e7b2..fe2ef2fc48d 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -69,7 +69,7 @@ async fn some_test() { assert_eq!(doc, &res_doc); let res_docs = - filtered_select_signed_docs(&DocsQueryFilter::DocId(doc.body.id), &QueryLimits::all()) + filtered_select_signed_docs(&DocsQueryFilter::DocId(doc.body.id), &QueryLimits::ALL) .await .unwrap(); assert_eq!(res_docs.len(), 1); @@ -77,7 +77,7 @@ async fn some_test() { let res_docs = filtered_select_signed_docs( &DocsQueryFilter::DocVer(doc.body.id, doc.body.ver), - &QueryLimits::all(), + &QueryLimits::ALL, ) .await .unwrap(); @@ -86,7 +86,7 @@ async fn some_test() { let res_docs = filtered_select_signed_docs( &DocsQueryFilter::Author(doc.body.author.clone()), - &QueryLimits::all(), + &QueryLimits::ALL, ) .await .unwrap(); @@ -95,7 +95,7 @@ async fn some_test() { } let res_docs = - filtered_select_signed_docs(&DocsQueryFilter::DocType(doc_type), &QueryLimits::all()) + filtered_select_signed_docs(&DocsQueryFilter::DocType(doc_type), &QueryLimits::ALL) .await .unwrap(); assert_eq!( @@ -103,7 +103,7 @@ async fn some_test() { res_docs.iter().rev().collect::>() ); - let res_docs = filtered_select_signed_docs(&DocsQueryFilter::All, &QueryLimits::all()) + let res_docs = filtered_select_signed_docs(&DocsQueryFilter::All, &QueryLimits::ALL) .await .unwrap(); assert_eq!( From 7e09a233b8140efdb9bbc69e3c53f3b8ab8f4543 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Thu, 19 Dec 2024 10:20:40 +0200 Subject: [PATCH 29/42] wip --- .../db/event/signed_docs/full_signed_doc.rs | 14 +++++++++ .../bin/src/db/event/signed_docs/mod.rs | 31 +++---------------- .../db/event/signed_docs/signed_doc_body.rs | 16 ++++++++++ 3 files changed, 35 insertions(+), 26 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs create mode 100644 catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs b/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs new file mode 100644 index 00000000000..44f58fc15b3 --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs @@ -0,0 +1,14 @@ +//! `FullSignedDoc` struct implementation. + +use super::SignedDocBody; + +/// Full signed doc event db struct +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct FullSignedDoc { + /// Signed doc body + pub(crate) body: SignedDocBody, + /// `signed_doc` table `payload` field + pub(crate) payload: Option, + /// `signed_doc` table `raw` field + pub(crate) raw: Vec, +} diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index 058e2b007f0..07f162c27b1 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -1,8 +1,13 @@ //! Signed docs queries +mod full_signed_doc; +mod signed_doc_body; #[cfg(test)] mod tests; +pub(crate) use full_signed_doc::FullSignedDoc; +pub(crate) use signed_doc_body::SignedDocBody; + use super::{common::query_limits::QueryLimits, EventDB, NotFoundError}; use crate::jinja::{get_template, JinjaTemplateSource}; @@ -21,32 +26,6 @@ pub(crate) const FILTERED_SELECT_SIGNED_DOCS_TEMPLATE: JinjaTemplateSource = Jin source: include_str!("./sql/filtered_select_signed_documents.sql.jinja"), }; -/// Signed doc body event db struct -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct SignedDocBody { - /// `signed_doc` table `id` field - id: uuid::Uuid, - /// `signed_doc` table `ver` field - ver: uuid::Uuid, - /// `signed_doc` table `type` field - doc_type: uuid::Uuid, - /// `signed_doc` table `author` field - author: String, - /// `signed_doc` table `metadata` field - metadata: Option, -} - -/// Full signed doc event db struct -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct FullSignedDoc { - /// Signed doc body - body: SignedDocBody, - /// `signed_doc` table `payload` field - payload: Option, - /// `signed_doc` table `raw` field - raw: Vec, -} - /// Make an insert query into the `event-db` by adding data into the `signed_docs` table. /// /// * IF the record primary key (id,ver) does not exist, then add the new record. Return diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs b/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs new file mode 100644 index 00000000000..199dece453e --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs @@ -0,0 +1,16 @@ +//! `SignedDocBody` struct implementation. + +/// Signed doc body event db struct +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct SignedDocBody { + /// `signed_doc` table `id` field + pub(crate) id: uuid::Uuid, + /// `signed_doc` table `ver` field + pub(crate) ver: uuid::Uuid, + /// `signed_doc` table `type` field + pub(crate) doc_type: uuid::Uuid, + /// `signed_doc` table `author` field + pub(crate) author: String, + /// `signed_doc` table `metadata` field + pub(crate) metadata: Option, +} From 780c7da6991a358d76d3cd898d3c857d88a2d058 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Thu, 19 Dec 2024 10:44:05 +0200 Subject: [PATCH 30/42] wip --- .../db/event/signed_docs/full_signed_doc.rs | 25 +++++++++++ .../bin/src/db/event/signed_docs/mod.rs | 43 ++----------------- .../db/event/signed_docs/signed_doc_body.rs | 18 ++++++++ 3 files changed, 47 insertions(+), 39 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs b/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs index 44f58fc15b3..807053bc4bf 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs @@ -12,3 +12,28 @@ pub(crate) struct FullSignedDoc { /// `signed_doc` table `raw` field pub(crate) raw: Vec, } + +impl FullSignedDoc { + /// Creates a `FullSignedDoc` from postgresql row object. + pub(crate) fn from_row( + id: &uuid::Uuid, ver: &Option, row: &tokio_postgres::Row, + ) -> anyhow::Result { + let ver = if let Some(ver) = ver { + *ver + } else { + row.try_get("ver")? + }; + + Ok(FullSignedDoc { + body: SignedDocBody { + id: *id, + ver, + doc_type: row.try_get("type")?, + author: row.try_get("author")?, + metadata: row.try_get("metadata")?, + }, + payload: row.try_get("payload")?, + raw: row.try_get("raw")?, + }) + } +} diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index 07f162c27b1..18d9657cec3 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -82,31 +82,9 @@ pub(crate) async fn select_signed_docs( "id": id, "ver": ver, }))?; - let res = EventDB::query_one(&query, &[]).await?; + let row = EventDB::query_one(&query, &[]).await?; - let ver = if let Some(ver) = ver { - *ver - } else { - res.try_get("ver")? - }; - - let doc_type = res.try_get("type")?; - let author = res.try_get("author")?; - let metadata = res.try_get("metadata")?; - let payload = res.try_get("payload")?; - let raw = res.try_get("raw")?; - - Ok(FullSignedDoc { - body: SignedDocBody { - id: *id, - ver, - doc_type, - author, - metadata, - }, - payload, - raw, - }) + FullSignedDoc::from_row(id, ver, &row) } /// A `select_signed_docs` query filtering argument. @@ -153,21 +131,8 @@ pub(crate) async fn filtered_select_signed_docs( let rows = EventDB::query(&query, &[]).await?; let docs = rows - .into_iter() - .map(|row| { - let id = row.try_get("id")?; - let ver = row.try_get("ver")?; - let doc_type = row.try_get("type")?; - let author = row.try_get("author")?; - let metadata = row.try_get("metadata")?; - Ok(SignedDocBody { - id, - ver, - doc_type, - author, - metadata, - }) - }) + .iter() + .map(SignedDocBody::from_row) .collect::>()?; Ok(docs) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs b/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs index 199dece453e..41149f35577 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs @@ -14,3 +14,21 @@ pub(crate) struct SignedDocBody { /// `signed_doc` table `metadata` field pub(crate) metadata: Option, } + +impl SignedDocBody { + /// Creates a `SignedDocBody` from postgresql row object. + pub(crate) fn from_row(row: &tokio_postgres::Row) -> anyhow::Result { + let id = row.try_get("id")?; + let ver = row.try_get("ver")?; + let doc_type = row.try_get("type")?; + let author = row.try_get("author")?; + let metadata = row.try_get("metadata")?; + Ok(Self { + id, + ver, + doc_type, + author, + metadata, + }) + } +} From e0e78ac02f0e40e1b8bde61c91edaa6eafb9c9a7 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Thu, 19 Dec 2024 12:14:47 +0200 Subject: [PATCH 31/42] refactor --- .../db/event/signed_docs/full_signed_doc.rs | 39 ------ .../bin/src/db/event/signed_docs/mod.rs | 21 +-- .../db/event/signed_docs/signed_doc_body.rs | 34 ----- .../bin/src/db/event/signed_docs/tests/mod.rs | 96 +++++++------ .../bin/src/db/event/signed_docs/types.rs | 129 ++++++++++++++++++ 5 files changed, 180 insertions(+), 139 deletions(-) delete mode 100644 catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs delete mode 100644 catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs create mode 100644 catalyst-gateway/bin/src/db/event/signed_docs/types.rs diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs b/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs deleted file mode 100644 index 807053bc4bf..00000000000 --- a/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs +++ /dev/null @@ -1,39 +0,0 @@ -//! `FullSignedDoc` struct implementation. - -use super::SignedDocBody; - -/// Full signed doc event db struct -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct FullSignedDoc { - /// Signed doc body - pub(crate) body: SignedDocBody, - /// `signed_doc` table `payload` field - pub(crate) payload: Option, - /// `signed_doc` table `raw` field - pub(crate) raw: Vec, -} - -impl FullSignedDoc { - /// Creates a `FullSignedDoc` from postgresql row object. - pub(crate) fn from_row( - id: &uuid::Uuid, ver: &Option, row: &tokio_postgres::Row, - ) -> anyhow::Result { - let ver = if let Some(ver) = ver { - *ver - } else { - row.try_get("ver")? - }; - - Ok(FullSignedDoc { - body: SignedDocBody { - id: *id, - ver, - doc_type: row.try_get("type")?, - author: row.try_get("author")?, - metadata: row.try_get("metadata")?, - }, - payload: row.try_get("payload")?, - raw: row.try_get("raw")?, - }) - } -} diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index 18d9657cec3..b100311dd5d 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -1,12 +1,10 @@ //! Signed docs queries -mod full_signed_doc; -mod signed_doc_body; +mod types; #[cfg(test)] mod tests; -pub(crate) use full_signed_doc::FullSignedDoc; -pub(crate) use signed_doc_body::SignedDocBody; +pub(crate) use types::{FullSignedDoc, SignedDocBody}; use super::{common::query_limits::QueryLimits, EventDB, NotFoundError}; use crate::jinja::{get_template, JinjaTemplateSource}; @@ -39,7 +37,7 @@ pub(crate) const FILTERED_SELECT_SIGNED_DOCS_TEMPLATE: JinjaTemplateSource = Jin /// - `doc_type` is a UUID v4 #[allow(dead_code)] pub(crate) async fn insert_signed_docs(doc: &FullSignedDoc) -> anyhow::Result<()> { - match select_signed_docs(&doc.body.id, &Some(doc.body.ver)).await { + match select_signed_docs(doc.id(), Some(doc.ver())).await { Ok(res_doc) => { anyhow::ensure!( &res_doc == doc, @@ -51,16 +49,7 @@ pub(crate) async fn insert_signed_docs(doc: &FullSignedDoc) -> anyhow::Result<() Err(err) => return Err(err), } - EventDB::modify(INSERT_SIGNED_DOCS, &[ - &doc.body.id, - &doc.body.ver, - &doc.body.doc_type, - &doc.body.author, - &doc.body.metadata, - &doc.payload, - &doc.raw, - ]) - .await?; + EventDB::modify(INSERT_SIGNED_DOCS, &doc.db_fields()).await?; Ok(()) } @@ -75,7 +64,7 @@ pub(crate) async fn insert_signed_docs(doc: &FullSignedDoc) -> anyhow::Result<() /// - `id` is a UUID v7 /// - `ver` is a UUID v7 pub(crate) async fn select_signed_docs( - id: &uuid::Uuid, ver: &Option, + id: &uuid::Uuid, ver: Option<&uuid::Uuid>, ) -> anyhow::Result { let query_template = get_template(&SELECT_SIGNED_DOCS_TEMPLATE)?; let query = query_template.render(serde_json::json!({ diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs b/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs deleted file mode 100644 index 41149f35577..00000000000 --- a/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! `SignedDocBody` struct implementation. - -/// Signed doc body event db struct -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct SignedDocBody { - /// `signed_doc` table `id` field - pub(crate) id: uuid::Uuid, - /// `signed_doc` table `ver` field - pub(crate) ver: uuid::Uuid, - /// `signed_doc` table `type` field - pub(crate) doc_type: uuid::Uuid, - /// `signed_doc` table `author` field - pub(crate) author: String, - /// `signed_doc` table `metadata` field - pub(crate) metadata: Option, -} - -impl SignedDocBody { - /// Creates a `SignedDocBody` from postgresql row object. - pub(crate) fn from_row(row: &tokio_postgres::Row) -> anyhow::Result { - let id = row.try_get("id")?; - let ver = row.try_get("ver")?; - let doc_type = row.try_get("type")?; - let author = row.try_get("author")?; - let metadata = row.try_get("metadata")?; - Ok(Self { - id, - ver, - doc_type, - author, - metadata, - }) - } -} diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index fe2ef2fc48d..e28516ba38a 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -11,87 +11,83 @@ async fn some_test() { let doc_type = uuid::Uuid::new_v4(); let docs = vec![ - FullSignedDoc { - body: SignedDocBody { - id: uuid::Uuid::now_v7(), - ver: uuid::Uuid::now_v7(), + FullSignedDoc::new( + SignedDocBody::new( + uuid::Uuid::now_v7(), + uuid::Uuid::now_v7(), doc_type, - author: "Alex".to_string(), - metadata: Some(serde_json::Value::Null), - }, - payload: Some(serde_json::Value::Null), - raw: vec![1, 2, 3, 4], - }, - FullSignedDoc { - body: SignedDocBody { - id: uuid::Uuid::now_v7(), - ver: uuid::Uuid::now_v7(), + "Alex".to_string(), + Some(serde_json::Value::Null), + ), + Some(serde_json::Value::Null), + vec![1, 2, 3, 4], + ), + FullSignedDoc::new( + SignedDocBody::new( + uuid::Uuid::now_v7(), + uuid::Uuid::now_v7(), doc_type, - author: "Steven".to_string(), - metadata: Some(serde_json::Value::Null), - }, - payload: Some(serde_json::Value::Null), - raw: vec![5, 6, 7, 8], - }, - FullSignedDoc { - body: SignedDocBody { - id: uuid::Uuid::now_v7(), - ver: uuid::Uuid::now_v7(), + "Steven".to_string(), + Some(serde_json::Value::Null), + ), + Some(serde_json::Value::Null), + vec![5, 6, 7, 8], + ), + FullSignedDoc::new( + SignedDocBody::new( + uuid::Uuid::now_v7(), + uuid::Uuid::now_v7(), doc_type, - author: "Sasha".to_string(), - metadata: None, - }, - payload: None, - raw: vec![9, 10, 11, 12], - }, + "Sasha".to_string(), + None, + ), + None, + vec![9, 10, 11, 12], + ), ]; for doc in &docs { insert_signed_docs(doc).await.unwrap(); // try to insert the same data again insert_signed_docs(doc).await.unwrap(); - // try another doc with different `author` and same other fields - let another_doc = FullSignedDoc { - body: SignedDocBody { - author: "Neil".to_string(), - ..doc.body.clone() - }, - ..doc.clone() - }; + // try another doc with the same `id` and `ver` and with different other fields + let another_doc = FullSignedDoc::new( + SignedDocBody::new(*doc.id(), *doc.ver(), doc_type, "Neil".to_string(), None), + None, + vec![], + ); assert!(insert_signed_docs(&another_doc).await.is_err()); - let res_doc = select_signed_docs(&doc.body.id, &Some(doc.body.ver)) - .await - .unwrap(); + let res_doc = select_signed_docs(doc.id(), Some(doc.ver())).await.unwrap(); assert_eq!(doc, &res_doc); - let res_doc = select_signed_docs(&doc.body.id, &None).await.unwrap(); + let res_doc = select_signed_docs(doc.id(), None).await.unwrap(); assert_eq!(doc, &res_doc); let res_docs = - filtered_select_signed_docs(&DocsQueryFilter::DocId(doc.body.id), &QueryLimits::ALL) + filtered_select_signed_docs(&DocsQueryFilter::DocId(*doc.id()), &QueryLimits::ALL) .await .unwrap(); assert_eq!(res_docs.len(), 1); - assert_eq!(&doc.body, res_docs.first().unwrap()); + assert_eq!(doc.body(), res_docs.first().unwrap()); let res_docs = filtered_select_signed_docs( - &DocsQueryFilter::DocVer(doc.body.id, doc.body.ver), + &DocsQueryFilter::DocVer(*doc.id(), *doc.ver()), &QueryLimits::ALL, ) .await .unwrap(); assert_eq!(res_docs.len(), 1); - assert_eq!(&doc.body, res_docs.first().unwrap()); + assert_eq!(doc.body(), res_docs.first().unwrap()); let res_docs = filtered_select_signed_docs( - &DocsQueryFilter::Author(doc.body.author.clone()), + &DocsQueryFilter::Author(doc.author().clone()), &QueryLimits::ALL, ) .await .unwrap(); assert_eq!(res_docs.len(), 1); - assert_eq!(&doc.body, res_docs.first().unwrap()); + assert_eq!(doc.body(), res_docs.first().unwrap()); } let res_docs = @@ -99,7 +95,7 @@ async fn some_test() { .await .unwrap(); assert_eq!( - docs.iter().map(|doc| &doc.body).collect::>(), + docs.iter().map(FullSignedDoc::body).collect::>(), res_docs.iter().rev().collect::>() ); @@ -107,7 +103,7 @@ async fn some_test() { .await .unwrap(); assert_eq!( - docs.iter().map(|doc| &doc.body).collect::>(), + docs.iter().map(FullSignedDoc::body).collect::>(), res_docs.iter().rev().collect::>() ); } diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/types.rs b/catalyst-gateway/bin/src/db/event/signed_docs/types.rs new file mode 100644 index 00000000000..8770c3f4767 --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/signed_docs/types.rs @@ -0,0 +1,129 @@ +//! `FullSignedDoc` struct implementation. + +/// Full signed doc event db struct +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct FullSignedDoc { + /// Signed doc body + body: SignedDocBody, + /// `signed_doc` table `payload` field + payload: Option, + /// `signed_doc` table `raw` field + raw: Vec, +} + +/// Signed doc body event db struct +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct SignedDocBody { + /// `signed_doc` table `id` field + id: uuid::Uuid, + /// `signed_doc` table `ver` field + ver: uuid::Uuid, + /// `signed_doc` table `type` field + doc_type: uuid::Uuid, + /// `signed_doc` table `author` field + author: String, + /// `signed_doc` table `metadata` field + metadata: Option, +} + +impl FullSignedDoc { + /// Creates a `FullSignedDoc` instance. + #[allow(dead_code)] + pub(crate) fn new( + body: SignedDocBody, payload: Option, raw: Vec, + ) -> Self { + Self { body, payload, raw } + } + + /// Returns the document id. + pub(crate) fn id(&self) -> &uuid::Uuid { + &self.body.id + } + + /// Returns the document version. + pub(crate) fn ver(&self) -> &uuid::Uuid { + &self.body.ver + } + + /// Returns the document author. + #[allow(dead_code)] + pub(crate) fn author(&self) -> &String { + &self.body.author + } + + /// Returns the `SignedDocBody`. + #[allow(dead_code)] + pub(crate) fn body(&self) -> &SignedDocBody { + &self.body + } + + /// Returns all signed document fields in that order which requires + /// `INSERT_SIGNED_DOCS` query + pub(crate) fn db_fields(&self) -> [&(dyn tokio_postgres::types::ToSql + Sync); 7] { + [ + &self.body.id, + &self.body.ver, + &self.body.doc_type, + &self.body.author, + &self.body.metadata, + &self.payload, + &self.raw, + ] + } + + /// Creates a `FullSignedDoc` from postgresql row object. + pub(crate) fn from_row( + id: &uuid::Uuid, ver: Option<&uuid::Uuid>, row: &tokio_postgres::Row, + ) -> anyhow::Result { + let ver = if let Some(ver) = ver { + *ver + } else { + row.try_get("ver")? + }; + + Ok(FullSignedDoc { + body: SignedDocBody { + id: *id, + ver, + doc_type: row.try_get("type")?, + author: row.try_get("author")?, + metadata: row.try_get("metadata")?, + }, + payload: row.try_get("payload")?, + raw: row.try_get("raw")?, + }) + } +} + +impl SignedDocBody { + /// Creates a `SignedDocBody` instance. + #[allow(dead_code)] + pub(crate) fn new( + id: uuid::Uuid, ver: uuid::Uuid, doc_type: uuid::Uuid, author: String, + metadata: Option, + ) -> Self { + Self { + id, + ver, + doc_type, + author, + metadata, + } + } + + /// Creates a `SignedDocBody` from postgresql row object. + pub(crate) fn from_row(row: &tokio_postgres::Row) -> anyhow::Result { + let id = row.try_get("id")?; + let ver = row.try_get("ver")?; + let doc_type = row.try_get("type")?; + let author = row.try_get("author")?; + let metadata = row.try_get("metadata")?; + Ok(Self { + id, + ver, + doc_type, + author, + metadata, + }) + } +} From 6904aae745e16dd2a32231ea1ed7b800b11dfe01 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Thu, 19 Dec 2024 12:20:04 +0200 Subject: [PATCH 32/42] fix fmt --- catalyst-gateway/bin/src/db/event/signed_docs/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index b100311dd5d..a87cec8cb28 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -1,8 +1,8 @@ //! Signed docs queries -mod types; #[cfg(test)] mod tests; +mod types; pub(crate) use types::{FullSignedDoc, SignedDocBody}; From 0b15dd05293ae1d235449310c4274000ed4c9620 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Thu, 19 Dec 2024 12:58:34 +0200 Subject: [PATCH 33/42] wip --- .../db/event/signed_docs/full_signed_doc.rs | 83 +++++++++++ .../bin/src/db/event/signed_docs/mod.rs | 8 +- .../db/event/signed_docs/signed_doc_body.rs | 75 ++++++++++ .../bin/src/db/event/signed_docs/types.rs | 129 ------------------ 4 files changed, 163 insertions(+), 132 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs create mode 100644 catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs delete mode 100644 catalyst-gateway/bin/src/db/event/signed_docs/types.rs diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs b/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs new file mode 100644 index 00000000000..2f7d720486c --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs @@ -0,0 +1,83 @@ +//! `FullSignedDoc` struct implementation. + +use super::SignedDocBody; + +/// Full signed doc event db struct +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct FullSignedDoc { + /// Signed doc body + body: SignedDocBody, + /// `signed_doc` table `payload` field + payload: Option, + /// `signed_doc` table `raw` field + raw: Vec, +} + +impl FullSignedDoc { + /// Creates a `FullSignedDoc` instance. + #[allow(dead_code)] + pub(crate) fn new( + body: SignedDocBody, payload: Option, raw: Vec, + ) -> Self { + Self { body, payload, raw } + } + + /// Returns the document id. + pub(crate) fn id(&self) -> &uuid::Uuid { + self.body.id() + } + + /// Returns the document version. + pub(crate) fn ver(&self) -> &uuid::Uuid { + self.body.ver() + } + + /// Returns the document author. + #[allow(dead_code)] + pub(crate) fn author(&self) -> &String { + self.body.author() + } + + /// Returns the `SignedDocBody`. + #[allow(dead_code)] + pub(crate) fn body(&self) -> &SignedDocBody { + &self.body + } + + /// Returns all signed document fields for the event db queries + pub(crate) fn postgres_db_fields(&self) -> [&(dyn tokio_postgres::types::ToSql + Sync); 7] { + let body_fields = self.body.postgres_db_fields(); + [ + body_fields[0], + body_fields[1], + body_fields[2], + body_fields[3], + body_fields[4], + &self.payload, + &self.raw, + ] + } + + /// Creates a `FullSignedDoc` from postgresql row object. + pub(crate) fn from_row( + id: &uuid::Uuid, ver: Option<&uuid::Uuid>, row: &tokio_postgres::Row, + ) -> anyhow::Result { + let ver = if let Some(ver) = ver { + *ver + } else { + row.try_get("ver")? + }; + + Ok(FullSignedDoc { + body: SignedDocBody::new( + *id, + ver, + row.try_get("type")?, + row.try_get("author")?, + row.try_get("metadata")?, + ), + payload: row.try_get("payload")?, + raw: row.try_get("raw")?, + }) + } +} diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index a87cec8cb28..856a1ed5b92 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -1,10 +1,12 @@ //! Signed docs queries +mod full_signed_doc; +mod signed_doc_body; #[cfg(test)] mod tests; -mod types; -pub(crate) use types::{FullSignedDoc, SignedDocBody}; +pub(crate) use full_signed_doc::FullSignedDoc; +pub(crate) use signed_doc_body::SignedDocBody; use super::{common::query_limits::QueryLimits, EventDB, NotFoundError}; use crate::jinja::{get_template, JinjaTemplateSource}; @@ -49,7 +51,7 @@ pub(crate) async fn insert_signed_docs(doc: &FullSignedDoc) -> anyhow::Result<() Err(err) => return Err(err), } - EventDB::modify(INSERT_SIGNED_DOCS, &doc.db_fields()).await?; + EventDB::modify(INSERT_SIGNED_DOCS, &doc.postgres_db_fields()).await?; Ok(()) } diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs b/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs new file mode 100644 index 00000000000..69894216381 --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs @@ -0,0 +1,75 @@ +//! `SignedDocBody` struct implementation. + +/// Signed doc body event db struct +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct SignedDocBody { + /// `signed_doc` table `id` field + id: uuid::Uuid, + /// `signed_doc` table `ver` field + ver: uuid::Uuid, + /// `signed_doc` table `type` field + doc_type: uuid::Uuid, + /// `signed_doc` table `author` field + author: String, + /// `signed_doc` table `metadata` field + metadata: Option, +} + +impl SignedDocBody { + /// Returns the document id. + pub(crate) fn id(&self) -> &uuid::Uuid { + &self.id + } + + /// Returns the document version. + pub(crate) fn ver(&self) -> &uuid::Uuid { + &self.ver + } + + /// Returns the document author. + pub(crate) fn author(&self) -> &String { + &self.author + } + + /// Returns all signed document fields for the event db queries + pub(crate) fn postgres_db_fields(&self) -> [&(dyn tokio_postgres::types::ToSql + Sync); 5] { + [ + &self.id, + &self.ver, + &self.doc_type, + &self.author, + &self.metadata, + ] + } + + /// Creates a `SignedDocBody` instance. + #[allow(dead_code)] + pub(crate) fn new( + id: uuid::Uuid, ver: uuid::Uuid, doc_type: uuid::Uuid, author: String, + metadata: Option, + ) -> Self { + Self { + id, + ver, + doc_type, + author, + metadata, + } + } + + /// Creates a `SignedDocBody` from postgresql row object. + pub(crate) fn from_row(row: &tokio_postgres::Row) -> anyhow::Result { + let id = row.try_get("id")?; + let ver = row.try_get("ver")?; + let doc_type = row.try_get("type")?; + let author = row.try_get("author")?; + let metadata = row.try_get("metadata")?; + Ok(Self { + id, + ver, + doc_type, + author, + metadata, + }) + } +} diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/types.rs b/catalyst-gateway/bin/src/db/event/signed_docs/types.rs deleted file mode 100644 index 8770c3f4767..00000000000 --- a/catalyst-gateway/bin/src/db/event/signed_docs/types.rs +++ /dev/null @@ -1,129 +0,0 @@ -//! `FullSignedDoc` struct implementation. - -/// Full signed doc event db struct -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct FullSignedDoc { - /// Signed doc body - body: SignedDocBody, - /// `signed_doc` table `payload` field - payload: Option, - /// `signed_doc` table `raw` field - raw: Vec, -} - -/// Signed doc body event db struct -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct SignedDocBody { - /// `signed_doc` table `id` field - id: uuid::Uuid, - /// `signed_doc` table `ver` field - ver: uuid::Uuid, - /// `signed_doc` table `type` field - doc_type: uuid::Uuid, - /// `signed_doc` table `author` field - author: String, - /// `signed_doc` table `metadata` field - metadata: Option, -} - -impl FullSignedDoc { - /// Creates a `FullSignedDoc` instance. - #[allow(dead_code)] - pub(crate) fn new( - body: SignedDocBody, payload: Option, raw: Vec, - ) -> Self { - Self { body, payload, raw } - } - - /// Returns the document id. - pub(crate) fn id(&self) -> &uuid::Uuid { - &self.body.id - } - - /// Returns the document version. - pub(crate) fn ver(&self) -> &uuid::Uuid { - &self.body.ver - } - - /// Returns the document author. - #[allow(dead_code)] - pub(crate) fn author(&self) -> &String { - &self.body.author - } - - /// Returns the `SignedDocBody`. - #[allow(dead_code)] - pub(crate) fn body(&self) -> &SignedDocBody { - &self.body - } - - /// Returns all signed document fields in that order which requires - /// `INSERT_SIGNED_DOCS` query - pub(crate) fn db_fields(&self) -> [&(dyn tokio_postgres::types::ToSql + Sync); 7] { - [ - &self.body.id, - &self.body.ver, - &self.body.doc_type, - &self.body.author, - &self.body.metadata, - &self.payload, - &self.raw, - ] - } - - /// Creates a `FullSignedDoc` from postgresql row object. - pub(crate) fn from_row( - id: &uuid::Uuid, ver: Option<&uuid::Uuid>, row: &tokio_postgres::Row, - ) -> anyhow::Result { - let ver = if let Some(ver) = ver { - *ver - } else { - row.try_get("ver")? - }; - - Ok(FullSignedDoc { - body: SignedDocBody { - id: *id, - ver, - doc_type: row.try_get("type")?, - author: row.try_get("author")?, - metadata: row.try_get("metadata")?, - }, - payload: row.try_get("payload")?, - raw: row.try_get("raw")?, - }) - } -} - -impl SignedDocBody { - /// Creates a `SignedDocBody` instance. - #[allow(dead_code)] - pub(crate) fn new( - id: uuid::Uuid, ver: uuid::Uuid, doc_type: uuid::Uuid, author: String, - metadata: Option, - ) -> Self { - Self { - id, - ver, - doc_type, - author, - metadata, - } - } - - /// Creates a `SignedDocBody` from postgresql row object. - pub(crate) fn from_row(row: &tokio_postgres::Row) -> anyhow::Result { - let id = row.try_get("id")?; - let ver = row.try_get("ver")?; - let doc_type = row.try_get("type")?; - let author = row.try_get("author")?; - let metadata = row.try_get("metadata")?; - Ok(Self { - id, - ver, - doc_type, - author, - metadata, - }) - } -} From 38d160678bd3005c8510984eed10d824c050b86e Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Thu, 19 Dec 2024 14:56:49 +0200 Subject: [PATCH 34/42] move docs query filters into different mod --- .../bin/src/db/event/signed_docs/mod.rs | 32 ++----------------- .../src/db/event/signed_docs/query_filter.rs | 31 ++++++++++++++++++ 2 files changed, 33 insertions(+), 30 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/event/signed_docs/query_filter.rs diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index 856a1ed5b92..a8a958de0ca 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -1,11 +1,13 @@ //! Signed docs queries mod full_signed_doc; +mod query_filter; mod signed_doc_body; #[cfg(test)] mod tests; pub(crate) use full_signed_doc::FullSignedDoc; +pub(crate) use query_filter::DocsQueryFilter; pub(crate) use signed_doc_body::SignedDocBody; use super::{common::query_limits::QueryLimits, EventDB, NotFoundError}; @@ -78,36 +80,6 @@ pub(crate) async fn select_signed_docs( FullSignedDoc::from_row(id, ver, &row) } -/// A `select_signed_docs` query filtering argument. -#[allow(dead_code)] -pub(crate) enum DocsQueryFilter { - /// All entries - All, - /// Select docs with the specific `type` field - DocType(uuid::Uuid), - /// Select docs with the specific `id` field - DocId(uuid::Uuid), - /// Select docs with the specific `id` and `ver` field - DocVer(uuid::Uuid, uuid::Uuid), - /// Select docs with the specific `author` field - Author(String), -} - -impl DocsQueryFilter { - /// Returns a string with the corresponding query docs filter statement - pub(crate) fn query_stmt(&self) -> String { - match self { - Self::All => "TRUE".to_string(), - Self::DocType(doc_type) => format!("signed_docs.type = '{doc_type}'"), - Self::DocId(id) => format!("signed_docs.id = '{id}'"), - Self::DocVer(id, ver) => { - format!("signed_docs.id = '{id}' AND signed_docs.ver = '{ver}'") - }, - Self::Author(author) => format!("signed_docs.author = '{author}'"), - } - } -} - /// Make an filtered select query into the `event-db` by getting data from the /// `signed_docs` table. #[allow(dead_code)] diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/query_filter.rs b/catalyst-gateway/bin/src/db/event/signed_docs/query_filter.rs new file mode 100644 index 00000000000..bf292b55e66 --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/signed_docs/query_filter.rs @@ -0,0 +1,31 @@ +//! `DocsQueryFilter` struct implementation. + +/// A `select_signed_docs` query filtering argument. +#[allow(dead_code)] +pub(crate) enum DocsQueryFilter { + /// All entries + All, + /// Select docs with the specific `type` field + DocType(uuid::Uuid), + /// Select docs with the specific `id` field + DocId(uuid::Uuid), + /// Select docs with the specific `id` and `ver` field + DocVer(uuid::Uuid, uuid::Uuid), + /// Select docs with the specific `author` field + Author(String), +} + +impl DocsQueryFilter { + /// Returns a string with the corresponding query docs filter statement + pub(crate) fn query_stmt(&self) -> String { + match self { + Self::All => "TRUE".to_string(), + Self::DocType(doc_type) => format!("signed_docs.type = '{doc_type}'"), + Self::DocId(id) => format!("signed_docs.id = '{id}'"), + Self::DocVer(id, ver) => { + format!("signed_docs.id = '{id}' AND signed_docs.ver = '{ver}'") + }, + Self::Author(author) => format!("signed_docs.author = '{author}'"), + } + } +} From b787ae06fcbcb69153d9957559091bce6a74fff5 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Fri, 20 Dec 2024 11:33:52 +0200 Subject: [PATCH 35/42] refactor `filtered_select_signed_docs` function --- .../bin/src/db/event/signed_docs/mod.rs | 31 ++--------------- .../db/event/signed_docs/signed_doc_body.rs | 34 ++++++++++++++++++- .../bin/src/db/event/signed_docs/tests/mod.rs | 12 +++---- 3 files changed, 41 insertions(+), 36 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index a8a958de0ca..bb3629de7bc 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -8,9 +8,9 @@ mod tests; pub(crate) use full_signed_doc::FullSignedDoc; pub(crate) use query_filter::DocsQueryFilter; -pub(crate) use signed_doc_body::SignedDocBody; +pub(crate) use signed_doc_body::{SignedDocBody, FILTERED_SELECT_SIGNED_DOCS_TEMPLATE}; -use super::{common::query_limits::QueryLimits, EventDB, NotFoundError}; +use super::{EventDB, NotFoundError}; use crate::jinja::{get_template, JinjaTemplateSource}; /// Insert sql query @@ -22,12 +22,6 @@ pub(crate) const SELECT_SIGNED_DOCS_TEMPLATE: JinjaTemplateSource = JinjaTemplat source: include_str!("./sql/select_signed_documents.sql.jinja"), }; -/// Filtered select sql query jinja template -pub(crate) const FILTERED_SELECT_SIGNED_DOCS_TEMPLATE: JinjaTemplateSource = JinjaTemplateSource { - name: "filtered_select_signed_documents.jinja.template", - source: include_str!("./sql/filtered_select_signed_documents.sql.jinja"), -}; - /// Make an insert query into the `event-db` by adding data into the `signed_docs` table. /// /// * IF the record primary key (id,ver) does not exist, then add the new record. Return @@ -79,24 +73,3 @@ pub(crate) async fn select_signed_docs( FullSignedDoc::from_row(id, ver, &row) } - -/// Make an filtered select query into the `event-db` by getting data from the -/// `signed_docs` table. -#[allow(dead_code)] -pub(crate) async fn filtered_select_signed_docs( - conditions: &DocsQueryFilter, query_limits: &QueryLimits, -) -> anyhow::Result> { - let query_template = get_template(&FILTERED_SELECT_SIGNED_DOCS_TEMPLATE)?; - let query = query_template.render(serde_json::json!({ - "conditions": conditions.query_stmt(), - "query_limits": query_limits.query_stmt(), - }))?; - let rows = EventDB::query(&query, &[]).await?; - - let docs = rows - .iter() - .map(SignedDocBody::from_row) - .collect::>()?; - - Ok(docs) -} diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs b/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs index 69894216381..d7e10e50775 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs @@ -1,5 +1,17 @@ //! `SignedDocBody` struct implementation. +use super::DocsQueryFilter; +use crate::{ + db::event::{common::query_limits::QueryLimits, EventDB}, + jinja::{get_template, JinjaTemplateSource}, +}; + +/// Filtered select sql query jinja template +pub(crate) const FILTERED_SELECT_SIGNED_DOCS_TEMPLATE: JinjaTemplateSource = JinjaTemplateSource { + name: "filtered_select_signed_documents.jinja.template", + source: include_str!("./sql/filtered_select_signed_documents.sql.jinja"), +}; + /// Signed doc body event db struct #[derive(Debug, Clone, PartialEq)] pub(crate) struct SignedDocBody { @@ -57,8 +69,28 @@ impl SignedDocBody { } } + /// Loads a vector of `SignedDocBody` from the event db. + #[allow(dead_code)] + pub(crate) async fn load_from_db( + conditions: &DocsQueryFilter, query_limits: &QueryLimits, + ) -> anyhow::Result> { + let query_template = get_template(&FILTERED_SELECT_SIGNED_DOCS_TEMPLATE)?; + let query = query_template.render(serde_json::json!({ + "conditions": conditions.query_stmt(), + "query_limits": query_limits.query_stmt(), + }))?; + let rows = EventDB::query(&query, &[]).await?; + + let docs = rows + .iter() + .map(SignedDocBody::from_row) + .collect::>()?; + + Ok(docs) + } + /// Creates a `SignedDocBody` from postgresql row object. - pub(crate) fn from_row(row: &tokio_postgres::Row) -> anyhow::Result { + fn from_row(row: &tokio_postgres::Row) -> anyhow::Result { let id = row.try_get("id")?; let ver = row.try_get("ver")?; let doc_type = row.try_get("type")?; diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index e28516ba38a..3fce29fbd28 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -1,7 +1,7 @@ //! Integration tests of the `signed docs` queries use super::*; -use crate::db::event::establish_connection; +use crate::db::event::{common::query_limits::QueryLimits, establish_connection}; #[ignore = "An integration test which requires a running EventDB instance, disabled from `testunit` CI run"] #[tokio::test] @@ -65,13 +65,13 @@ async fn some_test() { assert_eq!(doc, &res_doc); let res_docs = - filtered_select_signed_docs(&DocsQueryFilter::DocId(*doc.id()), &QueryLimits::ALL) + SignedDocBody::load_from_db(&DocsQueryFilter::DocId(*doc.id()), &QueryLimits::ALL) .await .unwrap(); assert_eq!(res_docs.len(), 1); assert_eq!(doc.body(), res_docs.first().unwrap()); - let res_docs = filtered_select_signed_docs( + let res_docs = SignedDocBody::load_from_db( &DocsQueryFilter::DocVer(*doc.id(), *doc.ver()), &QueryLimits::ALL, ) @@ -80,7 +80,7 @@ async fn some_test() { assert_eq!(res_docs.len(), 1); assert_eq!(doc.body(), res_docs.first().unwrap()); - let res_docs = filtered_select_signed_docs( + let res_docs = SignedDocBody::load_from_db( &DocsQueryFilter::Author(doc.author().clone()), &QueryLimits::ALL, ) @@ -91,7 +91,7 @@ async fn some_test() { } let res_docs = - filtered_select_signed_docs(&DocsQueryFilter::DocType(doc_type), &QueryLimits::ALL) + SignedDocBody::load_from_db(&DocsQueryFilter::DocType(doc_type), &QueryLimits::ALL) .await .unwrap(); assert_eq!( @@ -99,7 +99,7 @@ async fn some_test() { res_docs.iter().rev().collect::>() ); - let res_docs = filtered_select_signed_docs(&DocsQueryFilter::All, &QueryLimits::ALL) + let res_docs = SignedDocBody::load_from_db(&DocsQueryFilter::All, &QueryLimits::ALL) .await .unwrap(); assert_eq!( From eb1441de10d6218c91d206f9c17c2acd2f17f4bf Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Fri, 20 Dec 2024 12:22:52 +0200 Subject: [PATCH 36/42] wip --- .../db/event/signed_docs/full_signed_doc.rs | 77 ++++++++++++++++++- .../bin/src/db/event/signed_docs/mod.rs | 67 +--------------- .../bin/src/db/event/signed_docs/tests/mod.rs | 12 +-- 3 files changed, 84 insertions(+), 72 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs b/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs index 2f7d720486c..2b0787c9f2c 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs @@ -1,6 +1,19 @@ //! `FullSignedDoc` struct implementation. use super::SignedDocBody; +use crate::{ + db::event::{EventDB, NotFoundError}, + jinja::{get_template, JinjaTemplateSource}, +}; + +/// Insert sql query +const INSERT_SIGNED_DOCS: &str = include_str!("./sql/insert_signed_documents.sql"); + +/// Select sql query jinja template +pub(crate) const SELECT_SIGNED_DOCS_TEMPLATE: JinjaTemplateSource = JinjaTemplateSource { + name: "select_signed_documents.jinja.template", + source: include_str!("./sql/select_signed_documents.sql.jinja"), +}; /// Full signed doc event db struct #[derive(Debug, Clone, PartialEq)] @@ -44,8 +57,68 @@ impl FullSignedDoc { &self.body } + /// Uploads a `FullSignedDoc` to the event db. + /// + /// Make an insert query into the `event-db` by adding data into the `signed_docs` + /// table. + /// + /// * IF the record primary key (id,ver) does not exist, then add the new record. + /// Return success. + /// * IF the record does exist, but all values are the same as stored, return Success. + /// * Otherwise return an error. (Can not over-write an existing record with new + /// data). + /// + /// # Arguments: + /// - `id` is a UUID v7 + /// - `ver` is a UUID v7 + /// - `doc_type` is a UUID v4 + #[allow(dead_code)] + pub(crate) async fn upload_to_db(&self) -> anyhow::Result<()> { + match Self::load_from_db(self.id(), Some(self.ver())).await { + Ok(res_doc) => { + anyhow::ensure!( + &res_doc == self, + "Document with the same `id` and `ver` already exists" + ); + return Ok(()); + }, + Err(err) if err.is::() => {}, + Err(err) => return Err(err), + } + + EventDB::modify(INSERT_SIGNED_DOCS, &self.postgres_db_fields()).await?; + Ok(()) + } + + /// Loads a `FullSignedDoc` from the event db. + /// + /// Make a select query into the `event-db` by getting data from the `signed_docs` + /// table. + /// + /// * This returns a single document. All data from the document is returned, + /// including the `payload` and `raw` fields. + /// * `ver` should be able to be optional, in which case get the latest ver of the + /// given `id`. + /// + /// # Arguments: + /// - `id` is a UUID v7 + /// - `ver` is a UUID v7 + #[allow(dead_code)] + pub(crate) async fn load_from_db( + id: &uuid::Uuid, ver: Option<&uuid::Uuid>, + ) -> anyhow::Result { + let query_template = get_template(&SELECT_SIGNED_DOCS_TEMPLATE)?; + let query = query_template.render(serde_json::json!({ + "id": id, + "ver": ver, + }))?; + let row = EventDB::query_one(&query, &[]).await?; + + Self::from_row(id, ver, &row) + } + /// Returns all signed document fields for the event db queries - pub(crate) fn postgres_db_fields(&self) -> [&(dyn tokio_postgres::types::ToSql + Sync); 7] { + fn postgres_db_fields(&self) -> [&(dyn tokio_postgres::types::ToSql + Sync); 7] { let body_fields = self.body.postgres_db_fields(); [ body_fields[0], @@ -59,7 +132,7 @@ impl FullSignedDoc { } /// Creates a `FullSignedDoc` from postgresql row object. - pub(crate) fn from_row( + fn from_row( id: &uuid::Uuid, ver: Option<&uuid::Uuid>, row: &tokio_postgres::Row, ) -> anyhow::Result { let ver = if let Some(ver) = ver { diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs index bb3629de7bc..893adfeddbc 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/mod.rs @@ -6,70 +6,7 @@ mod signed_doc_body; #[cfg(test)] mod tests; -pub(crate) use full_signed_doc::FullSignedDoc; +#[allow(unused_imports)] +pub(crate) use full_signed_doc::{FullSignedDoc, SELECT_SIGNED_DOCS_TEMPLATE}; pub(crate) use query_filter::DocsQueryFilter; pub(crate) use signed_doc_body::{SignedDocBody, FILTERED_SELECT_SIGNED_DOCS_TEMPLATE}; - -use super::{EventDB, NotFoundError}; -use crate::jinja::{get_template, JinjaTemplateSource}; - -/// Insert sql query -const INSERT_SIGNED_DOCS: &str = include_str!("./sql/insert_signed_documents.sql"); - -/// Select sql query jinja template -pub(crate) const SELECT_SIGNED_DOCS_TEMPLATE: JinjaTemplateSource = JinjaTemplateSource { - name: "select_signed_documents.jinja.template", - source: include_str!("./sql/select_signed_documents.sql.jinja"), -}; - -/// Make an insert query into the `event-db` by adding data into the `signed_docs` table. -/// -/// * IF the record primary key (id,ver) does not exist, then add the new record. Return -/// success. -/// * IF the record does exist, but all values are the same as stored, return Success. -/// * Otherwise return an error. (Can not over-write an existing record with new data). -/// -/// # Arguments: -/// - `id` is a UUID v7 -/// - `ver` is a UUID v7 -/// - `doc_type` is a UUID v4 -#[allow(dead_code)] -pub(crate) async fn insert_signed_docs(doc: &FullSignedDoc) -> anyhow::Result<()> { - match select_signed_docs(doc.id(), Some(doc.ver())).await { - Ok(res_doc) => { - anyhow::ensure!( - &res_doc == doc, - "Document with the same `id` and `ver` already exists" - ); - return Ok(()); - }, - Err(err) if err.is::() => {}, - Err(err) => return Err(err), - } - - EventDB::modify(INSERT_SIGNED_DOCS, &doc.postgres_db_fields()).await?; - Ok(()) -} - -/// Make a select query into the `event-db` by getting data from the `signed_docs` table. -/// -/// * This returns a single document. All data from the document is returned, including -/// the `payload` and `raw` fields. -/// * `ver` should be able to be optional, in which case get the latest ver of the given -/// `id`. -/// -/// # Arguments: -/// - `id` is a UUID v7 -/// - `ver` is a UUID v7 -pub(crate) async fn select_signed_docs( - id: &uuid::Uuid, ver: Option<&uuid::Uuid>, -) -> anyhow::Result { - let query_template = get_template(&SELECT_SIGNED_DOCS_TEMPLATE)?; - let query = query_template.render(serde_json::json!({ - "id": id, - "ver": ver, - }))?; - let row = EventDB::query_one(&query, &[]).await?; - - FullSignedDoc::from_row(id, ver, &row) -} diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index 3fce29fbd28..ea5625a9820 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -47,21 +47,23 @@ async fn some_test() { ]; for doc in &docs { - insert_signed_docs(doc).await.unwrap(); + doc.upload_to_db().await.unwrap(); // try to insert the same data again - insert_signed_docs(doc).await.unwrap(); + doc.upload_to_db().await.unwrap(); // try another doc with the same `id` and `ver` and with different other fields let another_doc = FullSignedDoc::new( SignedDocBody::new(*doc.id(), *doc.ver(), doc_type, "Neil".to_string(), None), None, vec![], ); - assert!(insert_signed_docs(&another_doc).await.is_err()); + assert!(another_doc.upload_to_db().await.is_err()); - let res_doc = select_signed_docs(doc.id(), Some(doc.ver())).await.unwrap(); + let res_doc = FullSignedDoc::load_from_db(doc.id(), Some(doc.ver())) + .await + .unwrap(); assert_eq!(doc, &res_doc); - let res_doc = select_signed_docs(doc.id(), None).await.unwrap(); + let res_doc = FullSignedDoc::load_from_db(doc.id(), None).await.unwrap(); assert_eq!(doc, &res_doc); let res_docs = From 8ce23c7b7691df54a66e4021e1a0b8d17c46437b Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Thu, 2 Jan 2025 10:22:39 +0200 Subject: [PATCH 37/42] replace query_stmt with trait Display impl --- .../bin/src/db/event/common/query_limits.rs | 25 +++++++++++-------- .../src/db/event/signed_docs/query_filter.rs | 17 +++++++------ .../db/event/signed_docs/signed_doc_body.rs | 4 +-- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/common/query_limits.rs b/catalyst-gateway/bin/src/db/event/common/query_limits.rs index 5d450894714..c6b367a0d33 100644 --- a/catalyst-gateway/bin/src/db/event/common/query_limits.rs +++ b/catalyst-gateway/bin/src/db/event/common/query_limits.rs @@ -2,6 +2,8 @@ #![allow(dead_code)] +use std::fmt::Display; + use crate::service::common::types::generic::query::pagination::{Limit, Page}; /// A query limits struct. @@ -17,6 +19,18 @@ enum QueryLimitsInner { LimitAndOffset(u64, u64), } +impl Display for QueryLimits { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self.0 { + QueryLimitsInner::All => write!(f, ""), + QueryLimitsInner::Limit(limit) => write!(f, "LIMIT {limit}"), + QueryLimitsInner::LimitAndOffset(limit, offset) => { + write!(f, "LIMIT {limit} OFFSET {offset}") + }, + } + } +} + impl QueryLimits { /// Create a `QueryLimits` object without the any limits. pub(crate) const ALL: QueryLimits = Self(QueryLimitsInner::All); @@ -43,15 +57,4 @@ impl QueryLimits { }, } } - - /// Returns a string with the corresponding query limit statement - pub(crate) fn query_stmt(&self) -> String { - match self.0 { - QueryLimitsInner::All => String::new(), - QueryLimitsInner::Limit(limit) => format!("LIMIT {limit}"), - QueryLimitsInner::LimitAndOffset(limit, offset) => { - format!("LIMIT {limit} OFFSET {offset}") - }, - } - } } diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/query_filter.rs b/catalyst-gateway/bin/src/db/event/signed_docs/query_filter.rs index bf292b55e66..9948fc92b48 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/query_filter.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/query_filter.rs @@ -1,5 +1,7 @@ //! `DocsQueryFilter` struct implementation. +use std::fmt::Display; + /// A `select_signed_docs` query filtering argument. #[allow(dead_code)] pub(crate) enum DocsQueryFilter { @@ -15,17 +17,16 @@ pub(crate) enum DocsQueryFilter { Author(String), } -impl DocsQueryFilter { - /// Returns a string with the corresponding query docs filter statement - pub(crate) fn query_stmt(&self) -> String { +impl Display for DocsQueryFilter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::All => "TRUE".to_string(), - Self::DocType(doc_type) => format!("signed_docs.type = '{doc_type}'"), - Self::DocId(id) => format!("signed_docs.id = '{id}'"), + Self::All => write!(f, "TRUE"), + Self::DocType(doc_type) => write!(f, "signed_docs.type = '{doc_type}'"), + Self::DocId(id) => write!(f, "signed_docs.id = '{id}'"), Self::DocVer(id, ver) => { - format!("signed_docs.id = '{id}' AND signed_docs.ver = '{ver}'") + write!(f, "signed_docs.id = '{id}' AND signed_docs.ver = '{ver}'") }, - Self::Author(author) => format!("signed_docs.author = '{author}'"), + Self::Author(author) => write!(f, "signed_docs.author = '{author}'"), } } } diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs b/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs index d7e10e50775..9eb9c41c5be 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs @@ -76,8 +76,8 @@ impl SignedDocBody { ) -> anyhow::Result> { let query_template = get_template(&FILTERED_SELECT_SIGNED_DOCS_TEMPLATE)?; let query = query_template.render(serde_json::json!({ - "conditions": conditions.query_stmt(), - "query_limits": query_limits.query_stmt(), + "conditions": conditions.to_string(), + "query_limits": query_limits.to_string(), }))?; let rows = EventDB::query(&query, &[]).await?; From 8c076e7f2e3e3bbec87d40565c84db4f84d407a1 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Thu, 2 Jan 2025 10:33:38 +0200 Subject: [PATCH 38/42] rename methods --- .../db/event/signed_docs/full_signed_doc.rs | 6 ++--- .../db/event/signed_docs/signed_doc_body.rs | 2 +- .../bin/src/db/event/signed_docs/tests/mod.rs | 25 +++++++++---------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs b/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs index 2b0787c9f2c..bf4b9e10529 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/full_signed_doc.rs @@ -73,8 +73,8 @@ impl FullSignedDoc { /// - `ver` is a UUID v7 /// - `doc_type` is a UUID v4 #[allow(dead_code)] - pub(crate) async fn upload_to_db(&self) -> anyhow::Result<()> { - match Self::load_from_db(self.id(), Some(self.ver())).await { + pub(crate) async fn store(&self) -> anyhow::Result<()> { + match Self::retrieve(self.id(), Some(self.ver())).await { Ok(res_doc) => { anyhow::ensure!( &res_doc == self, @@ -104,7 +104,7 @@ impl FullSignedDoc { /// - `id` is a UUID v7 /// - `ver` is a UUID v7 #[allow(dead_code)] - pub(crate) async fn load_from_db( + pub(crate) async fn retrieve( id: &uuid::Uuid, ver: Option<&uuid::Uuid>, ) -> anyhow::Result { let query_template = get_template(&SELECT_SIGNED_DOCS_TEMPLATE)?; diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs b/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs index 9eb9c41c5be..84c5a836010 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs @@ -71,7 +71,7 @@ impl SignedDocBody { /// Loads a vector of `SignedDocBody` from the event db. #[allow(dead_code)] - pub(crate) async fn load_from_db( + pub(crate) async fn retrieve( conditions: &DocsQueryFilter, query_limits: &QueryLimits, ) -> anyhow::Result> { let query_template = get_template(&FILTERED_SELECT_SIGNED_DOCS_TEMPLATE)?; diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index ea5625a9820..015cd36524c 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -47,33 +47,33 @@ async fn some_test() { ]; for doc in &docs { - doc.upload_to_db().await.unwrap(); + doc.store().await.unwrap(); // try to insert the same data again - doc.upload_to_db().await.unwrap(); + doc.store().await.unwrap(); // try another doc with the same `id` and `ver` and with different other fields let another_doc = FullSignedDoc::new( SignedDocBody::new(*doc.id(), *doc.ver(), doc_type, "Neil".to_string(), None), None, vec![], ); - assert!(another_doc.upload_to_db().await.is_err()); + assert!(another_doc.store().await.is_err()); - let res_doc = FullSignedDoc::load_from_db(doc.id(), Some(doc.ver())) + let res_doc = FullSignedDoc::retrieve(doc.id(), Some(doc.ver())) .await .unwrap(); assert_eq!(doc, &res_doc); - let res_doc = FullSignedDoc::load_from_db(doc.id(), None).await.unwrap(); + let res_doc = FullSignedDoc::retrieve(doc.id(), None).await.unwrap(); assert_eq!(doc, &res_doc); let res_docs = - SignedDocBody::load_from_db(&DocsQueryFilter::DocId(*doc.id()), &QueryLimits::ALL) + SignedDocBody::retrieve(&DocsQueryFilter::DocId(*doc.id()), &QueryLimits::ALL) .await .unwrap(); assert_eq!(res_docs.len(), 1); assert_eq!(doc.body(), res_docs.first().unwrap()); - let res_docs = SignedDocBody::load_from_db( + let res_docs = SignedDocBody::retrieve( &DocsQueryFilter::DocVer(*doc.id(), *doc.ver()), &QueryLimits::ALL, ) @@ -82,7 +82,7 @@ async fn some_test() { assert_eq!(res_docs.len(), 1); assert_eq!(doc.body(), res_docs.first().unwrap()); - let res_docs = SignedDocBody::load_from_db( + let res_docs = SignedDocBody::retrieve( &DocsQueryFilter::Author(doc.author().clone()), &QueryLimits::ALL, ) @@ -92,16 +92,15 @@ async fn some_test() { assert_eq!(doc.body(), res_docs.first().unwrap()); } - let res_docs = - SignedDocBody::load_from_db(&DocsQueryFilter::DocType(doc_type), &QueryLimits::ALL) - .await - .unwrap(); + let res_docs = SignedDocBody::retrieve(&DocsQueryFilter::DocType(doc_type), &QueryLimits::ALL) + .await + .unwrap(); assert_eq!( docs.iter().map(FullSignedDoc::body).collect::>(), res_docs.iter().rev().collect::>() ); - let res_docs = SignedDocBody::load_from_db(&DocsQueryFilter::All, &QueryLimits::ALL) + let res_docs = SignedDocBody::retrieve(&DocsQueryFilter::All, &QueryLimits::ALL) .await .unwrap(); assert_eq!( From d97601e966080397e41bd1253d6a3be64859efee Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Thu, 2 Jan 2025 15:01:47 +0200 Subject: [PATCH 39/42] add query_stream fn --- catalyst-gateway/bin/src/db/event/mod.rs | 39 +++++++++-- .../db/event/signed_docs/signed_doc_body.rs | 15 ++-- .../bin/src/db/event/signed_docs/tests/mod.rs | 68 +++++++++++-------- 3 files changed, 79 insertions(+), 43 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/mod.rs b/catalyst-gateway/bin/src/db/event/mod.rs index 0d737876505..0402b79f0c6 100644 --- a/catalyst-gateway/bin/src/db/event/mod.rs +++ b/catalyst-gateway/bin/src/db/event/mod.rs @@ -10,6 +10,7 @@ use std::{ use bb8::Pool; use bb8_postgres::PostgresConnectionManager; use error::NotFoundError; +use futures::{Stream, TryStreamExt}; use tokio_postgres::{types::ToSql, NoTls, Row}; use tracing::{debug, debug_span, error, Instrument}; @@ -73,11 +74,11 @@ impl EventDB { /// /// # Returns /// - /// `Result, anyhow::Error>` + /// `anyhow::Result>` #[must_use = "ONLY use this function for SELECT type operations which return row data, otherwise use `modify()`"] pub(crate) async fn query( stmt: &str, params: &[&(dyn ToSql + Sync)], - ) -> Result, anyhow::Error> { + ) -> anyhow::Result> { if Self::is_deep_query_enabled() { Self::explain_analyze_rollback(stmt, params).await?; } @@ -87,6 +88,33 @@ impl EventDB { Ok(rows) } + /// Query the database and return a async stream of rows. + /// + /// If deep query inspection is enabled, this will log the query plan inside a + /// rolled-back transaction, before running the query. + /// + /// # Arguments + /// + /// * `stmt` - `&str` SQL statement. + /// * `params` - `&[&(dyn ToSql + Sync)]` SQL parameters. + /// + /// # Returns + /// + /// `anyhow::Result>>` + #[must_use = "ONLY use this function for SELECT type operations which return row data, otherwise use `modify()`"] + pub(crate) async fn query_stream( + stmt: &str, params: &[&(dyn ToSql + Sync)], + ) -> anyhow::Result>> { + if Self::is_deep_query_enabled() { + Self::explain_analyze_rollback(stmt, params).await?; + } + let pool = EVENT_DB_POOL.get().ok_or(Error::DbPoolUninitialized)?; + let conn = pool.get().await?; + let params = params.iter().map(|s| -> &dyn ToSql { *s }); + let rows = conn.query_raw(stmt, params).await?; + Ok(rows.map_err(Into::into)) + } + /// Query the database for a single row. /// /// # Arguments @@ -100,16 +128,13 @@ impl EventDB { #[must_use = "ONLY use this function for SELECT type operations which return row data, otherwise use `modify()`"] pub(crate) async fn query_one( stmt: &str, params: &[&(dyn ToSql + Sync)], - ) -> Result { + ) -> anyhow::Result { if Self::is_deep_query_enabled() { Self::explain_analyze_rollback(stmt, params).await?; } let pool = EVENT_DB_POOL.get().ok_or(Error::DbPoolUninitialized)?; let conn = pool.get().await?; - let row = conn - .query_opt(stmt, params) - .await? - .ok_or::(NotFoundError.into())?; + let row = conn.query_opt(stmt, params).await?.ok_or(NotFoundError)?; Ok(row) } diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs b/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs index 84c5a836010..df85b1c40a0 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/signed_doc_body.rs @@ -1,5 +1,7 @@ //! `SignedDocBody` struct implementation. +use futures::{Stream, StreamExt}; + use super::DocsQueryFilter; use crate::{ db::event::{common::query_limits::QueryLimits, EventDB}, @@ -69,23 +71,18 @@ impl SignedDocBody { } } - /// Loads a vector of `SignedDocBody` from the event db. + /// Loads a async stream of `SignedDocBody` from the event db. #[allow(dead_code)] pub(crate) async fn retrieve( conditions: &DocsQueryFilter, query_limits: &QueryLimits, - ) -> anyhow::Result> { + ) -> anyhow::Result>> { let query_template = get_template(&FILTERED_SELECT_SIGNED_DOCS_TEMPLATE)?; let query = query_template.render(serde_json::json!({ "conditions": conditions.to_string(), "query_limits": query_limits.to_string(), }))?; - let rows = EventDB::query(&query, &[]).await?; - - let docs = rows - .iter() - .map(SignedDocBody::from_row) - .collect::>()?; - + let rows = EventDB::query_stream(&query, &[]).await?; + let docs = rows.map(|res_row| res_row.and_then(|row| SignedDocBody::from_row(&row))); Ok(docs) } diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index 015cd36524c..14f79eda251 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -1,5 +1,9 @@ //! Integration tests of the `signed docs` queries +use std::pin::pin; + +use futures::TryStreamExt; + use super::*; use crate::db::event::{common::query_limits::QueryLimits, establish_connection}; @@ -66,45 +70,55 @@ async fn some_test() { let res_doc = FullSignedDoc::retrieve(doc.id(), None).await.unwrap(); assert_eq!(doc, &res_doc); - let res_docs = - SignedDocBody::retrieve(&DocsQueryFilter::DocId(*doc.id()), &QueryLimits::ALL) - .await - .unwrap(); - assert_eq!(res_docs.len(), 1); - assert_eq!(doc.body(), res_docs.first().unwrap()); + let mut res_docs = pin!(SignedDocBody::retrieve( + &DocsQueryFilter::DocId(*doc.id()), + &QueryLimits::ALL + ) + .await + .unwrap()); + let res_doc = res_docs.try_next().await.unwrap().unwrap(); + assert_eq!(doc.body(), &res_doc); + assert!(res_docs.try_next().await.unwrap().is_none()); - let res_docs = SignedDocBody::retrieve( + let mut res_docs = pin!(SignedDocBody::retrieve( &DocsQueryFilter::DocVer(*doc.id(), *doc.ver()), &QueryLimits::ALL, ) .await - .unwrap(); - assert_eq!(res_docs.len(), 1); - assert_eq!(doc.body(), res_docs.first().unwrap()); + .unwrap()); + let res_doc = res_docs.try_next().await.unwrap().unwrap(); + assert_eq!(doc.body(), &res_doc); + assert!(res_docs.try_next().await.unwrap().is_none()); - let res_docs = SignedDocBody::retrieve( + let mut res_docs = pin!(SignedDocBody::retrieve( &DocsQueryFilter::Author(doc.author().clone()), &QueryLimits::ALL, ) .await - .unwrap(); - assert_eq!(res_docs.len(), 1); - assert_eq!(doc.body(), res_docs.first().unwrap()); + .unwrap()); + let res_doc = res_docs.try_next().await.unwrap().unwrap(); + assert_eq!(doc.body(), &res_doc); + assert!(res_docs.try_next().await.unwrap().is_none()); } - let res_docs = SignedDocBody::retrieve(&DocsQueryFilter::DocType(doc_type), &QueryLimits::ALL) - .await - .unwrap(); - assert_eq!( - docs.iter().map(FullSignedDoc::body).collect::>(), - res_docs.iter().rev().collect::>() - ); + let mut res_docs = pin!(SignedDocBody::retrieve( + &DocsQueryFilter::DocType(doc_type), + &QueryLimits::ALL + ) + .await + .unwrap()); + for exp_doc in &docs { + let res_doc = res_docs.try_next().await.unwrap().unwrap(); + assert_eq!(exp_doc.body(), &res_doc); + } - let res_docs = SignedDocBody::retrieve(&DocsQueryFilter::All, &QueryLimits::ALL) - .await - .unwrap(); - assert_eq!( - docs.iter().map(FullSignedDoc::body).collect::>(), - res_docs.iter().rev().collect::>() + let mut res_docs = pin!( + SignedDocBody::retrieve(&DocsQueryFilter::All, &QueryLimits::ALL) + .await + .unwrap() ); + for exp_doc in &docs { + let res_doc = res_docs.try_next().await.unwrap().unwrap(); + assert_eq!(exp_doc.body(), &res_doc); + } } From 74e2962f3a288d749ad8ef7069ec5a0b2569a1d0 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Thu, 2 Jan 2025 15:32:30 +0200 Subject: [PATCH 40/42] fix clippy --- catalyst-gateway/bin/src/db/event/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/mod.rs b/catalyst-gateway/bin/src/db/event/mod.rs index 0402b79f0c6..e70eb121924 100644 --- a/catalyst-gateway/bin/src/db/event/mod.rs +++ b/catalyst-gateway/bin/src/db/event/mod.rs @@ -110,8 +110,7 @@ impl EventDB { } let pool = EVENT_DB_POOL.get().ok_or(Error::DbPoolUninitialized)?; let conn = pool.get().await?; - let params = params.iter().map(|s| -> &dyn ToSql { *s }); - let rows = conn.query_raw(stmt, params).await?; + let rows = conn.query_raw(stmt, params.iter().copied()).await?; Ok(rows.map_err(Into::into)) } From 1074e01fdf5b3dbc1a0aa7c926f00310fdd513a9 Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Thu, 2 Jan 2025 16:22:09 +0200 Subject: [PATCH 41/42] fix test --- catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index 14f79eda251..a8067c3712e 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -9,7 +9,7 @@ use crate::db::event::{common::query_limits::QueryLimits, establish_connection}; #[ignore = "An integration test which requires a running EventDB instance, disabled from `testunit` CI run"] #[tokio::test] -async fn some_test() { +async fn queries_test() { establish_connection(); let doc_type = uuid::Uuid::new_v4(); @@ -107,7 +107,7 @@ async fn some_test() { ) .await .unwrap()); - for exp_doc in &docs { + for exp_doc in docs.iter().rev() { let res_doc = res_docs.try_next().await.unwrap().unwrap(); assert_eq!(exp_doc.body(), &res_doc); } @@ -117,7 +117,7 @@ async fn some_test() { .await .unwrap() ); - for exp_doc in &docs { + for exp_doc in docs.iter().rev() { let res_doc = res_docs.try_next().await.unwrap().unwrap(); assert_eq!(exp_doc.body(), &res_doc); } From 8af1cef8c037e8a97339f88f29a7dd9b2087314d Mon Sep 17 00:00:00 2001 From: Mr-Leshiy Date: Thu, 2 Jan 2025 17:25:30 +0200 Subject: [PATCH 42/42] get rid of usage of `pin!` --- catalyst-gateway/bin/src/db/event/mod.rs | 6 +-- .../bin/src/db/event/signed_docs/tests/mod.rs | 38 ++++++++----------- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/mod.rs b/catalyst-gateway/bin/src/db/event/mod.rs index e70eb121924..bed5b0d160c 100644 --- a/catalyst-gateway/bin/src/db/event/mod.rs +++ b/catalyst-gateway/bin/src/db/event/mod.rs @@ -10,7 +10,7 @@ use std::{ use bb8::Pool; use bb8_postgres::PostgresConnectionManager; use error::NotFoundError; -use futures::{Stream, TryStreamExt}; +use futures::{Stream, StreamExt, TryStreamExt}; use tokio_postgres::{types::ToSql, NoTls, Row}; use tracing::{debug, debug_span, error, Instrument}; @@ -100,7 +100,7 @@ impl EventDB { /// /// # Returns /// - /// `anyhow::Result>>` + /// `anyhow::Result>>` #[must_use = "ONLY use this function for SELECT type operations which return row data, otherwise use `modify()`"] pub(crate) async fn query_stream( stmt: &str, params: &[&(dyn ToSql + Sync)], @@ -111,7 +111,7 @@ impl EventDB { let pool = EVENT_DB_POOL.get().ok_or(Error::DbPoolUninitialized)?; let conn = pool.get().await?; let rows = conn.query_raw(stmt, params.iter().copied()).await?; - Ok(rows.map_err(Into::into)) + Ok(rows.map_err(Into::into).boxed()) } /// Query the database for a single row. diff --git a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs index a8067c3712e..65fd2c01d3f 100644 --- a/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs +++ b/catalyst-gateway/bin/src/db/event/signed_docs/tests/mod.rs @@ -1,7 +1,5 @@ //! Integration tests of the `signed docs` queries -use std::pin::pin; - use futures::TryStreamExt; use super::*; @@ -70,53 +68,47 @@ async fn queries_test() { let res_doc = FullSignedDoc::retrieve(doc.id(), None).await.unwrap(); assert_eq!(doc, &res_doc); - let mut res_docs = pin!(SignedDocBody::retrieve( - &DocsQueryFilter::DocId(*doc.id()), - &QueryLimits::ALL - ) - .await - .unwrap()); + let mut res_docs = + SignedDocBody::retrieve(&DocsQueryFilter::DocId(*doc.id()), &QueryLimits::ALL) + .await + .unwrap(); let res_doc = res_docs.try_next().await.unwrap().unwrap(); assert_eq!(doc.body(), &res_doc); assert!(res_docs.try_next().await.unwrap().is_none()); - let mut res_docs = pin!(SignedDocBody::retrieve( + let mut res_docs = SignedDocBody::retrieve( &DocsQueryFilter::DocVer(*doc.id(), *doc.ver()), &QueryLimits::ALL, ) .await - .unwrap()); + .unwrap(); let res_doc = res_docs.try_next().await.unwrap().unwrap(); assert_eq!(doc.body(), &res_doc); assert!(res_docs.try_next().await.unwrap().is_none()); - let mut res_docs = pin!(SignedDocBody::retrieve( + let mut res_docs = SignedDocBody::retrieve( &DocsQueryFilter::Author(doc.author().clone()), &QueryLimits::ALL, ) .await - .unwrap()); + .unwrap(); let res_doc = res_docs.try_next().await.unwrap().unwrap(); assert_eq!(doc.body(), &res_doc); assert!(res_docs.try_next().await.unwrap().is_none()); } - let mut res_docs = pin!(SignedDocBody::retrieve( - &DocsQueryFilter::DocType(doc_type), - &QueryLimits::ALL - ) - .await - .unwrap()); + let mut res_docs = + SignedDocBody::retrieve(&DocsQueryFilter::DocType(doc_type), &QueryLimits::ALL) + .await + .unwrap(); for exp_doc in docs.iter().rev() { let res_doc = res_docs.try_next().await.unwrap().unwrap(); assert_eq!(exp_doc.body(), &res_doc); } - let mut res_docs = pin!( - SignedDocBody::retrieve(&DocsQueryFilter::All, &QueryLimits::ALL) - .await - .unwrap() - ); + let mut res_docs = SignedDocBody::retrieve(&DocsQueryFilter::All, &QueryLimits::ALL) + .await + .unwrap(); for exp_doc in docs.iter().rev() { let res_doc = res_docs.try_next().await.unwrap().unwrap(); assert_eq!(exp_doc.body(), &res_doc);