From 30655de86ec7a03406c513191e1662803e3217d9 Mon Sep 17 00:00:00 2001 From: bkioshn Date: Wed, 9 Oct 2024 13:14:46 +0700 Subject: [PATCH 01/37] feat: add config endpoint Signed-off-by: bkioshn --- .../src/db/event/config/default/frontend.json | 1 + .../db/event/config/jsonschema/frontend.json | 4 + .../bin/src/db/event/config/key.rs | 90 ++++++++++++++++ .../bin/src/db/event/config/mod.rs | 48 +++++++++ .../bin/src/db/event/config/sql/get.sql | 3 + .../bin/src/db/event/config/sql/insert.sql | 4 + catalyst-gateway/bin/src/db/event/mod.rs | 1 + .../bin/src/service/api/config/mod.rs | 100 ++++++++++++++++++ catalyst-gateway/bin/src/service/api/mod.rs | 5 +- .../bin/src/service/common/tags.rs | 2 + 10 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 catalyst-gateway/bin/src/db/event/config/default/frontend.json create mode 100644 catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json create mode 100644 catalyst-gateway/bin/src/db/event/config/key.rs create mode 100644 catalyst-gateway/bin/src/db/event/config/mod.rs create mode 100644 catalyst-gateway/bin/src/db/event/config/sql/get.sql create mode 100644 catalyst-gateway/bin/src/db/event/config/sql/insert.sql create mode 100644 catalyst-gateway/bin/src/service/api/config/mod.rs diff --git a/catalyst-gateway/bin/src/db/event/config/default/frontend.json b/catalyst-gateway/bin/src/db/event/config/default/frontend.json new file mode 100644 index 00000000000..9e26dfeeb6e --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/config/default/frontend.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json b/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json new file mode 100644 index 00000000000..a555810c341 --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object" +} diff --git a/catalyst-gateway/bin/src/db/event/config/key.rs b/catalyst-gateway/bin/src/db/event/config/key.rs new file mode 100644 index 00000000000..6cc884a662b --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/config/key.rs @@ -0,0 +1,90 @@ +//! Configuration Key + +use std::net::IpAddr; + +use serde_json::Value; + +/// Configuration key +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum ConfigKey { + /// Frontend general configuration + Frontend, + /// Frontend configuration for a specific IP address + FrontendForIp(IpAddr), +} + +/// Frontend JSON schema +const FRONTEND_JSON_SCHEMA: &str = include_str!("jsonschema/frontend.json"); +/// Default frontend configuration +const FRONTEND_DEFAULT: &str = include_str!("default/frontend.json"); + +impl ConfigKey { + /// Create a `ConfigKey` from ids. + pub(crate) fn from_id(id1: &str, id2: &str, id3: &str) -> Option { + match (id1, id2, id3) { + ("frontend", "", "") => Some(ConfigKey::Frontend), + ("frontend", "ip", ip) => ip.parse().ok().map(ConfigKey::FrontendForIp), + _ => None, + } + } + + /// Convert a `ConfigKey` to its ids. + pub(super) fn to_id(&self) -> (String, String, String) { + match self { + ConfigKey::Frontend => ("frontend".to_string(), String::new(), String::new()), + ConfigKey::FrontendForIp(ip) => { + ("frontend".to_string(), "ip".to_string(), ip.to_string()) + }, + } + } + + /// Validate the provided value against the JSON schema. + pub(super) fn validate(&self, value: &Value) -> anyhow::Result<()> { + let schema: Value = serde_json::from_str(FRONTEND_JSON_SCHEMA) + .map_err(|e| anyhow::anyhow!("Failed to parse JSON schema: {:?}", e))?; + + let validator = jsonschema::validator_for(&schema) + .map_err(|e| anyhow::anyhow!("Failed to create JSON validator: {:?}", e))?; + + if validator.is_valid(value) { + Ok(()) + } else { + Err(anyhow::anyhow!( + "Provided JSON value does not match the schema for {:?}.", + self + )) + } + } + + /// Retrieve the default configuration value. + pub(super) fn default(&self) -> anyhow::Result { + let default_value: Value = serde_json::from_str(FRONTEND_DEFAULT) + .map_err(|e| anyhow::anyhow!("Failed to parse default JSON: {:?}", e))?; + + match self { + ConfigKey::Frontend | ConfigKey::FrontendForIp(_) => Ok(default_value), + } + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn test_validate() { + let value = json!({ + "test": "test" + }); + let result = ConfigKey::Frontend.validate(&value); + assert!(result.is_ok()); + } + + #[test] + fn test_default() { + let result = ConfigKey::Frontend.default(); + assert!(result.is_ok()); + } +} diff --git a/catalyst-gateway/bin/src/db/event/config/mod.rs b/catalyst-gateway/bin/src/db/event/config/mod.rs new file mode 100644 index 00000000000..eec54d310a5 --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/config/mod.rs @@ -0,0 +1,48 @@ +//! Configuration query + +use key::ConfigKey; +use serde_json::Value; + +use crate::db::event::EventDB; + +pub(crate) mod key; + +/// Configuration struct +pub(crate) struct Config {} + +/// SQL get configuration +const GET_CONFIG: &str = include_str!("sql/get.sql"); +/// SQL insert configuration +const INSERT_CONFIG: &str = include_str!("sql/insert.sql"); + +impl Config { + /// Retrieve configuration based on the given `ConfigKey`. + /// + /// # Returns + /// + /// - A JSON value of the configuration, if not found, returns the default value. + pub(crate) async fn get(id: ConfigKey) -> anyhow::Result { + let (id1, id2, id3) = id.to_id(); + let rows = EventDB::query(GET_CONFIG, &[&id1, &id2, &id3]).await?; + + if let Some(row) = rows.first() { + let value: Value = row.get(0); + id.validate(&value).map_err(|e| anyhow::anyhow!(e))?; + return Ok(value); + } + + // If data not found return default config value + id.default() + } + + /// Insert configuration for the given `ConfigKey`. + pub(crate) async fn insert(id: ConfigKey, value: Value) -> anyhow::Result<()> { + // Validate the value before inserting + id.validate(&value)?; + + let (id1, id2, id3) = id.to_id(); + EventDB::query(INSERT_CONFIG, &[&id1, &id2, &id3, &value]).await?; + + Ok(()) + } +} diff --git a/catalyst-gateway/bin/src/db/event/config/sql/get.sql b/catalyst-gateway/bin/src/db/event/config/sql/get.sql new file mode 100644 index 00000000000..82a93dbd0f8 --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/config/sql/get.sql @@ -0,0 +1,3 @@ +SELECT value +FROM config +WHERE id1 = $1 AND id2 = $2 AND id3 = $3; \ No newline at end of file diff --git a/catalyst-gateway/bin/src/db/event/config/sql/insert.sql b/catalyst-gateway/bin/src/db/event/config/sql/insert.sql new file mode 100644 index 00000000000..2d598473dfe --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/config/sql/insert.sql @@ -0,0 +1,4 @@ +INSERT INTO config + (id1, id2, id3, value) +VALUES + ($1, $2, $3, $4) \ No newline at end of file diff --git a/catalyst-gateway/bin/src/db/event/mod.rs b/catalyst-gateway/bin/src/db/event/mod.rs index 39e5ed3b869..d0914f3a888 100644 --- a/catalyst-gateway/bin/src/db/event/mod.rs +++ b/catalyst-gateway/bin/src/db/event/mod.rs @@ -14,6 +14,7 @@ use tracing::{debug, debug_span, error, Instrument}; use crate::settings::Settings; +pub(crate) mod config; pub(crate) mod error; pub(crate) mod legacy; pub(crate) mod schema_check; diff --git a/catalyst-gateway/bin/src/service/api/config/mod.rs b/catalyst-gateway/bin/src/service/api/config/mod.rs new file mode 100644 index 00000000000..f9cf068287a --- /dev/null +++ b/catalyst-gateway/bin/src/service/api/config/mod.rs @@ -0,0 +1,100 @@ +//! Configuration Endpoints + +use std::{net::IpAddr, str::FromStr}; + +use poem_openapi::{param::Path, payload::Json, ApiResponse, Multipart, OpenApi}; + +use crate::{ + db::event::config::{key::ConfigKey, Config}, + service::common::tags::ApiTags, +}; + +/// Configuration API struct +pub(crate) struct ConfigApi; + +/// Endpoint responses +#[derive(ApiResponse)] +enum Responses { + /// Configuration result + #[oai(status = 200)] + Ok(Json), + /// Configuration not found + #[oai(status = 404)] + NotFound(Json), + /// Bad request + #[oai(status = 400)] + BadRequest(Json), +} + +/// Configuration insert request +#[derive(Multipart)] +struct ConfigInsertRequest { + /// ID1 of config table + id1: String, + /// ID2 of config table + id2: Option, + /// ID3 of config table + id3: Option, + /// Value of config table + value: String, +} + +#[OpenApi(prefix_path = "/config", tag = "ApiTags::Config")] +impl ConfigApi { + /// Get the general configuration + #[oai(path = "/", method = "get", operation_id = "get_general_config")] + async fn get_general(&self) -> Responses { + match Config::get(ConfigKey::Frontend).await { + Ok(config) => Responses::Ok(Json(config.to_string())), + Err(err) => Responses::NotFound(Json(err.to_string())), + } + } + + /// Get configuration for a specific IP + #[oai(path = "/:ip", method = "get", operation_id = "get_ip_config")] + async fn get_with_ip(&self, ip: Path) -> Responses { + match IpAddr::from_str(&ip.0) { + Ok(parsed_ip) => { + match Config::get(ConfigKey::FrontendForIp(parsed_ip)).await { + Ok(config) => Responses::Ok(Json(config.to_string())), + Err(err) => Responses::NotFound(Json(err.to_string())), + } + }, + Err(err) => Responses::BadRequest(Json(format!("Invalid IP address: {err}"))), + } + } + + /// Insert configuration + #[oai(path = "/insert", method = "post", operation_id = "insert_config")] + async fn insert(&self, payload: ConfigInsertRequest) -> Responses { + let config_key = ConfigKey::from_id( + &payload.id1, + &payload.id2.unwrap_or_default(), + &payload.id3.unwrap_or_default(), + ); + + if let Some(config_key) = config_key { + match serde_json::from_str(&payload.value) { + Ok(parsed_value) => { + match Config::insert(config_key, parsed_value).await { + Ok(()) => { + Responses::Ok(Json("Configuration inserted successfully.".to_string())) + }, + Err(err) => { + Responses::BadRequest(Json(format!( + "Failed to insert configuration: {err}" + ))) + }, + } + }, + Err(err) => { + Responses::BadRequest(Json(format!("Failed to parse JSON value: {err}"))) + }, + } + } else { + Responses::BadRequest(Json( + "Invalid configuration key derives from ids.".to_string(), + )) + } + } +} diff --git a/catalyst-gateway/bin/src/service/api/mod.rs b/catalyst-gateway/bin/src/service/api/mod.rs index 2ab54ffb2ff..409e2890934 100644 --- a/catalyst-gateway/bin/src/service/api/mod.rs +++ b/catalyst-gateway/bin/src/service/api/mod.rs @@ -4,6 +4,7 @@ //! It however does NOT contain any processing for them, that is defined elsewhere. use std::net::IpAddr; +use config::ConfigApi; use gethostname::gethostname; use health::HealthApi; use legacy::LegacyApi; @@ -15,6 +16,7 @@ use crate::settings::Settings; /// Auth mod auth; pub(crate) mod cardano; +mod config; mod health; mod legacy; @@ -59,11 +61,12 @@ const TERMS_OF_SERVICE: &str = "https://github.com/input-output-hk/catalyst-voices/blob/main/CODE_OF_CONDUCT.md"; /// Create the `OpenAPI` definition -pub(crate) fn mk_api() -> OpenApiService<(HealthApi, CardanoApi, LegacyApi), ()> { +pub(crate) fn mk_api() -> OpenApiService<(HealthApi, CardanoApi, ConfigApi, LegacyApi), ()> { let mut service = OpenApiService::new( ( HealthApi, CardanoApi, + ConfigApi, (legacy::RegistrationApi, legacy::V0Api, legacy::V1Api), ), API_TITLE, diff --git a/catalyst-gateway/bin/src/service/common/tags.rs b/catalyst-gateway/bin/src/service/common/tags.rs index 7b9fbb3ee2c..c83e040340f 100644 --- a/catalyst-gateway/bin/src/service/common/tags.rs +++ b/catalyst-gateway/bin/src/service/common/tags.rs @@ -17,4 +17,6 @@ pub(crate) enum ApiTags { V0, /// API Version 1 Endpoints V1, + /// Config + Config, } From 241481bb44a42ad6ce76e5457909fe00de84f910 Mon Sep 17 00:00:00 2001 From: bkioshn Date: Wed, 9 Oct 2024 13:15:24 +0700 Subject: [PATCH 02/37] feat: add jsonschema lib Signed-off-by: bkioshn --- .config/dictionaries/project.dic | 1 + catalyst-gateway/bin/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic index d145a4877ef..d8d6d24af0c 100644 --- a/.config/dictionaries/project.dic +++ b/.config/dictionaries/project.dic @@ -119,6 +119,7 @@ Joaquín jorm jormungandr Jörmungandr +jsonschema junitreport junitxml Keyhash diff --git a/catalyst-gateway/bin/Cargo.toml b/catalyst-gateway/bin/Cargo.toml index 879d276320d..9ff9929eb43 100644 --- a/catalyst-gateway/bin/Cargo.toml +++ b/catalyst-gateway/bin/Cargo.toml @@ -93,6 +93,7 @@ base64 = "0.22.1" dashmap = "6.1.0" x509-cert = "0.2.5" der-parser = "9.0.0" +jsonschema = "0.22.3" [dev-dependencies] proptest = "1.5.0" From 9e15dff21acd39cd4eadadb52b66fdb436864a4c Mon Sep 17 00:00:00 2001 From: bkioshn Date: Wed, 9 Oct 2024 13:16:14 +0700 Subject: [PATCH 03/37] fix: config table Signed-off-by: bkioshn --- .../event-db/migrations/V1__config_tables.sql | 55 ++----------------- 1 file changed, 6 insertions(+), 49 deletions(-) diff --git a/catalyst-gateway/event-db/migrations/V1__config_tables.sql b/catalyst-gateway/event-db/migrations/V1__config_tables.sql index cae7c4ea73a..529f2bfdbaf 100644 --- a/catalyst-gateway/event-db/migrations/V1__config_tables.sql +++ b/catalyst-gateway/event-db/migrations/V1__config_tables.sql @@ -71,10 +71,10 @@ Must match the `name` component of the $id URI inside the schema.'; -- ------------------------------------------------------------------------------------------------- -- Config Table --- This table is looked up with three keys, `id`, `id2` and `id3` +-- This table is looked up with three keys, `id1`, `id2` and `id3` CREATE TABLE config ( row_id SERIAL PRIMARY KEY, - id VARCHAR NOT NULL, + id1 VARCHAR NOT NULL, id2 VARCHAR NOT NULL, id3 VARCHAR NOT NULL, value JSONB NULL, @@ -84,7 +84,7 @@ CREATE TABLE config ( ); -- cardano+follower+preview must be unique, they are a combined key. -CREATE UNIQUE INDEX config_idx ON config (id, id2, id3); +CREATE UNIQUE INDEX config_idx ON config (id1, id2, id3); COMMENT ON TABLE config IS 'General JSON Configuration and Data Values. @@ -95,17 +95,16 @@ Defined Data Formats: COMMENT ON COLUMN config.row_id IS 'Synthetic unique key. Always lookup using `cardano.follower.preview`'; -COMMENT ON COLUMN config.id IS -'The name/id of the general config value/variable'; +COMMENT ON COLUMN config.id1 IS +'The name/id1 of the general config value/variable'; COMMENT ON COLUMN config.id2 IS '2nd ID of the general config value. Must be defined, use "" if not required.'; - COMMENT ON COLUMN config.id3 IS '3rd ID of the general config value. Must be defined, use "" if not required.'; COMMENT ON COLUMN config.value IS -'The JSON value of the system variable `cardano.follower.preview`'; +'The configuration value in JSON format.'; COMMENT ON COLUMN config.value_schema IS 'The Schema the Config Value conforms to. The `value` field must conform to this schema.'; @@ -114,48 +113,6 @@ COMMENT ON INDEX config_idx IS 'We use three keys combined uniquely rather than forcing string concatenation at the app level to allow for querying groups of data.'; - -INSERT INTO config (id, id2, id3, value) -VALUES --- ( --- 'cardano', --- 'follower', --- 'mainnet', --- '{ --- "relay": "relays-new.cardano-mainnet.iohk.io:3001", --- "mithril_snapshot": { --- "path": "/tmp/mainnet/immutable", --- "timing_pattern": 25 --- } --- }' --- ), --- ( --- 'cardano', --- 'follower', --- 'preview', --- '{ --- "relay": "preview-node.play.dev.cardano.org:3001", --- "mithril_snapshot": { --- "path": "/tmp/preview/immutable", --- "timing_pattern": 25 --- } --- }' --- ), -( - 'cardano', - 'follower', - 'preprod', - '{ - "relay": "preprod-node.play.dev.cardano.org:3001", - "mithril_snapshot": { - "path": "/tmp/preprod/immutable", - "timing_pattern": 25 - } - }' -); - - - -- ------------------------------------------------------------------------------------------------- -- * Temporary. From c5aa5abeb40c1ea657c95b1a9531f64a944fd494 Mon Sep 17 00:00:00 2001 From: bkioshn Date: Wed, 9 Oct 2024 13:28:14 +0700 Subject: [PATCH 04/37] fix: sql format Signed-off-by: bkioshn --- catalyst-gateway/bin/src/db/event/config/sql/get.sql | 2 +- catalyst-gateway/bin/src/db/event/config/sql/insert.sql | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/config/sql/get.sql b/catalyst-gateway/bin/src/db/event/config/sql/get.sql index 82a93dbd0f8..1664d9f5160 100644 --- a/catalyst-gateway/bin/src/db/event/config/sql/get.sql +++ b/catalyst-gateway/bin/src/db/event/config/sql/get.sql @@ -1,3 +1,3 @@ SELECT value FROM config -WHERE id1 = $1 AND id2 = $2 AND id3 = $3; \ No newline at end of file +WHERE id1 = $1 AND id2 = $2 AND id3 = $3; diff --git a/catalyst-gateway/bin/src/db/event/config/sql/insert.sql b/catalyst-gateway/bin/src/db/event/config/sql/insert.sql index 2d598473dfe..a583553732c 100644 --- a/catalyst-gateway/bin/src/db/event/config/sql/insert.sql +++ b/catalyst-gateway/bin/src/db/event/config/sql/insert.sql @@ -1,4 +1,2 @@ -INSERT INTO config - (id1, id2, id3, value) -VALUES - ($1, $2, $3, $4) \ No newline at end of file +INSERT INTO config (id1, id2, id3, value) +VALUES ($1, $2, $3, $4) From e3fd2cebfa5237a67e2c6684636eaf7f7e7e837f Mon Sep 17 00:00:00 2001 From: bkioshn Date: Thu, 10 Oct 2024 13:56:47 +0700 Subject: [PATCH 05/37] fix: comment to sql file Signed-off-by: bkioshn --- catalyst-gateway/bin/src/db/event/config/sql/get.sql | 6 +++++- catalyst-gateway/bin/src/db/event/config/sql/insert.sql | 3 ++- catalyst-gateway/event-db/migrations/V1__config_tables.sql | 6 +++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/config/sql/get.sql b/catalyst-gateway/bin/src/db/event/config/sql/get.sql index 1664d9f5160..ee018289ba5 100644 --- a/catalyst-gateway/bin/src/db/event/config/sql/get.sql +++ b/catalyst-gateway/bin/src/db/event/config/sql/get.sql @@ -1,3 +1,7 @@ +-- Select the 'value' column from the 'config' table SELECT value FROM config -WHERE id1 = $1 AND id2 = $2 AND id3 = $3; +-- Filter the results based on the following conditions +WHERE id1 = $1 -- Match rows where 'id1' equals the first parameter + AND id2 = $2 -- Match rows where 'id2' equals the second parameter + AND id3 = $3; -- Match rows where 'id3' equals the third parameter diff --git a/catalyst-gateway/bin/src/db/event/config/sql/insert.sql b/catalyst-gateway/bin/src/db/event/config/sql/insert.sql index a583553732c..bc8973aab08 100644 --- a/catalyst-gateway/bin/src/db/event/config/sql/insert.sql +++ b/catalyst-gateway/bin/src/db/event/config/sql/insert.sql @@ -1,2 +1,3 @@ +-- Insert a new row into the 'config' table INSERT INTO config (id1, id2, id3, value) -VALUES ($1, $2, $3, $4) +VALUES ($1, $2, $3, $4); -- The values to insert for each column diff --git a/catalyst-gateway/event-db/migrations/V1__config_tables.sql b/catalyst-gateway/event-db/migrations/V1__config_tables.sql index 529f2bfdbaf..5a2f137a79a 100644 --- a/catalyst-gateway/event-db/migrations/V1__config_tables.sql +++ b/catalyst-gateway/event-db/migrations/V1__config_tables.sql @@ -96,12 +96,12 @@ COMMENT ON COLUMN config.row_id IS 'Synthetic unique key. Always lookup using `cardano.follower.preview`'; COMMENT ON COLUMN config.id1 IS -'The name/id1 of the general config value/variable'; +'The primary ID of the config.'; COMMENT ON COLUMN config.id2 IS -'2nd ID of the general config value. +'The secondary ID of the config. Must be defined, use "" if not required.'; COMMENT ON COLUMN config.id3 IS -'3rd ID of the general config value. +'The tertiary ID of the config. Must be defined, use "" if not required.'; COMMENT ON COLUMN config.value IS 'The configuration value in JSON format.'; From 3c78d55918af506b310fe22b024d2b1dfcc6046e Mon Sep 17 00:00:00 2001 From: bkioshn Date: Thu, 10 Oct 2024 13:57:02 +0700 Subject: [PATCH 06/37] feat: add upsert sql Signed-off-by: bkioshn --- .../bin/src/db/event/config/sql/upsert.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 catalyst-gateway/bin/src/db/event/config/sql/upsert.sql diff --git a/catalyst-gateway/bin/src/db/event/config/sql/upsert.sql b/catalyst-gateway/bin/src/db/event/config/sql/upsert.sql new file mode 100644 index 00000000000..392da538440 --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/config/sql/upsert.sql @@ -0,0 +1,14 @@ +-- Insert a new configuration entry into the 'config' table +INSERT INTO config + (id1, id2, id3, value) +VALUES + ($1, $2, $3, $4) +-- Values to insert for each column + +-- Handle conflicts when attempting to insert a row that would violate the unique constraint +ON CONFLICT +(id1, id2, id3) -- Specify the unique constraint columns that identify conflicts + +-- If a conflict occurs, update the existing row 'value' column with the new value provided +DO +UPDATE SET value = EXCLUDED.value; -- 'EXCLUDED' refers to the values that were proposed for insertion From 42b35b43478bb7b0c1060bab0c847a6062847c0e Mon Sep 17 00:00:00 2001 From: bkioshn Date: Thu, 10 Oct 2024 13:57:24 +0700 Subject: [PATCH 07/37] fix: update endpoint Signed-off-by: bkioshn --- .../bin/src/service/api/config/mod.rs | 151 +++++++++++------- 1 file changed, 92 insertions(+), 59 deletions(-) diff --git a/catalyst-gateway/bin/src/service/api/config/mod.rs b/catalyst-gateway/bin/src/service/api/config/mod.rs index f9cf068287a..dfc02d1f8a4 100644 --- a/catalyst-gateway/bin/src/service/api/config/mod.rs +++ b/catalyst-gateway/bin/src/service/api/config/mod.rs @@ -2,7 +2,12 @@ use std::{net::IpAddr, str::FromStr}; -use poem_openapi::{param::Path, payload::Json, ApiResponse, Multipart, OpenApi}; +use poem_openapi::{ + param::{Header, Query}, + payload::Json, + ApiResponse, OpenApi, +}; +use serde_json::Value; use crate::{ db::event::config::{key::ConfigKey, Config}, @@ -26,75 +31,103 @@ enum Responses { BadRequest(Json), } -/// Configuration insert request -#[derive(Multipart)] -struct ConfigInsertRequest { - /// ID1 of config table - id1: String, - /// ID2 of config table - id2: Option, - /// ID3 of config table - id3: Option, - /// Value of config table - value: String, -} - -#[OpenApi(prefix_path = "/config", tag = "ApiTags::Config")] +#[OpenApi(tag = "ApiTags::Config")] impl ConfigApi { - /// Get the general configuration - #[oai(path = "/", method = "get", operation_id = "get_general_config")] - async fn get_general(&self) -> Responses { - match Config::get(ConfigKey::Frontend).await { - Ok(config) => Responses::Ok(Json(config.to_string())), + /// Get the configuration for the frontend. + /// Retrieving IP from X-Forwarded-For header if provided. + /// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For + #[oai( + path = "/draft/config/frontend", + method = "get", + operation_id = "get_config_frontend" + )] + async fn get_frontend( + &self, #[oai(name = "X-Forwarded-For")] header: Header>, + ) -> Responses { + // Retrieve the IP address from the header + // According to the X-Forwarded-For header spec, the first value is the IP address + let ip_address = header + .0 + .as_ref() + .and_then(|h| h.split(',').next()) + .map(String::from); + + // Fetch the general configuration + let general_config = Config::get(ConfigKey::Frontend).await; + + // Attempt to fetch the IP configuration + let ip_config = if let Some(ip) = ip_address { + match IpAddr::from_str(&ip) { + Ok(parsed_ip) => Config::get(ConfigKey::FrontendForIp(parsed_ip)).await.ok(), + Err(_) => { + return Responses::BadRequest(Json(format!("Invalid IP address: {ip}"))); + }, + } + } else { + None + }; + + // Handle the response + match general_config { + Ok(general) => { + // If there is IP specific config, replace any key in the general config with + // the keys from the IP specific config + let response_config = if let Some(ip_specific) = ip_config { + merge_configs(&general, &ip_specific) + } else { + general + }; + + Responses::Ok(Json(response_config.to_string())) + }, Err(err) => Responses::NotFound(Json(err.to_string())), } } - /// Get configuration for a specific IP - #[oai(path = "/:ip", method = "get", operation_id = "get_ip_config")] - async fn get_with_ip(&self, ip: Path) -> Responses { - match IpAddr::from_str(&ip.0) { - Ok(parsed_ip) => { - match Config::get(ConfigKey::FrontendForIp(parsed_ip)).await { - Ok(config) => Responses::Ok(Json(config.to_string())), - Err(err) => Responses::NotFound(Json(err.to_string())), + /// Insert or update the frontend configuration. + #[oai( + path = "/draft/config/frontend", + method = "put", + operation_id = "put_config_frontend" + )] + async fn put_frontend(&self, ip_query: Query>, body: Json) -> Responses { + let body_value = body.0; + + match ip_query.0 { + Some(ip) => { + match IpAddr::from_str(&ip) { + Ok(parsed_ip) => upsert(ConfigKey::FrontendForIp(parsed_ip), body_value).await, + Err(err) => Responses::BadRequest(Json(format!("Invalid IP address: {err}"))), } }, - Err(err) => Responses::BadRequest(Json(format!("Invalid IP address: {err}"))), + None => upsert(ConfigKey::Frontend, body_value).await, } } +} - /// Insert configuration - #[oai(path = "/insert", method = "post", operation_id = "insert_config")] - async fn insert(&self, payload: ConfigInsertRequest) -> Responses { - let config_key = ConfigKey::from_id( - &payload.id1, - &payload.id2.unwrap_or_default(), - &payload.id3.unwrap_or_default(), - ); - - if let Some(config_key) = config_key { - match serde_json::from_str(&payload.value) { - Ok(parsed_value) => { - match Config::insert(config_key, parsed_value).await { - Ok(()) => { - Responses::Ok(Json("Configuration inserted successfully.".to_string())) - }, - Err(err) => { - Responses::BadRequest(Json(format!( - "Failed to insert configuration: {err}" - ))) - }, - } - }, - Err(err) => { - Responses::BadRequest(Json(format!("Failed to parse JSON value: {err}"))) - }, +/// Helper function to merge two JSON values. +fn merge_configs(general: &Value, ip_specific: &Value) -> Value { + let mut merged = general.clone(); + + if let Some(ip_specific_obj) = ip_specific.as_object() { + if let Some(merged_obj) = merged.as_object_mut() { + for (key, value) in ip_specific_obj { + if let Some(existing_value) = merged_obj.get_mut(key) { + *existing_value = value.clone(); + } else { + merged_obj.insert(key.clone(), value.clone()); + } } - } else { - Responses::BadRequest(Json( - "Invalid configuration key derives from ids.".to_string(), - )) } } + + merged +} + +/// Helper function to handle upsert. +async fn upsert(key: ConfigKey, value: Value) -> Responses { + match Config::upsert(key, value).await { + Ok(()) => Responses::Ok(Json("Configuration upsert successfully.".to_string())), + Err(err) => Responses::BadRequest(Json(format!("Failed to upsert configuration: {err}"))), + } } From d9748e735b954076ed211f3629cd9dab4349c2a4 Mon Sep 17 00:00:00 2001 From: bkioshn Date: Thu, 10 Oct 2024 13:57:58 +0700 Subject: [PATCH 08/37] fix: frontend key implementation Signed-off-by: bkioshn --- .../bin/src/db/event/config/key.rs | 82 +++++++++++-------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/config/key.rs b/catalyst-gateway/bin/src/db/event/config/key.rs index 6cc884a662b..d6fc687fd72 100644 --- a/catalyst-gateway/bin/src/db/event/config/key.rs +++ b/catalyst-gateway/bin/src/db/event/config/key.rs @@ -1,34 +1,42 @@ //! Configuration Key -use std::net::IpAddr; +use std::{net::IpAddr, sync::LazyLock}; use serde_json::Value; +use tracing::error; /// Configuration key #[derive(Debug, Clone, PartialEq)] pub(crate) enum ConfigKey { - /// Frontend general configuration + /// Frontend general configuration. Frontend, - /// Frontend configuration for a specific IP address + /// Frontend configuration for a specific IP address. FrontendForIp(IpAddr), } -/// Frontend JSON schema -const FRONTEND_JSON_SCHEMA: &str = include_str!("jsonschema/frontend.json"); -/// Default frontend configuration -const FRONTEND_DEFAULT: &str = include_str!("default/frontend.json"); +/// Frontend schema configuration. +static FRONTEND_SCHEMA: LazyLock> = + LazyLock::new(|| load_json_lazy(include_str!("jsonschema/frontend.json"))); -impl ConfigKey { - /// Create a `ConfigKey` from ids. - pub(crate) fn from_id(id1: &str, id2: &str, id3: &str) -> Option { - match (id1, id2, id3) { - ("frontend", "", "") => Some(ConfigKey::Frontend), - ("frontend", "ip", ip) => ip.parse().ok().map(ConfigKey::FrontendForIp), - _ => None, - } +/// Frontend default configuration. +static FRONTEND_DEFAULT: LazyLock> = + LazyLock::new(|| load_json_lazy(include_str!("default/frontend.json"))); + +/// Helper function to convert a JSON string to a JSON value. +fn load_json_lazy(data: &str) -> Option { + let json = serde_json::from_str(data); + + match json { + Ok(value) => Some(value), + Err(err) => { + error!("Error parsing JSON : {:?}", err); + None + }, } +} - /// Convert a `ConfigKey` to its ids. +impl ConfigKey { + /// Convert a `ConfigKey` to its corresponding IDs. pub(super) fn to_id(&self) -> (String, String, String) { match self { ConfigKey::Frontend => ("frontend".to_string(), String::new(), String::new()), @@ -40,30 +48,38 @@ impl ConfigKey { /// Validate the provided value against the JSON schema. pub(super) fn validate(&self, value: &Value) -> anyhow::Result<()> { - let schema: Value = serde_json::from_str(FRONTEND_JSON_SCHEMA) - .map_err(|e| anyhow::anyhow!("Failed to parse JSON schema: {:?}", e))?; + // Retrieve the schema based on ConfigKey + #[allow(clippy::match_same_arms)] + let schema = match self { + ConfigKey::Frontend => &*FRONTEND_SCHEMA, + ConfigKey::FrontendForIp(_) => &*FRONTEND_SCHEMA, + }; - let validator = jsonschema::validator_for(&schema) - .map_err(|e| anyhow::anyhow!("Failed to create JSON validator: {:?}", e))?; + let validator = match schema { + Some(s) => { + jsonschema::validator_for(s) + .map_err(|e| anyhow::anyhow!("Failed to create JSON validator: {:?}", e))? + }, + None => return Err(anyhow::anyhow!("Failed to retrieve JSON schema")), + }; + // Validate the value against the schema if validator.is_valid(value) { - Ok(()) - } else { - Err(anyhow::anyhow!( - "Provided JSON value does not match the schema for {:?}.", - self - )) + return Ok(()); } + Err(anyhow::anyhow!("Invalid configuration")) } /// Retrieve the default configuration value. - pub(super) fn default(&self) -> anyhow::Result { - let default_value: Value = serde_json::from_str(FRONTEND_DEFAULT) - .map_err(|e| anyhow::anyhow!("Failed to parse default JSON: {:?}", e))?; + pub(super) fn default(&self) -> Option { + // Retrieve the default value based on the ConfigKey + #[allow(clippy::match_same_arms)] + let default = match self { + ConfigKey::Frontend => &*FRONTEND_DEFAULT, + ConfigKey::FrontendForIp(_) => &*FRONTEND_DEFAULT, + }; - match self { - ConfigKey::Frontend | ConfigKey::FrontendForIp(_) => Ok(default_value), - } + default.clone() } } @@ -85,6 +101,6 @@ mod tests { #[test] fn test_default() { let result = ConfigKey::Frontend.default(); - assert!(result.is_ok()); + assert!(result.is_some()); } } From 8f57b94a20be411aec95b60cd3367cc5a9e8ad26 Mon Sep 17 00:00:00 2001 From: bkioshn Date: Thu, 10 Oct 2024 13:58:21 +0700 Subject: [PATCH 09/37] fix: config query Signed-off-by: bkioshn --- .../bin/src/db/event/config/mod.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/config/mod.rs b/catalyst-gateway/bin/src/db/event/config/mod.rs index eec54d310a5..0ce3d1c922b 100644 --- a/catalyst-gateway/bin/src/db/event/config/mod.rs +++ b/catalyst-gateway/bin/src/db/event/config/mod.rs @@ -10,10 +10,10 @@ pub(crate) mod key; /// Configuration struct pub(crate) struct Config {} -/// SQL get configuration +/// SQL get configuration. const GET_CONFIG: &str = include_str!("sql/get.sql"); -/// SQL insert configuration -const INSERT_CONFIG: &str = include_str!("sql/insert.sql"); +/// SQL update if exist or else insert configuration. +const UPSERT_CONFIG: &str = include_str!("sql/upsert.sql"); impl Config { /// Retrieve configuration based on the given `ConfigKey`. @@ -32,16 +32,19 @@ impl Config { } // If data not found return default config value - id.default() + match id.default() { + Some(default) => Ok(default), + None => Err(anyhow::anyhow!("Default value not found for {:?}", id)), + } } - /// Insert configuration for the given `ConfigKey`. - pub(crate) async fn insert(id: ConfigKey, value: Value) -> anyhow::Result<()> { - // Validate the value before inserting + /// Update or insert (upsert) configuration for the given `ConfigKey`. + pub(crate) async fn upsert(id: ConfigKey, value: Value) -> anyhow::Result<()> { + // Validate the value id.validate(&value)?; let (id1, id2, id3) = id.to_id(); - EventDB::query(INSERT_CONFIG, &[&id1, &id2, &id3, &value]).await?; + EventDB::query(UPSERT_CONFIG, &[&id1, &id2, &id3, &value]).await?; Ok(()) } From bf7f8e0b75289be8551c99e16062f24c1e469f88 Mon Sep 17 00:00:00 2001 From: bkioshn Date: Thu, 10 Oct 2024 14:48:36 +0700 Subject: [PATCH 10/37] fix: sql lint Signed-off-by: bkioshn --- .../bin/src/db/event/config/sql/get.sql | 6 +++--- .../bin/src/db/event/config/sql/upsert.sql | 13 ++++--------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/config/sql/get.sql b/catalyst-gateway/bin/src/db/event/config/sql/get.sql index ee018289ba5..25581803649 100644 --- a/catalyst-gateway/bin/src/db/event/config/sql/get.sql +++ b/catalyst-gateway/bin/src/db/event/config/sql/get.sql @@ -1,7 +1,7 @@ -- Select the 'value' column from the 'config' table SELECT value FROM config --- Filter the results based on the following conditions -WHERE id1 = $1 -- Match rows where 'id1' equals the first parameter - AND id2 = $2 -- Match rows where 'id2' equals the second parameter +WHERE + id1 = $1 -- Match rows where 'id1' equals the first parameter + AND id2 = $2 -- Match rows where 'id2' equals the second parameter AND id3 = $3; -- Match rows where 'id3' equals the third parameter diff --git a/catalyst-gateway/bin/src/db/event/config/sql/upsert.sql b/catalyst-gateway/bin/src/db/event/config/sql/upsert.sql index 392da538440..a8e47779a2c 100644 --- a/catalyst-gateway/bin/src/db/event/config/sql/upsert.sql +++ b/catalyst-gateway/bin/src/db/event/config/sql/upsert.sql @@ -1,14 +1,9 @@ -- Insert a new configuration entry into the 'config' table -INSERT INTO config - (id1, id2, id3, value) -VALUES - ($1, $2, $3, $4) --- Values to insert for each column +INSERT INTO config (id1, id2, id3, value) +VALUES ($1, $2, $3, $4) -- Values to insert for each column -- Handle conflicts when attempting to insert a row that would violate the unique constraint -ON CONFLICT -(id1, id2, id3) -- Specify the unique constraint columns that identify conflicts +ON CONFLICT (id1, id2, id3) -- Specify the unique constraint columns that identify conflicts -- If a conflict occurs, update the existing row 'value' column with the new value provided -DO -UPDATE SET value = EXCLUDED.value; -- 'EXCLUDED' refers to the values that were proposed for insertion +DO UPDATE SET value = excluded.value; -- 'EXCLUDED' refers to the values that were proposed for insertion From 798b73f332531f820257da3b0fa3ddd2f6de9518 Mon Sep 17 00:00:00 2001 From: bkioshn Date: Fri, 11 Oct 2024 19:02:09 +0700 Subject: [PATCH 11/37] fix: refactor Signed-off-by: bkioshn --- .../bin/src/db/event/config/key.rs | 33 ++++++------------- .../bin/src/db/event/config/mod.rs | 12 +++---- .../bin/src/service/api/config/mod.rs | 4 ++- 3 files changed, 18 insertions(+), 31 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/config/key.rs b/catalyst-gateway/bin/src/db/event/config/key.rs index d6fc687fd72..d7e90756737 100644 --- a/catalyst-gateway/bin/src/db/event/config/key.rs +++ b/catalyst-gateway/bin/src/db/event/config/key.rs @@ -26,13 +26,8 @@ static FRONTEND_DEFAULT: LazyLock> = fn load_json_lazy(data: &str) -> Option { let json = serde_json::from_str(data); - match json { - Ok(value) => Some(value), - Err(err) => { - error!("Error parsing JSON : {:?}", err); - None - }, - } + json.inspect_err(|e| error!("Error parsing JSON : {e}")) + .ok() } impl ConfigKey { @@ -49,34 +44,26 @@ impl ConfigKey { /// Validate the provided value against the JSON schema. pub(super) fn validate(&self, value: &Value) -> anyhow::Result<()> { // Retrieve the schema based on ConfigKey - #[allow(clippy::match_same_arms)] let schema = match self { - ConfigKey::Frontend => &*FRONTEND_SCHEMA, - ConfigKey::FrontendForIp(_) => &*FRONTEND_SCHEMA, + ConfigKey::Frontend | ConfigKey::FrontendForIp(_) => &*FRONTEND_SCHEMA, }; - let validator = match schema { - Some(s) => { - jsonschema::validator_for(s) - .map_err(|e| anyhow::anyhow!("Failed to create JSON validator: {:?}", e))? - }, - None => return Err(anyhow::anyhow!("Failed to retrieve JSON schema")), + let Some(schema) = schema else { + return Err(anyhow::anyhow!("Failed to retrieve JSON schema")); }; + let validator = jsonschema::validator_for(schema) + .map_err(|e| anyhow::anyhow!("Failed to create JSON validator: {:?}", e))?; // Validate the value against the schema - if validator.is_valid(value) { - return Ok(()); - } - Err(anyhow::anyhow!("Invalid configuration")) + anyhow::ensure!(validator.is_valid(value), "Invalid configuration"); + Ok(()) } /// Retrieve the default configuration value. pub(super) fn default(&self) -> Option { // Retrieve the default value based on the ConfigKey - #[allow(clippy::match_same_arms)] let default = match self { - ConfigKey::Frontend => &*FRONTEND_DEFAULT, - ConfigKey::FrontendForIp(_) => &*FRONTEND_DEFAULT, + ConfigKey::Frontend | ConfigKey::FrontendForIp(_) => &*FRONTEND_DEFAULT, }; default.clone() diff --git a/catalyst-gateway/bin/src/db/event/config/mod.rs b/catalyst-gateway/bin/src/db/event/config/mod.rs index 0ce3d1c922b..ded01dd736e 100644 --- a/catalyst-gateway/bin/src/db/event/config/mod.rs +++ b/catalyst-gateway/bin/src/db/event/config/mod.rs @@ -28,13 +28,11 @@ impl Config { if let Some(row) = rows.first() { let value: Value = row.get(0); id.validate(&value).map_err(|e| anyhow::anyhow!(e))?; - return Ok(value); - } - - // If data not found return default config value - match id.default() { - Some(default) => Ok(default), - None => Err(anyhow::anyhow!("Default value not found for {:?}", id)), + Ok(value) + } else { + // If data not found return default config value + id.default() + .ok_or(anyhow::anyhow!("Default value not found for {:?}", id)) } } diff --git a/catalyst-gateway/bin/src/service/api/config/mod.rs b/catalyst-gateway/bin/src/service/api/config/mod.rs index dfc02d1f8a4..e322ef9d252 100644 --- a/catalyst-gateway/bin/src/service/api/config/mod.rs +++ b/catalyst-gateway/bin/src/service/api/config/mod.rs @@ -90,7 +90,9 @@ impl ConfigApi { method = "put", operation_id = "put_config_frontend" )] - async fn put_frontend(&self, ip_query: Query>, body: Json) -> Responses { + async fn put_frontend( + &self, #[oai(name = "IP")] ip_query: Query>, body: Json, + ) -> Responses { let body_value = body.0; match ip_query.0 { From c522ce468b300c7c970528c988f6e88c8fb6e9ee Mon Sep 17 00:00:00 2001 From: bkioshn Date: Tue, 15 Oct 2024 16:05:14 +0700 Subject: [PATCH 12/37] fix: config endpoint Signed-off-by: bkioshn --- .../bin/src/db/event/config/key.rs | 35 +++++++++---------- .../bin/src/db/event/config/mod.rs | 3 +- .../bin/src/service/api/config/mod.rs | 35 ++++++------------- 3 files changed, 27 insertions(+), 46 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/config/key.rs b/catalyst-gateway/bin/src/db/event/config/key.rs index d7e90756737..62ea1bce984 100644 --- a/catalyst-gateway/bin/src/db/event/config/key.rs +++ b/catalyst-gateway/bin/src/db/event/config/key.rs @@ -2,7 +2,7 @@ use std::{net::IpAddr, sync::LazyLock}; -use serde_json::Value; +use serde_json::{json, Value}; use tracing::error; /// Configuration key @@ -15,19 +15,21 @@ pub(crate) enum ConfigKey { } /// Frontend schema configuration. -static FRONTEND_SCHEMA: LazyLock> = +static FRONTEND_SCHEMA: LazyLock = LazyLock::new(|| load_json_lazy(include_str!("jsonschema/frontend.json"))); /// Frontend default configuration. -static FRONTEND_DEFAULT: LazyLock> = +static FRONTEND_DEFAULT: LazyLock = LazyLock::new(|| load_json_lazy(include_str!("default/frontend.json"))); /// Helper function to convert a JSON string to a JSON value. -fn load_json_lazy(data: &str) -> Option { - let json = serde_json::from_str(data); - - json.inspect_err(|e| error!("Error parsing JSON : {e}")) - .ok() +fn load_json_lazy(data: &str) -> Value { + serde_json::from_str(data).unwrap_or_else(|e| { + // Log the error + error!("Error parsing JSON: {}", e); + // And return an empty JSON object + json!({}) + }) } impl ConfigKey { @@ -48,25 +50,20 @@ impl ConfigKey { ConfigKey::Frontend | ConfigKey::FrontendForIp(_) => &*FRONTEND_SCHEMA, }; - let Some(schema) = schema else { - return Err(anyhow::anyhow!("Failed to retrieve JSON schema")); - }; let validator = jsonschema::validator_for(schema) .map_err(|e| anyhow::anyhow!("Failed to create JSON validator: {:?}", e))?; // Validate the value against the schema - anyhow::ensure!(validator.is_valid(value), "Invalid configuration"); + anyhow::ensure!(validator.is_valid(value), "Failed schema validation"); Ok(()) } /// Retrieve the default configuration value. - pub(super) fn default(&self) -> Option { + pub(super) fn default(&self) -> Value { // Retrieve the default value based on the ConfigKey - let default = match self { - ConfigKey::Frontend | ConfigKey::FrontendForIp(_) => &*FRONTEND_DEFAULT, - }; - - default.clone() + match self { + ConfigKey::Frontend | ConfigKey::FrontendForIp(_) => FRONTEND_DEFAULT.clone(), + } } } @@ -88,6 +85,6 @@ mod tests { #[test] fn test_default() { let result = ConfigKey::Frontend.default(); - assert!(result.is_some()); + assert!(result.is_object()); } } diff --git a/catalyst-gateway/bin/src/db/event/config/mod.rs b/catalyst-gateway/bin/src/db/event/config/mod.rs index ded01dd736e..3a408201bba 100644 --- a/catalyst-gateway/bin/src/db/event/config/mod.rs +++ b/catalyst-gateway/bin/src/db/event/config/mod.rs @@ -31,8 +31,7 @@ impl Config { Ok(value) } else { // If data not found return default config value - id.default() - .ok_or(anyhow::anyhow!("Default value not found for {:?}", id)) + Ok(id.default()) } } diff --git a/catalyst-gateway/bin/src/service/api/config/mod.rs b/catalyst-gateway/bin/src/service/api/config/mod.rs index e322ef9d252..fc0410472ed 100644 --- a/catalyst-gateway/bin/src/service/api/config/mod.rs +++ b/catalyst-gateway/bin/src/service/api/config/mod.rs @@ -2,12 +2,10 @@ use std::{net::IpAddr, str::FromStr}; -use poem_openapi::{ - param::{Header, Query}, - payload::Json, - ApiResponse, OpenApi, -}; +use poem::web::RealIp; +use poem_openapi::{param::Query, payload::Json, ApiResponse, OpenApi}; use serde_json::Value; +use tracing::info; use crate::{ db::event::config::{key::ConfigKey, Config}, @@ -34,35 +32,22 @@ enum Responses { #[OpenApi(tag = "ApiTags::Config")] impl ConfigApi { /// Get the configuration for the frontend. - /// Retrieving IP from X-Forwarded-For header if provided. - /// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For + /// + /// Retrieving IP from X-Real-IP, Forwarded, X-Forwarded-For or Remote Address. #[oai( path = "/draft/config/frontend", method = "get", operation_id = "get_config_frontend" )] - async fn get_frontend( - &self, #[oai(name = "X-Forwarded-For")] header: Header>, - ) -> Responses { - // Retrieve the IP address from the header - // According to the X-Forwarded-For header spec, the first value is the IP address - let ip_address = header - .0 - .as_ref() - .and_then(|h| h.split(',').next()) - .map(String::from); + async fn get_frontend(&self, ip_address: RealIp) -> Responses { + info!("IP Address: {:?}", ip_address.0); // Fetch the general configuration let general_config = Config::get(ConfigKey::Frontend).await; // Attempt to fetch the IP configuration - let ip_config = if let Some(ip) = ip_address { - match IpAddr::from_str(&ip) { - Ok(parsed_ip) => Config::get(ConfigKey::FrontendForIp(parsed_ip)).await.ok(), - Err(_) => { - return Responses::BadRequest(Json(format!("Invalid IP address: {ip}"))); - }, - } + let ip_config = if let Some(ip) = ip_address.0 { + Config::get(ConfigKey::FrontendForIp(ip)).await.ok() } else { None }; @@ -84,7 +69,7 @@ impl ConfigApi { } } - /// Insert or update the frontend configuration. + /// Set the frontend configuration. #[oai( path = "/draft/config/frontend", method = "put", From eef973f854edab1e22cb4e8879c8752eb72f9bdc Mon Sep 17 00:00:00 2001 From: bkioshn Date: Tue, 15 Oct 2024 21:46:00 +0700 Subject: [PATCH 13/37] fix: lazy lock validator and rename function Signed-off-by: bkioshn --- .../bin/src/db/event/config/key.rs | 37 +++++++++++++------ .../bin/src/db/event/config/mod.rs | 4 +- .../bin/src/db/event/config/sql/insert.sql | 3 -- .../bin/src/service/api/config/mod.rs | 14 +++---- 4 files changed, 34 insertions(+), 24 deletions(-) delete mode 100644 catalyst-gateway/bin/src/db/event/config/sql/insert.sql diff --git a/catalyst-gateway/bin/src/db/event/config/key.rs b/catalyst-gateway/bin/src/db/event/config/key.rs index 62ea1bce984..e40601ba31e 100644 --- a/catalyst-gateway/bin/src/db/event/config/key.rs +++ b/catalyst-gateway/bin/src/db/event/config/key.rs @@ -2,6 +2,7 @@ use std::{net::IpAddr, sync::LazyLock}; +use jsonschema::Validator; use serde_json::{json, Value}; use tracing::error; @@ -14,20 +15,35 @@ pub(crate) enum ConfigKey { FrontendForIp(IpAddr), } -/// Frontend schema configuration. -static FRONTEND_SCHEMA: LazyLock = - LazyLock::new(|| load_json_lazy(include_str!("jsonschema/frontend.json"))); +/// Frontend schema validator. +static FRONTEND_SCHEMA_VALIDATOR: LazyLock = + LazyLock::new(|| schema_validator(&load_json_lazy(include_str!("jsonschema/frontend.json")))); /// Frontend default configuration. static FRONTEND_DEFAULT: LazyLock = LazyLock::new(|| load_json_lazy(include_str!("default/frontend.json"))); +/// Helper function to create a JSON validator from a JSON schema. +/// If the schema is invalid, a default JSON validator is created. +fn schema_validator(schema: &Value) -> Validator { + jsonschema::validator_for(schema).unwrap_or_else(|e| { + error!("Error creating JSON validator: {}", e); + + // Create a default JSON validator as a fallback + // This should not fail since it is hard coded + #[allow(clippy::expect_used)] + Validator::new(&json!({ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object" + })) + .expect("Failed to create default JSON validator") + }) +} + /// Helper function to convert a JSON string to a JSON value. fn load_json_lazy(data: &str) -> Value { serde_json::from_str(data).unwrap_or_else(|e| { - // Log the error error!("Error parsing JSON: {}", e); - // And return an empty JSON object json!({}) }) } @@ -45,16 +61,13 @@ impl ConfigKey { /// Validate the provided value against the JSON schema. pub(super) fn validate(&self, value: &Value) -> anyhow::Result<()> { - // Retrieve the schema based on ConfigKey - let schema = match self { - ConfigKey::Frontend | ConfigKey::FrontendForIp(_) => &*FRONTEND_SCHEMA, + // Retrieve the validator based on ConfigKey + let validator = match self { + ConfigKey::Frontend | ConfigKey::FrontendForIp(_) => &*FRONTEND_SCHEMA_VALIDATOR, }; - let validator = jsonschema::validator_for(schema) - .map_err(|e| anyhow::anyhow!("Failed to create JSON validator: {:?}", e))?; - // Validate the value against the schema - anyhow::ensure!(validator.is_valid(value), "Failed schema validation"); + anyhow::ensure!(validator.is_valid(value), "Invalid JSON, Failed schema validation"); Ok(()) } diff --git a/catalyst-gateway/bin/src/db/event/config/mod.rs b/catalyst-gateway/bin/src/db/event/config/mod.rs index 3a408201bba..c931b4596dc 100644 --- a/catalyst-gateway/bin/src/db/event/config/mod.rs +++ b/catalyst-gateway/bin/src/db/event/config/mod.rs @@ -35,8 +35,8 @@ impl Config { } } - /// Update or insert (upsert) configuration for the given `ConfigKey`. - pub(crate) async fn upsert(id: ConfigKey, value: Value) -> anyhow::Result<()> { + /// Set the configuration for the given `ConfigKey`. + pub(crate) async fn set(id: ConfigKey, value: Value) -> anyhow::Result<()> { // Validate the value id.validate(&value)?; diff --git a/catalyst-gateway/bin/src/db/event/config/sql/insert.sql b/catalyst-gateway/bin/src/db/event/config/sql/insert.sql deleted file mode 100644 index bc8973aab08..00000000000 --- a/catalyst-gateway/bin/src/db/event/config/sql/insert.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Insert a new row into the 'config' table -INSERT INTO config (id1, id2, id3, value) -VALUES ($1, $2, $3, $4); -- The values to insert for each column diff --git a/catalyst-gateway/bin/src/service/api/config/mod.rs b/catalyst-gateway/bin/src/service/api/config/mod.rs index fc0410472ed..46df7a58c30 100644 --- a/catalyst-gateway/bin/src/service/api/config/mod.rs +++ b/catalyst-gateway/bin/src/service/api/config/mod.rs @@ -83,11 +83,11 @@ impl ConfigApi { match ip_query.0 { Some(ip) => { match IpAddr::from_str(&ip) { - Ok(parsed_ip) => upsert(ConfigKey::FrontendForIp(parsed_ip), body_value).await, + Ok(parsed_ip) => set(ConfigKey::FrontendForIp(parsed_ip), body_value).await, Err(err) => Responses::BadRequest(Json(format!("Invalid IP address: {err}"))), } }, - None => upsert(ConfigKey::Frontend, body_value).await, + None => set(ConfigKey::Frontend, body_value).await, } } } @@ -111,10 +111,10 @@ fn merge_configs(general: &Value, ip_specific: &Value) -> Value { merged } -/// Helper function to handle upsert. -async fn upsert(key: ConfigKey, value: Value) -> Responses { - match Config::upsert(key, value).await { - Ok(()) => Responses::Ok(Json("Configuration upsert successfully.".to_string())), - Err(err) => Responses::BadRequest(Json(format!("Failed to upsert configuration: {err}"))), +/// Helper function to handle set. +async fn set(key: ConfigKey, value: Value) -> Responses { + match Config::set(key, value).await { + Ok(()) => Responses::Ok(Json("Configuration successfully set.".to_string())), + Err(err) => Responses::BadRequest(Json(format!("Failed to set configuration: {err}"))), } } From 816447e8da7a169b53b601c23097ad3e4fb5444c Mon Sep 17 00:00:00 2001 From: bkioshn Date: Tue, 15 Oct 2024 22:10:59 +0700 Subject: [PATCH 14/37] fix: frontend default and json schema Signed-off-by: bkioshn --- .../src/db/event/config/default/frontend.json | 4 ++- .../db/event/config/jsonschema/frontend.json | 27 +++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/config/default/frontend.json b/catalyst-gateway/bin/src/db/event/config/default/frontend.json index 9e26dfeeb6e..bd8f2ba3856 100644 --- a/catalyst-gateway/bin/src/db/event/config/default/frontend.json +++ b/catalyst-gateway/bin/src/db/event/config/default/frontend.json @@ -1 +1,3 @@ -{} \ No newline at end of file +{ + "sentry_environment": "dev" +} diff --git a/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json b/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json index a555810c341..e8233bb95ae 100644 --- a/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json +++ b/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json @@ -1,4 +1,27 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object" + "$id": "https://www.stephenlewis.me/sentry-schema.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "sentry_dsn": { + "$ref": "#/definitions/saneUrl", + "description": "The Data Source Name (DSN) for Sentry." + }, + "sentry_release": { + "type": "string", + "description": "A version of the code deployed to an environment" + }, + "sentry_environment": { + "type": "string", + "description": "The environment in which the application is running, e.g., 'dev', 'qa'." + } + }, + "additionalProperties": true, + "definitions": { + "saneUrl": { + "type": "string", + "format": "uri", + "pattern": "^https?://" + } + } } From 087cd96bf3af0526ac1ab047aac0e51f681d0cd1 Mon Sep 17 00:00:00 2001 From: bkioshn Date: Tue, 15 Oct 2024 22:12:53 +0700 Subject: [PATCH 15/37] chore:add license MIT Signed-off-by: bkioshn --- catalyst-gateway/LICENSE-MIT | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 catalyst-gateway/LICENSE-MIT diff --git a/catalyst-gateway/LICENSE-MIT b/catalyst-gateway/LICENSE-MIT new file mode 100644 index 00000000000..3a2971efd2e --- /dev/null +++ b/catalyst-gateway/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2023 Input Output (IOG). + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. \ No newline at end of file From 96e2cb538a1d51ba427e77e0b0849494f537c462 Mon Sep 17 00:00:00 2001 From: bkioshn Date: Tue, 15 Oct 2024 22:27:18 +0700 Subject: [PATCH 16/37] fix: remove migration v2 to v9 Signed-off-by: bkioshn --- .../event-db/migrations/V2__event_tables.sql | 154 -------- .../migrations/V3__objective_tables.sql | 146 -------- .../migrations/V4__proposal_tables.sql | 221 ----------- .../event-db/migrations/V5__vote_plan.sql | 94 ----- .../event-db/migrations/V6__registration.sql | 345 ------------------ .../event-db/migrations/V7__vote_tables.sql | 34 -- .../migrations/V8__catalyst_automation.sql | 83 ----- .../migrations/V9__moderation_stage.sql | 45 --- 8 files changed, 1122 deletions(-) delete mode 100644 catalyst-gateway/event-db/migrations/V2__event_tables.sql delete mode 100644 catalyst-gateway/event-db/migrations/V3__objective_tables.sql delete mode 100644 catalyst-gateway/event-db/migrations/V4__proposal_tables.sql delete mode 100644 catalyst-gateway/event-db/migrations/V5__vote_plan.sql delete mode 100644 catalyst-gateway/event-db/migrations/V6__registration.sql delete mode 100644 catalyst-gateway/event-db/migrations/V7__vote_tables.sql delete mode 100644 catalyst-gateway/event-db/migrations/V8__catalyst_automation.sql delete mode 100644 catalyst-gateway/event-db/migrations/V9__moderation_stage.sql diff --git a/catalyst-gateway/event-db/migrations/V2__event_tables.sql b/catalyst-gateway/event-db/migrations/V2__event_tables.sql deleted file mode 100644 index 8c9b5dd8ac3..00000000000 --- a/catalyst-gateway/event-db/migrations/V2__event_tables.sql +++ /dev/null @@ -1,154 +0,0 @@ --- Catalyst Voices Database - Event Data --- sqlfluff:dialect:postgres - --- Title : Event Data - --- `id` mapping. --- * `id` is a UUID, in the past it was an auto incrementing value. --- * it is changed to a UUID so that the data can be generated independently and it is more friendly --- * and simpler for a decentralized or distributed system to safely create UUID than incremental number. --- * However we need compatibility with the rpe-existing incremental numbers. --- * Accordingly we will abuse the UUID format. --- * A V8 UUID will be defined where the low 32 bits are 0s. --- * If the ID is an incremental ID it will be mapped to this special uuid, by or-ing the incremental ID --- * with the mapping UUID, creating a compatible UUID. --- * As ALL autogenerated UUID's will be type 4, there is no possibility of collision. --- --- The Mapping UUID is defined as: `164fba58-31ff-8084-96cb-eb9d00000000` - - --- Event Tables - --- ------------------------------------------------------------------------------------------------- - -CREATE TABLE event_type ( - id UUID PRIMARY KEY DEFAULT GEN_RANDOM_UUID(), - name TEXT NOT NULL, - description_schema UUID NOT NULL, - data_schema UUID NOT NULL, - - FOREIGN KEY (description_schema) REFERENCES json_schema_type ( - id - ) ON DELETE CASCADE, - FOREIGN KEY (data_schema) REFERENCES json_schema_type (id) ON DELETE CASCADE -); - -CREATE UNIQUE INDEX event_type_name_idx ON event_type (name); - -COMMENT ON TABLE event_type IS -'The types of event which have been defined.'; - -COMMENT ON COLUMN event_type.id IS -'Synthetic Unique ID for each event_type (UUIDv4).'; -COMMENT ON COLUMN event_type.name IS -'The name of the event type. -eg. "Catalyst V1"'; -COMMENT ON COLUMN event_type.description_schema IS -'The JSON Schema which defines the structure of the data in the -`description` field in the event record.'; -COMMENT ON COLUMN event_type.data_schema IS -'The JSON Schema which defines the structure of the data in the -`extra_data` field in the event record.'; - --- TODO: Would be better to read the schemas, extract the ID, and add or update new schemas. --- Run as required after migrations. - --- Add Event Schemas to the known schema types. --- INSERT INTO json_schema_type_names (id) --- VALUES --- ('event_description'), -- Event Description schemas --- ('event_data'); -- Event Data Schemas - --- Add the Initial Schemas for events. --- INSERT INTO json_schema_type (id, type, name, schema) --- VALUES ---( --- 'd899cd44-3513-487b-ab46-fdca662a724d', -- From the schema file. --- 'event_description', --- 'multiline_text', --- ( --- SELECT jsonb --- FROM PG_READ_FILE('../json_schemas/event/description/multiline_text.json') --- ) ---), ---( --- '9c5df318-fa9a-4310-80fa-490f46d1cc43', -- From the schema file. --- 'event_data', --- 'catalyst_v1', --- ( --- SELECT jsonb --- FROM PG_READ_FILE('../json_schemas/event/description/catalyst_v1.json') --- ) ---); - --- Define a Catalyst V1 Event. - ---INSERT INTO event_type (name, description_schema, data_schema) ---VALUES ---( --- 'Catalyst V1', --- 'd899cd44-3513-487b-ab46-fdca662a724d', --- '9c5df318-fa9a-4310-80fa-490f46d1cc43' ---); - --- ------------------------------------------------------------------------------------------------- - --- Event Table - Defines each voting or decision event -CREATE TABLE event ( - id UUID PRIMARY KEY DEFAULT GEN_RANDOM_UUID(), - -- The Organizer/Administrator of this event. - -- Update once RBAC is defined, as Organizer is an RBAC Role. - organizer TEXT NOT NULL, - type UUID REFERENCES event_type (id), - name TEXT NOT NULL, - description JSONB NOT NULL, - start_time TIMESTAMP, - backing_start TIMESTAMP, - backing_end TIMESTAMP, - end_time TIMESTAMP, - data JSONB NOT NULL -); - -CREATE UNIQUE INDEX event_name_idx ON event (name); - -COMMENT ON TABLE event IS -'The basic parameters of a related set of funding campaigns.'; - -COMMENT ON COLUMN event.id IS -'Synthetic Unique ID for each event (UUIDv4).'; - -COMMENT ON COLUMN event.organizer IS -'Name of the Event Organizer. -Placeholder, this will need to be replaced -with a reference to an authorized organization.'; - -COMMENT ON COLUMN event.name IS -'The name of the event. -eg. "Fund9" or "SVE1"'; - -COMMENT ON COLUMN event.type IS -'The type of the event.'; - -COMMENT ON COLUMN event.description IS -'A detailed description of the purpose of the event. -Must conform to the JSON Schema defined by -`event_type.description_schema.`'; -COMMENT ON COLUMN event.start_time IS -'The time (UTC) the event starts. -NULL = Not yet defined.'; -COMMENT ON COLUMN event.backing_start IS -'The time (UTC) when backers may start backing the events campaign/s. -This must >= event.start_time. -NULL = Not yet defined.'; -COMMENT ON COLUMN event.backing_end IS -'The time (UTC) when backers may no longer back the events campaign/s. -This must > event.backing_start and <= event.end_time. -NULL = Not yet defined.'; -COMMENT ON COLUMN event.end_time IS -'The time (UTC) the event ends. -Must be >= event.backing_end. -NULL = Not yet defined.'; -COMMENT ON COLUMN event.data IS -'Event Type specific data defined about the event. -Must conform to the JSON Schema defined by -`event_type.extra_data_schema.`'; diff --git a/catalyst-gateway/event-db/migrations/V3__objective_tables.sql b/catalyst-gateway/event-db/migrations/V3__objective_tables.sql deleted file mode 100644 index f24de876399..00000000000 --- a/catalyst-gateway/event-db/migrations/V3__objective_tables.sql +++ /dev/null @@ -1,146 +0,0 @@ --- Catalyst Event Database - --- Title: Objective Tables - --- objective types table - Defines all currently known objectives types. -CREATE TABLE objective_category -( - name TEXT PRIMARY KEY, - description TEXT -); - -COMMENT ON TABLE objective_category IS 'Defines all known and valid objective categories.'; -COMMENT ON COLUMN objective_category.name IS 'The name of this objective category.'; -COMMENT ON COLUMN objective_category.description IS 'A Description of this kind of objective category.'; - --- Define known objective categories -INSERT INTO objective_category (name, description) -VALUES -('catalyst-simple', 'A Simple choice'), -('catalyst-native', '??'), -('catalyst-community-choice', 'Community collective decision'), -('sve-decision', 'Special voting event decision'); - --- known currencies - Defines all currently known currencies. -CREATE TABLE currency -( - name TEXT PRIMARY KEY, - description TEXT -); - -COMMENT ON TABLE currency IS 'Defines all known and valid currencies.'; -COMMENT ON COLUMN currency.name IS 'The name of this currency type.'; -COMMENT ON COLUMN currency.description IS 'A Description of this kind of currency type.'; - - --- Define known currencies -INSERT INTO currency (name, description) -VALUES -( - 'USD_ADA', - 'US Dollars, converted to Cardano ADA at time of reward calculation.' -), -('ADA', 'Cardano ADA.'), -('CLAP', 'CLAP tokens.'), -('COTI', 'COTI tokens.'); - --- known vote options - Defines all currently known vote options. -CREATE TABLE vote_options -( - id SERIAL PRIMARY KEY, - - idea_scale TEXT ARRAY UNIQUE, - objective TEXT ARRAY UNIQUE -); - -COMMENT ON TABLE vote_options IS 'Defines all known vote plan option types.'; -COMMENT ON COLUMN vote_options.id IS 'Unique ID for each possible option set.'; -COMMENT ON COLUMN vote_options.idea_scale IS 'How this vote option is represented in idea scale.'; -COMMENT ON COLUMN vote_options.objective IS 'How the vote options is represented in the objective.'; - --- Define known vote_options -INSERT INTO vote_options (idea_scale, objective) -VALUES -('{"blank", "yes", "no"}', '{"yes", "no"}'); - - - --- goals - -CREATE TABLE goal -( - id SERIAL PRIMARY KEY, - event_id UUID NOT NULL, - - idx INTEGER NOT NULL, - name VARCHAR NOT NULL, - - FOREIGN KEY (event_id) REFERENCES event (id) ON DELETE CASCADE -); - -CREATE UNIQUE INDEX goal_index ON goal (event_id, idx); - -COMMENT ON TABLE goal IS 'The list of campaign goals for this event.'; -COMMENT ON COLUMN goal.id IS 'Synthetic Unique Key.'; -COMMENT ON COLUMN goal.idx IS 'The index specifying the order/priority of the goals.'; -COMMENT ON COLUMN goal.name IS 'The description of this event goal.'; -COMMENT ON COLUMN goal.event_id IS 'The ID of the event this goal belongs to.'; -COMMENT ON INDEX goal_index IS 'An index to enforce uniqueness of the relative `idx` field per event.'; - - --- objective table - Defines all objectives for all known funds. - - -CREATE TABLE objective -( - row_id SERIAL PRIMARY KEY, - - id INTEGER NOT NULL, - event UUID NOT NULL, - - category TEXT NOT NULL, - title TEXT NOT NULL, - description TEXT NOT NULL, - - rewards_currency TEXT, - rewards_total BIGINT, - rewards_total_lovelace BIGINT, - proposers_rewards BIGINT, - vote_options INTEGER, - - extra JSONB, - - FOREIGN KEY (event) REFERENCES event (id) ON DELETE CASCADE, - FOREIGN KEY (category) REFERENCES objective_category (name) ON DELETE CASCADE, - FOREIGN KEY (rewards_currency) REFERENCES currency (name) ON DELETE CASCADE, - FOREIGN KEY (vote_options) REFERENCES vote_options (id) ON DELETE CASCADE -); - -CREATE UNIQUE INDEX objective_idx ON objective (id, event); - -COMMENT ON TABLE objective IS -'All objectives for all events. -A objective is a group category for selection in an event.'; -COMMENT ON COLUMN objective.row_id IS 'Synthetic Unique Key'; -COMMENT ON COLUMN objective.id IS -'Event specific objective ID. -Can be non-unique between events (Eg, Ideascale ID for objective).'; -COMMENT ON COLUMN objective.event IS 'The specific Event ID this objective is part of.'; -COMMENT ON COLUMN objective.category IS -'What category of objective is this. -See the objective_category table for allowed values.'; -COMMENT ON COLUMN objective.title IS 'The title of the objective.'; -COMMENT ON COLUMN objective.description IS 'Long form description of the objective.'; - -COMMENT ON COLUMN objective.rewards_currency IS 'The currency rewards values are represented as.'; -COMMENT ON COLUMN objective.rewards_total IS -'The total reward pool to pay on this objective to winning proposals. In the Objective Currency.'; -COMMENT ON COLUMN objective.rewards_total_lovelace IS -'The total reward pool to pay on this objective to winning proposals. In Lovelace.'; -COMMENT ON COLUMN objective.proposers_rewards IS 'Not sure how this is different from rewards_total???'; -COMMENT ON COLUMN objective.vote_options IS 'The Vote Options applicable to all proposals in this objective.'; -COMMENT ON COLUMN objective.extra IS -'Extra Data for this objective represented as JSON. -"url"."objective" is a URL for more info about the objective. -"highlights" is ??? -'; diff --git a/catalyst-gateway/event-db/migrations/V4__proposal_tables.sql b/catalyst-gateway/event-db/migrations/V4__proposal_tables.sql deleted file mode 100644 index d53ee814a88..00000000000 --- a/catalyst-gateway/event-db/migrations/V4__proposal_tables.sql +++ /dev/null @@ -1,221 +0,0 @@ --- Catalyst Event Database - --- Title : Proposals - --- Proposals Table - -CREATE TABLE proposal -( - row_id SERIAL PRIMARY KEY, - id INTEGER NOT NULL, - objective INTEGER NOT NULL, - title TEXT NOT NULL, - summary TEXT NOT NULL, - public_key TEXT NOT NULL, - funds BIGINT NOT NULL, - url TEXT NOT NULL, - files_url TEXT NOT NULL, - impact_score BIGINT NOT NULL, - - extra JSONB, - - proposer_name TEXT NOT NULL, - proposer_contact TEXT NOT NULL, - proposer_url TEXT NOT NULL, - proposer_relevant_experience TEXT NOT NULL, - bb_proposal_id BYTEA, - - FOREIGN KEY (objective) REFERENCES objective (row_id) ON DELETE CASCADE - --FOREIGN KEY (bb_vote_options) REFERENCES vote_options ( - -- objective - --) ON DELETE CASCADE -); - -CREATE UNIQUE INDEX proposal_index ON proposal (id, objective); - - -COMMENT ON TABLE proposal IS 'All Proposals for the current fund.'; -COMMENT ON COLUMN proposal.row_id IS 'Synthetic Unique Key'; -COMMENT ON COLUMN proposal.id IS 'Actual Proposal Unique ID'; -COMMENT ON COLUMN proposal.objective IS 'The Objective this proposal falls under.'; -COMMENT ON COLUMN proposal.title IS 'Brief title of the proposal.'; -COMMENT ON COLUMN proposal.summary IS 'A Summary of the proposal to be implemented.'; -COMMENT ON COLUMN proposal.public_key IS 'Proposals Reward Address (CIP-19 Payment Key)'; -COMMENT ON COLUMN proposal.funds IS 'How much funds (in the currency of the fund)'; -COMMENT ON COLUMN proposal.url IS 'A URL with supporting information for the proposal.'; -COMMENT ON COLUMN proposal.files_url IS 'A URL link to relevant files supporting the proposal.'; -COMMENT ON COLUMN proposal.impact_score IS 'The Impact score assigned to this proposal by the Assessors.'; -COMMENT ON COLUMN proposal.proposer_name IS 'The proposers name.'; -COMMENT ON COLUMN proposal.proposer_contact IS 'Contact details for the proposer.'; -COMMENT ON COLUMN proposal.proposer_url IS 'A URL with details of the proposer.'; -COMMENT ON COLUMN proposal.proposer_relevant_experience IS -'A freeform string describing the proposers experience relating to their capability to implement the proposal.'; -COMMENT ON COLUMN proposal.bb_proposal_id IS -'The ID used by the voting ledger (bulletin board) to refer to this proposal.'; -COMMENT ON COLUMN proposal.extra IS -'Extra data about the proposal. - The types of extra data are defined by the proposal type and are not enforced. - Extra Fields for `native` challenges: - NONE. - - Extra Fields for `simple` challenges: - "problem" : - Statement of the problem the proposal tries to address. - "solution" : - The Solution to the challenge. - - Extra Fields for `community choice` challenge: - "brief" : - Brief explanation of a proposal. - "importance" : - The importance of the proposal. - "goal" : - The goal of the proposal is addressed to meet. - "metrics" : - The metrics of the proposal or how success will be determined.'; - --- Reviewer's levels table - -CREATE TABLE reviewer_level ( - row_id SERIAL PRIMARY KEY, - name TEXT NOT NULL, - total_reward_pct NUMERIC(6, 3) CONSTRAINT percentage CHECK ( - total_reward_pct <= 100 AND total_reward_pct >= 0 - ), - - event_id UUID NOT NULL, - - FOREIGN KEY (event_id) REFERENCES event (id) ON DELETE CASCADE -); - -COMMENT ON TABLE reviewer_level IS -'All levels of reviewers. -This table represents all different types of reviewer`s levels, which is taken into account during rewarding process.'; -COMMENT ON COLUMN reviewer_level.row_id IS 'Synthetic Unique Key'; -COMMENT ON COLUMN reviewer_level.name IS 'Name of the reviewer level'; -COMMENT ON COLUMN reviewer_level.total_reward_pct IS -'Total reviewer`s reward assigned to the specific level, which is defined as a percentage from the -total pot of Community Review rewards (See `event.review_rewards` column).'; -COMMENT ON COLUMN reviewer_level.event_id IS 'The specific Event ID this review level is part of.'; - --- community advisor reviews - --- I feel like these ratings and notes should be in a general json field to --- suit adaptability without needing schema changes. - -CREATE TABLE proposal_review ( - row_id SERIAL PRIMARY KEY, - proposal_id INTEGER NOT NULL, - assessor VARCHAR NOT NULL, - assessor_level INTEGER, - reward_address TEXT, - - flags JSONB NULL, - - FOREIGN KEY (proposal_id) REFERENCES proposal (row_id) ON DELETE CASCADE, - FOREIGN KEY (assessor_level) REFERENCES reviewer_level ( - row_id - ) ON DELETE CASCADE -); - -COMMENT ON TABLE proposal_review IS 'All Reviews.'; -COMMENT ON COLUMN proposal_review.row_id IS 'Synthetic Unique Key.'; -COMMENT ON COLUMN proposal_review.proposal_id IS 'The Proposal this review is for.'; -COMMENT ON COLUMN proposal_review.assessor IS 'Assessors Anonymized ID'; -COMMENT ON COLUMN proposal_review.assessor_level IS 'Assessors level ID'; -COMMENT ON COLUMN proposal_review.reward_address IS 'Assessors reward address'; - -COMMENT ON COLUMN proposal_review.flags IS -'OPTIONAL: JSON Array which defines the flags raised for this review. -Flags can be raised for different reasons and have different metadata. -Each entry = -```jsonc -{ - "flag_type": "", // Enum of the flag type (0: Profanity, 1: Similarity 2: AI generated). - "score": , // Profanity score, similarity score, or AI generated score. 0-1. - "related_reviews": [] // Array of review IDs that this flag is related to (valid for similarity). -} -``` -'; - -CREATE TABLE review_metric ( - row_id SERIAL PRIMARY KEY, - name VARCHAR NOT NULL, - description VARCHAR NULL, - min INTEGER NOT NULL, - max INTEGER NOT NULL, - map JSONB ARRAY NULL -); -COMMENT ON TABLE review_metric IS 'Definition of all possible review metrics.'; -COMMENT ON COLUMN review_metric.row_id IS 'The synthetic ID of this metric.'; -COMMENT ON COLUMN review_metric.name IS 'The short name for this review metric.'; -COMMENT ON COLUMN review_metric.description IS 'Long form description of what the review metric means.'; -COMMENT ON COLUMN review_metric.min IS 'The minimum value of the metric (inclusive).'; -COMMENT ON COLUMN review_metric.max IS 'The maximum value of the metric (inclusive).'; -COMMENT ON COLUMN review_metric.map IS -'OPTIONAL: JSON Array which defines extra details for each metric score. -There MUST be one entry per possible score in the range. -Entries are IN ORDER, from the lowest numeric score to the highest. -Each entry = -```jsonc -{ - "name" : "", // Symbolic Name for the metric score. - "description" : "", // Description of what the named metric score means. -} -``` -'; - --- Define known review metrics -INSERT INTO review_metric (name, description, min, max, map) -VALUES -('impact', 'Impact Rating', 0, 5, null), -('feasibility', 'Feasibility Rating', 0, 5, null), -('auditability', 'Auditability Rating', 0, 5, null), -('value', 'Value Proposition Rating', 0, 5, null), -('vpa_ranking', 'VPA Ranking of the review', 0, 3, ARRAY[ - '{"name":"Excellent","desc":"Excellent Review"}', - '{"name":"Good","desc":"Could be improved."}', - '{"name":"FilteredOut","desc":"Exclude this review"}', - '{"name":"NA", "desc":"Not Applicable"}' -]::JSON []); - -CREATE TABLE objective_review_metric ( - row_id SERIAL PRIMARY KEY, - objective INTEGER NOT NULL, - metric INTEGER NOT NULL, - note BOOLEAN, - review_group VARCHAR, - - UNIQUE (objective, metric, review_group), - - FOREIGN KEY (objective) REFERENCES objective (row_id) ON DELETE CASCADE, - FOREIGN KEY (metric) REFERENCES review_metric (row_id) ON DELETE CASCADE -); - - -COMMENT ON TABLE objective_review_metric IS 'All valid metrics for reviews on an objective.'; -COMMENT ON COLUMN objective_review_metric.objective IS 'The objective that can use this review metric.'; -COMMENT ON COLUMN objective_review_metric.metric IS 'The review metric that the objective can use.'; -COMMENT ON COLUMN objective_review_metric.note IS -'Does the metric require a Note? -NULL = Optional. -True = MUST include Note. -False = MUST NOT include Note.'; -COMMENT ON COLUMN objective_review_metric.review_group IS 'The review group that can give this metric. Details TBD.'; - - - -CREATE TABLE review_rating ( - row_id SERIAL PRIMARY KEY, - review_id INTEGER NOT NULL, - metric INTEGER NOT NULL, - rating INTEGER NOT NULL, - note VARCHAR, - - UNIQUE (review_id, metric), - - FOREIGN KEY (review_id) REFERENCES proposal_review (row_id) ON DELETE CASCADE, - FOREIGN KEY (metric) REFERENCES review_metric (row_id) ON DELETE CASCADE -); - - -COMMENT ON TABLE review_rating IS 'An Individual rating for a metric given on a review.'; -COMMENT ON COLUMN review_rating.row_id IS 'Synthetic ID of this individual rating.'; -COMMENT ON COLUMN review_rating.review_id IS 'The review the metric is being given for.'; -COMMENT ON COLUMN review_rating.metric IS 'Metric the rating is being given for.'; -COMMENT ON COLUMN review_rating.rating IS 'The rating being given to the metric.'; -COMMENT ON COLUMN review_rating.note IS 'OPTIONAL: Note about the rating given.'; diff --git a/catalyst-gateway/event-db/migrations/V5__vote_plan.sql b/catalyst-gateway/event-db/migrations/V5__vote_plan.sql deleted file mode 100644 index 20d1b6120ec..00000000000 --- a/catalyst-gateway/event-db/migrations/V5__vote_plan.sql +++ /dev/null @@ -1,94 +0,0 @@ --- Catalyst Event Database - --- Title : Vote Plan - --- Vote Plan Categories - -CREATE TABLE voteplan_category -( - name TEXT PRIMARY KEY, - public_key BOOL -); - - -INSERT INTO voteplan_category (name, public_key) -VALUES -('public', false), -- Fully public votes only -('private', true), -- Fully private votes only. -('cast-private', true); -- Private until tally, then decrypted. - -COMMENT ON TABLE voteplan_category IS 'The category of vote plan currently supported.'; -COMMENT ON COLUMN voteplan_category.name IS 'The UNIQUE name of this voteplan category.'; -COMMENT ON COLUMN voteplan_category.public_key IS 'Does this vote plan category require a public key.'; - - --- groups - -CREATE TABLE voting_group ( - name TEXT PRIMARY KEY -); - -INSERT INTO voting_group (name) -VALUES -('direct'), -- Direct Voters -('rep'); -- Delegated Voter (Check what is the real name for this group we already use in snapshot) - -COMMENT ON TABLE voting_group IS 'All Groups.'; -COMMENT ON COLUMN voting_group.name IS 'The ID of this voting group.'; - --- Vote Plans - -CREATE TABLE voteplan -( - row_id SERIAL PRIMARY KEY, - objective_id INTEGER NOT NULL, - - id VARCHAR NOT NULL, - category TEXT NOT NULL, - encryption_key VARCHAR, - group_id TEXT, - token_id TEXT, - - FOREIGN KEY (objective_id) REFERENCES objective (row_id) ON DELETE CASCADE, - FOREIGN KEY (category) REFERENCES voteplan_category (name) ON DELETE CASCADE, - FOREIGN KEY (group_id) REFERENCES voting_group (name) ON DELETE CASCADE -); - -COMMENT ON TABLE voteplan IS 'All Vote plans.'; - -COMMENT ON COLUMN voteplan.row_id IS 'Synthetic Unique Key'; -COMMENT ON COLUMN voteplan.id IS -'The ID of the Vote plan in the voting ledger/bulletin board. -A Binary value encoded as hex.'; -COMMENT ON COLUMN voteplan.category IS 'The kind of vote which can be cast on this vote plan.'; -COMMENT ON COLUMN voteplan.encryption_key IS -'The public encryption key used. -ONLY if required by the voteplan category.'; -COMMENT ON COLUMN voteplan.group_id IS 'The identifier of voting power token used withing this plan.'; - --- Table to link Proposals to Vote plans in a many-many relationship. --- This Many-Many relationship arises because: --- in the vote ledger/bulletin board, --- one proposal may be within multiple different vote plans, --- and each voteplan can contain multiple proposals. -CREATE TABLE proposal_voteplan -( - row_id SERIAL PRIMARY KEY, - proposal_id INTEGER, - voteplan_id INTEGER, - bb_proposal_index BIGINT, - - FOREIGN KEY (proposal_id) REFERENCES proposal (row_id) ON DELETE CASCADE, - FOREIGN KEY (voteplan_id) REFERENCES voteplan (row_id) ON DELETE CASCADE -); - -CREATE UNIQUE INDEX proposal_voteplan_idx ON proposal_voteplan ( - proposal_id, voteplan_id, bb_proposal_index -); - -COMMENT ON TABLE proposal_voteplan IS 'Table to link Proposals to Vote plans in a Many to Many relationship.'; -COMMENT ON COLUMN proposal_voteplan.row_id IS 'Synthetic ID of this Voteplan/Proposal M-M relationship.'; -COMMENT ON COLUMN proposal_voteplan.proposal_id IS 'The link to the Proposal primary key that links to this voteplan.'; -COMMENT ON COLUMN proposal_voteplan.voteplan_id IS 'The link to the Voteplan primary key that links to this proposal.'; -COMMENT ON COLUMN proposal_voteplan.bb_proposal_index IS -'The Index with the voteplan used by the voting ledger/bulletin board that references this proposal.'; diff --git a/catalyst-gateway/event-db/migrations/V6__registration.sql b/catalyst-gateway/event-db/migrations/V6__registration.sql deleted file mode 100644 index 71f5cc62df3..00000000000 --- a/catalyst-gateway/event-db/migrations/V6__registration.sql +++ /dev/null @@ -1,345 +0,0 @@ --- Catalyst Voices Database - Role Registration Data --- sqlfluff:dialect:postgres - --- Title : Role Registration Data - --- Configuration Tables - --- ------------------------------------------------------------------------------------------------- - --- Slot Index Table -CREATE TABLE cardano_slot_index ( - slot_no BIGINT NOT NULL, - network TEXT NOT NULL, - epoch_no BIGINT NOT NULL, - block_time TIMESTAMP WITH TIME ZONE NOT NULL, - block_hash BYTEA NOT NULL CHECK (LENGTH(block_hash) = 32), - - PRIMARY KEY (slot_no, network) -); - -CREATE INDEX cardano_slot_index_time_idx ON cardano_slot_index (block_time, network); -COMMENT ON INDEX cardano_slot_index_time_idx IS -'Index to allow us to efficiently lookup a slot by time for a particular network.'; - -CREATE INDEX cardano_slot_index_epoch_idx ON cardano_slot_index (epoch_no, network); -COMMENT ON INDEX cardano_slot_index_epoch_idx IS -'Index to allow us to efficiently lookup a slot by epoch for a particular network.'; - -COMMENT ON TABLE cardano_slot_index IS -'This is an index of cardano blockchain slots. -It allows us to quickly find data about every block in the cardano network. -This data is created when each block is first seen.'; - -COMMENT ON COLUMN cardano_slot_index.slot_no IS -'The slot number of the block. -This is the first half of the Primary key.'; -COMMENT ON COLUMN cardano_slot_index.network IS -'The Cardano network for this slot. -This is the second half of the primary key, as each network could use th same slot numbers.'; -COMMENT ON COLUMN cardano_slot_index.epoch_no IS -'The epoch number the slot appeared in.'; -COMMENT ON COLUMN cardano_slot_index.block_time IS -'The time of the slot/block.'; -COMMENT ON COLUMN cardano_slot_index.block_hash IS -'The hash of the block.'; - - --- ------------------------------------------------------------------------------------------------- - --- Transaction Index Table -CREATE TABLE cardano_txn_index ( - id BYTEA NOT NULL PRIMARY KEY CHECK (LENGTH(id) = 32), - - slot_no BIGINT NULL, - network TEXT NOT NULL, - - FOREIGN KEY (slot_no, network) REFERENCES cardano_slot_index (slot_no, network) -); - -CREATE INDEX cardano_txn_index_idx ON cardano_txn_index (id, network); - -COMMENT ON INDEX cardano_txn_index_idx IS -'Index to allow us to efficiently get the slot a particular transaction is in.'; - -COMMENT ON TABLE cardano_txn_index IS -'This is an index of all transactions in the cardano network. -It allows us to quickly find a transaction by its id, and its slot number.'; - -COMMENT ON COLUMN cardano_txn_index.id IS -'The ID of the transaction. -This is a 32 Byte Hash.'; -COMMENT ON COLUMN cardano_txn_index.network IS -'The Cardano network for this transaction. -This is the second half of the primary key, as each network could use the same transactions.'; -COMMENT ON COLUMN cardano_txn_index.slot_no IS -'The slot number the transaction appeared in. -If this is NULL, then the Transaction is no longer in a known slot, due to a rollback. -Such transactions should be considered invalid until the appear in a new slot. -We only need to index transactions we care about and not every single transaction in the cardano network.'; - - --- ------------------------------------------------------------------------------------------------- - --- cardano update state table. --- Keeps a record of each update to the Role Registration state data. --- Used internally to track the updates to the database. -CREATE TABLE cardano_update_state ( - - id BIGSERIAL PRIMARY KEY, - - started TIMESTAMP WITH TIME ZONE NOT NULL, - ended TIMESTAMP WITH TIME ZONE NOT NULL, - updater_id TEXT NOT NULL, - - slot_no BIGINT NOT NULL, - network TEXT NOT NULL, - block_hash BYTEA NOT NULL CHECK (LENGTH(block_hash) = 32), - - update BOOLEAN, - rollback BOOLEAN, - - stats JSONB, - - FOREIGN KEY (slot_no, network) REFERENCES cardano_slot_index (slot_no, network) -); - - -CREATE INDEX cardano_update_state_idx ON cardano_update_state (id, network); - -COMMENT ON INDEX cardano_update_state_idx IS -'Index to allow us to efficiently get find an update state record by its id. -This index can be used to find the latest state record for a particular network.'; - -CREATE INDEX cardano_update_state_time_idx ON cardano_update_state ( - started, network -); - -COMMENT ON INDEX cardano_update_state_time_idx IS -'Index to allow us to efficiently get find an update state record by time for a particular network.'; - - -COMMENT ON TABLE cardano_update_state IS -'A record of the updates to the Cardano Registration data state. -Every time the state is updated, a new record is created. -On update, an updating node, must check if the slot it is updating already exists. -If it does, it checks if the indexed block is the same (same hash). -If it is, it sets `update` to false, and just saves its update state with no further action. -This allows us to run multiple followers and update the database simultaneously. - -IF the hash is different, then we need to handle that, logic not yet defined... - -This table also serves as a central lock source for updates to the registration state -which must be atomic. -Should be accessed with a pattern like: - -```sql - BEGIN; - LOCK TABLE cardano_update_state IN ACCESS EXCLUSIVE MODE; - -- Read state, update any other tables as needed - INSERT INTO cardano_update_state SET ...; -- Set latest state - COMMIT; -``` -'; - -COMMENT ON COLUMN cardano_update_state.id IS -'Sequential ID of successive updates to the registration state data.'; -COMMENT ON COLUMN cardano_update_state.started IS -'The time the update started for this network.'; -COMMENT ON COLUMN cardano_update_state.ended IS -'The time the update was complete for this network.'; -COMMENT ON COLUMN cardano_update_state.updater_id IS -'The ID of the node which performed the update. -This helps us track which instance of the backend did which updates.'; -COMMENT ON COLUMN cardano_update_state.slot_no IS -'The slot_no this update was run for.'; -COMMENT ON COLUMN cardano_update_state.network IS -'The Cardano network that was updated. -As networks are independent and updates are event driven, only one network -will be updated at a time.'; -COMMENT ON COLUMN cardano_update_state.update IS -'True when this update updated any other tables. -False when a duplicate update was detected.'; -COMMENT ON COLUMN cardano_update_state.rollback IS -'True when this update is as a result of a rollback on-chain. -False when its a normal consecutive update.'; -COMMENT ON COLUMN cardano_update_state.stats IS -'A JSON stats record containing extra data about this update. -Must conform to Schema: - `catalyst_schema://0f917b13-afac-40d2-8263-b17ca8219914/registration/update_stats`.'; - - --- ------------------------------------------------------------------------------------------------- - --- UTXO Table -- Unspent + Staked TX Outputs --- Populated from the transactions in each block -CREATE TABLE cardano_utxo ( - tx_id BYTEA NOT NULL REFERENCES cardano_txn_index (id), - index INTEGER NOT NULL CHECK (index >= 0), - - value BIGINT NOT NULL, - asset JSONB NULL, - - stake_credential BYTEA NULL, - - spent_tx_id BYTEA NULL REFERENCES cardano_txn_index (id), - - PRIMARY KEY (tx_id, index) -); - - -COMMENT ON TABLE cardano_utxo IS -'This table holds all UTXOs for any transaction which is tied to a stake address. -This data allows us to calculate staked ADA at any particular instant in time.'; - -COMMENT ON COLUMN cardano_utxo.tx_id IS -'The ID of the transaction containing the UTXO. -32 Byte Hash.'; -COMMENT ON COLUMN cardano_utxo.index IS -'The index of the UTXO within the transaction.'; - -COMMENT ON COLUMN cardano_utxo.value IS -'The value of the UTXO, in Lovelace if the asset is not defined.'; -COMMENT ON COLUMN cardano_utxo.asset IS -'The asset of the UTXO, if any. -NULL = Ada/Lovelace.'; - -COMMENT ON COLUMN cardano_utxo.stake_credential IS -'The stake credential of the address which owns the UTXO.'; - -COMMENT ON COLUMN cardano_utxo.spent_tx_id IS -'The ID of the transaction which Spent the TX Output. -If we consider this UTXO Spent will depend on when it was spent.'; - - --- ------------------------------------------------------------------------------------------------- - --- Rewards Table -- Earned Rewards -CREATE TABLE cardano_reward ( - slot_no BIGINT NOT NULL, -- First slot of the epoch following the epoch the rewards were earned for. - network TEXT NOT NULL, - stake_credential BYTEA NOT NULL, - - earned_epoch_no BIGINT NOT NULL, - - value BIGINT NOT NULL, - - PRIMARY KEY (slot_no, network, stake_credential), - FOREIGN KEY (slot_no, network) REFERENCES cardano_slot_index (slot_no, network) -); - -CREATE INDEX cardano_rewards_stake_credential_idx ON cardano_reward ( - stake_credential, slot_no -); - -COMMENT ON INDEX cardano_rewards_stake_credential_idx IS -'Index to allow us to efficiently lookup a set of Rewards by stake credential relative to a slot_no.'; - -COMMENT ON TABLE cardano_reward IS -'This table holds all earned rewards per stake address. -It is possible for a Stake Address to earn multiple rewards in the same epoch. -This record contains the Total of all rewards earned in the relevant epoch. -This data structure is preliminary pending the exact method of determining -the rewards earned by any particular stake address.'; - -COMMENT ON COLUMN cardano_reward.slot_no IS -'The slot number the rewards were earned for. -This is the first slot of the epoch following the epoch the rewards were earned for.'; -COMMENT ON COLUMN cardano_reward.network IS -'The Cardano network for this rewards.'; -COMMENT ON COLUMN cardano_reward.stake_credential IS -'The stake credential of the address who earned the rewards.'; -COMMENT ON COLUMN cardano_reward.earned_epoch_no IS -'The epoch number the rewards were earned for.'; -COMMENT ON COLUMN cardano_reward.value IS -'The value of the reward earned, in Lovelace'; - --- ------------------------------------------------------------------------------------------------- - --- Withdrawn Rewards Table -- Withdrawn Rewards -CREATE TABLE cardano_withdrawn_reward ( - slot_no BIGINT NOT NULL, - network TEXT NOT NULL, - stake_credential BYTEA NOT NULL, - value BIGINT NOT NULL, - - PRIMARY KEY (slot_no, network), - FOREIGN KEY (slot_no, network) REFERENCES cardano_slot_index (slot_no, network) -); - -COMMENT ON TABLE cardano_withdrawn_reward IS -'This table holds all withdrawn rewards data. -This makes it possible to accurately calculate the rewards which are still available for a specific Stake Address .'; - -COMMENT ON COLUMN cardano_withdrawn_reward.slot_no IS -'The slot number the rewards were withdrawn for.'; -COMMENT ON COLUMN cardano_withdrawn_reward.network IS -'The Cardano network this withdrawal occurred on.'; -COMMENT ON COLUMN cardano_withdrawn_reward.stake_credential IS -'The stake credential of the address who earned the rewards.'; -COMMENT ON COLUMN cardano_withdrawn_reward.value IS -'The value of the reward withdrawn, in Lovelace'; - --- ------------------------------------------------------------------------------------------------- - --- Cardano Voter Registrations Table -- Voter Registrations -CREATE TABLE cardano_voter_registration ( - tx_id BYTEA PRIMARY KEY NOT NULL REFERENCES cardano_txn_index (id), - - stake_credential BYTEA NULL, - public_voting_key BYTEA NULL, - payment_address BYTEA NULL, - nonce BIGINT NULL, - - metadata_cip36 BYTEA NULL, -- We can purge metadata for valid registrations that are old to save storage space. - - valid BOOLEAN NOT NULL DEFAULT false, - stats JSONB - -- record rolled back in stats if the registration was lost during a rollback, its also invalid at this point. - -- Other stats we can record are is it a CIP-36 or CIP-15 registration format. - -- does it have a valid reward address but not a payment address, so we can't pay to it. - -- other flags about why the registration was invalid. - -- other flags about statistical data (if any). -); - -CREATE INDEX cardano_voter_registration_stake_credential_idx ON cardano_voter_registration ( - stake_credential, nonce, valid -); -COMMENT ON INDEX cardano_voter_registration_stake_credential_idx IS -'Optimize lookups for "stake_credential" or "stake_credential"+"nonce" or "stake_credential"+"nonce"+"valid".'; - -CREATE INDEX cardano_voter_registration_voting_key_idx ON cardano_voter_registration ( - public_voting_key, nonce, valid -); -COMMENT ON INDEX cardano_voter_registration_voting_key_idx IS -'Optimize lookups for "public_voting_key" or "public_voting_key"+"nonce" or "public_voting_key"+"nonce"+"valid".'; - -COMMENT ON TABLE cardano_voter_registration IS -'All CIP15/36 Voter Registrations that are on-chain. -This tables stores all found registrations, even if they are invalid, or have been rolled back.'; - -COMMENT ON COLUMN cardano_voter_registration.tx_id IS -'The Transaction hash of the Transaction holding the registration metadata. -This is used as the Primary Key because it is immutable in the face of potential rollbacks.'; - -COMMENT ON COLUMN cardano_voter_registration.stake_credential IS -'The stake credential of the address who registered.'; -COMMENT ON COLUMN cardano_voter_registration.public_voting_key IS -'The public voting key of the address who registered.'; -COMMENT ON COLUMN cardano_voter_registration.payment_address IS -'The payment address where any voter rewards associated with this registration will be sent.'; -COMMENT ON COLUMN cardano_voter_registration.nonce IS -'The nonce of the registration. Registrations for the same stake address with higher nonces have priority.'; - -COMMENT ON COLUMN cardano_voter_registration.metadata_cip36 IS -'The raw metadata for the CIP-15/36 registration. -This data is optional, a parameter in config specifies how long raw registration metadata should be kept. -Outside this time, the Registration record will be kept, but the raw metadata will be purged.'; - -COMMENT ON COLUMN cardano_voter_registration.valid IS -'True if the registration is valid, false if the registration is invalid. -`stats` can be checked to determine WHY the registration is considered invalid.'; -COMMENT ON COLUMN cardano_voter_registration.stats IS -'Statistical information about the registration. -Must conform to Schema: - `catalyst_schema://fd5a2f8f-afb4-4cf7-ae6b-b7a370c85c82/registration/cip36_stats`.'; diff --git a/catalyst-gateway/event-db/migrations/V7__vote_tables.sql b/catalyst-gateway/event-db/migrations/V7__vote_tables.sql deleted file mode 100644 index 8ef0f96e97a..00000000000 --- a/catalyst-gateway/event-db/migrations/V7__vote_tables.sql +++ /dev/null @@ -1,34 +0,0 @@ --- Catalyst Event Database - Vote Storage - --- Title : Vote Storage - --- vote storage (replicates on-chain data for easy querying) - -CREATE TABLE ballot ( - row_id SERIAL8 PRIMARY KEY, - objective INTEGER NOT NULL, - proposal INTEGER NULL, - - voter BYTEA NOT NULL, - fragment_id TEXT NOT NULL, - cast_at TIMESTAMP NOT NULL, - choice SMALLINT NULL, - raw_fragment BYTEA NOT NULL, - - FOREIGN KEY (objective) REFERENCES objective (row_id) ON DELETE CASCADE, - FOREIGN KEY (proposal) REFERENCES proposal (row_id) ON DELETE CASCADE -); - -CREATE UNIQUE INDEX ballot_proposal_idx ON ballot (proposal, fragment_id); -CREATE UNIQUE INDEX ballot_objective_idx ON ballot (objective, fragment_id); - -COMMENT ON TABLE ballot IS 'All Ballots cast on an event.'; -COMMENT ON COLUMN ballot.fragment_id IS 'Unique ID of this Ballot'; -COMMENT ON COLUMN ballot.voter IS 'Voters Voting Key who cast the ballot'; -COMMENT ON COLUMN ballot.objective IS 'Reference to the Objective the ballot was for.'; -COMMENT ON COLUMN ballot.proposal IS -'Reference to the Proposal the ballot was for. -May be NULL if this ballot covers ALL proposals in the challenge.'; -COMMENT ON COLUMN ballot.cast_at IS 'When this ballot was recorded as properly cast'; -COMMENT ON COLUMN ballot.choice IS 'If a public vote, the choice on the ballot, otherwise NULL.'; -COMMENT ON COLUMN ballot.raw_fragment IS 'The raw ballot record.'; diff --git a/catalyst-gateway/event-db/migrations/V8__catalyst_automation.sql b/catalyst-gateway/event-db/migrations/V8__catalyst_automation.sql deleted file mode 100644 index 53c42e4fa82..00000000000 --- a/catalyst-gateway/event-db/migrations/V8__catalyst_automation.sql +++ /dev/null @@ -1,83 +0,0 @@ --- Catalyst Event Database - --- Title : Catalyst Automation - --- Voting Nodes Table - Defines nodes in the network --- This table is looked up by hostname and event -CREATE TABLE voting_node ( - hostname TEXT NOT NULL, - event UUID NOT NULL, - - pubkey TEXT NOT NULL, - seckey TEXT NOT NULL, - netkey TEXT NOT NULL, - - PRIMARY KEY (hostname, event), - FOREIGN KEY (event) REFERENCES event (id) ON DELETE CASCADE -); - -COMMENT ON TABLE voting_node IS -'This table holds information for all nodes in the event. -It is used by nodes to self-bootstrap the blockchain.'; -COMMENT ON COLUMN voting_node.hostname IS 'Unique hostname for the voting node.'; -COMMENT ON COLUMN voting_node.event IS 'Unique event this node was configured for.'; -COMMENT ON COLUMN voting_node.seckey IS 'Encrypted secret key from Ed25519 pair for the node. Used as the node secret.'; -COMMENT ON COLUMN voting_node.pubkey IS -'Public key from Ed25519 pair for the node. Used as consensus_leader_id when the node is a leader.'; -COMMENT ON COLUMN voting_node.netkey IS 'Encrypted Ed25519 secret key for the node. Used as the node p2p topology key.'; - - --- Tally Committee Table - Stores data about the tally committee per voting event --- This table is looked up by event -CREATE TABLE tally_committee ( - row_id SERIAL PRIMARY KEY, - - event UUID NOT NULL UNIQUE, - - committee_pk TEXT NOT NULL, - committee_id TEXT NOT NULL, - member_crs TEXT, - election_key TEXT, - - FOREIGN KEY (event) REFERENCES event (id) ON DELETE CASCADE -); - -COMMENT ON TABLE tally_committee IS 'Table for storing data about the tally committee per voting event.'; -COMMENT ON COLUMN tally_committee.row_id IS 'Unique ID for this committee member for this event.'; -COMMENT ON COLUMN tally_committee.event IS 'The event this committee member is for.'; -COMMENT ON COLUMN tally_committee.committee_pk IS -'Encrypted private key for the committee wallet. This key can be used to get the committee public address.'; -COMMENT ON COLUMN tally_committee.committee_id IS 'The hex-encoded public key for the committee wallet.'; -COMMENT ON COLUMN tally_committee.member_crs IS -'Encrypted Common Reference String shared in the creation of every set of committee member keys.'; -COMMENT ON COLUMN tally_committee.election_key IS -'Public key generated with all committee member public keys, and is used to encrypt votes. -NULL if the event.committee_size is 0.'; - - --- Committee Member Table - Stores data about the tally committee members --- This table is looked up by committee -CREATE TABLE committee_member ( - row_id SERIAL PRIMARY KEY, - - committee INTEGER NOT NULL, - - member_index INTEGER NOT NULL, - threshold INTEGER NOT NULL, - comm_pk TEXT NOT NULL, - comm_sk TEXT NOT NULL, - member_pk TEXT NOT NULL, - member_sk TEXT NOT NULL, - - FOREIGN KEY (committee) REFERENCES tally_committee (row_id) -); - -COMMENT ON TABLE committee_member IS 'Table for storing data about the tally committee members.'; -COMMENT ON COLUMN committee_member.row_id IS 'Unique ID for this committee member for this event.'; -COMMENT ON COLUMN committee_member.member_index IS -'the zero-based index of the member, ranging from 0 <= index < committee_size.'; -COMMENT ON COLUMN committee_member.committee IS 'The committee this member belongs to.'; -COMMENT ON COLUMN committee_member.comm_pk IS 'Committee member communication public key.'; -COMMENT ON COLUMN committee_member.comm_sk IS 'Encrypted committee member communication secret key.'; -COMMENT ON COLUMN committee_member.member_pk IS 'Committee member public key'; -COMMENT ON COLUMN committee_member.member_sk IS 'Encrypted committee member secret key'; diff --git a/catalyst-gateway/event-db/migrations/V9__moderation_stage.sql b/catalyst-gateway/event-db/migrations/V9__moderation_stage.sql deleted file mode 100644 index 954fd9be072..00000000000 --- a/catalyst-gateway/event-db/migrations/V9__moderation_stage.sql +++ /dev/null @@ -1,45 +0,0 @@ --- Catalyst Event Database - --- Title : Moderation Stage Data - --- ModerationAllocation - Defines the relationship between users and proposals_reviews --- to describe the allocation of moderations that needs to be done. - -CREATE TABLE moderation_allocation ( - row_id SERIAL PRIMARY KEY, - review_id INTEGER NOT NULL, - user_id INTEGER NOT NULL, - - FOREIGN KEY (review_id) REFERENCES proposal_review (row_id) ON DELETE CASCADE, - FOREIGN KEY (user_id) REFERENCES config (row_id) ON DELETE CASCADE -); - - -COMMENT ON TABLE moderation_allocation IS 'The relationship between users and proposals_reviews.'; -COMMENT ON COLUMN moderation_allocation.row_id IS 'Synthetic ID of this relationship.'; -COMMENT ON COLUMN moderation_allocation.review_id IS 'The review the relationship is related to.'; -COMMENT ON COLUMN moderation_allocation.user_id IS 'The user the relationship is related to.'; - - --- Moderation - Defines the moderation submitted by users for each proposal_review. - -CREATE TABLE moderation ( - row_id SERIAL PRIMARY KEY, - review_id INTEGER NOT NULL, - user_id INTEGER NOT NULL, - classification INTEGER NOT NULL, - rationale VARCHAR, - UNIQUE (review_id, user_id), - - FOREIGN KEY (review_id) REFERENCES proposal_review (row_id) ON DELETE CASCADE, - FOREIGN KEY (user_id) REFERENCES config (row_id) ON DELETE CASCADE -); - - -COMMENT ON TABLE moderation IS 'An individual moderation for a proposal review.'; -COMMENT ON COLUMN moderation.row_id IS 'Synthetic ID of this moderation.'; -COMMENT ON COLUMN moderation.review_id IS 'The review the moderation is related to.'; -COMMENT ON COLUMN moderation.user_id IS 'The user the moderation is submitted from.'; -COMMENT ON COLUMN moderation.classification IS -'The value used to describe the moderation (e.g. 0: excluded, 1: included).'; -COMMENT ON COLUMN moderation.rationale IS 'The rationale for the given classification.'; From 54258ef62c29ab500bbb221b3671b5c78b5985b5 Mon Sep 17 00:00:00 2001 From: bkioshn Date: Tue, 15 Oct 2024 22:32:21 +0700 Subject: [PATCH 17/37] fix: format Signed-off-by: bkioshn --- catalyst-gateway/bin/src/db/event/config/key.rs | 5 ++++- catalyst-gateway/bin/src/settings/mod.rs | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/config/key.rs b/catalyst-gateway/bin/src/db/event/config/key.rs index e40601ba31e..ad84bd70cac 100644 --- a/catalyst-gateway/bin/src/db/event/config/key.rs +++ b/catalyst-gateway/bin/src/db/event/config/key.rs @@ -67,7 +67,10 @@ impl ConfigKey { }; // Validate the value against the schema - anyhow::ensure!(validator.is_valid(value), "Invalid JSON, Failed schema validation"); + anyhow::ensure!( + validator.is_valid(value), + "Invalid JSON, Failed schema validation" + ); Ok(()) } diff --git a/catalyst-gateway/bin/src/settings/mod.rs b/catalyst-gateway/bin/src/settings/mod.rs index 6d86cddefcd..d46c6c7958a 100644 --- a/catalyst-gateway/bin/src/settings/mod.rs +++ b/catalyst-gateway/bin/src/settings/mod.rs @@ -53,7 +53,8 @@ const CHECK_CONFIG_TICK_DEFAULT: &str = "5s"; /// Default Event DB URL. const EVENT_DB_URL_DEFAULT: &str = - "postgresql://postgres:postgres@localhost/catalyst_events?sslmode=disable"; + // "postgresql://postgres:postgres@localhost/catalyst_events?sslmode=disable"; + "postgres://catalyst-event-dev:12341234@localhost:5432/CatalystEventDev?sslmode=disable"; /// Hash the Public IPv4 and IPv6 address of the machine, and convert to a 128 bit V4 /// UUID. From 7114d39e838f02ce9ed3ef8a3be91a8c2c49f758 Mon Sep 17 00:00:00 2001 From: bkioshn Date: Tue, 15 Oct 2024 22:36:10 +0700 Subject: [PATCH 18/37] chore: change license to MIT-0 Signed-off-by: bkioshn --- catalyst-gateway/LICENSE-MIT | 25 ------------------------- catalyst-gateway/MIT-0 | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 25 deletions(-) delete mode 100644 catalyst-gateway/LICENSE-MIT create mode 100644 catalyst-gateway/MIT-0 diff --git a/catalyst-gateway/LICENSE-MIT b/catalyst-gateway/LICENSE-MIT deleted file mode 100644 index 3a2971efd2e..00000000000 --- a/catalyst-gateway/LICENSE-MIT +++ /dev/null @@ -1,25 +0,0 @@ -Copyright (c) 2023 Input Output (IOG). - -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/catalyst-gateway/MIT-0 b/catalyst-gateway/MIT-0 new file mode 100644 index 00000000000..e36a8bb6b17 --- /dev/null +++ b/catalyst-gateway/MIT-0 @@ -0,0 +1,16 @@ +MIT No Attribution + +Copyright 2023 Input Output (IOG) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, +merge, publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file From 27e5310636e5fb2c59b79f1c6a7acf383edfc04e Mon Sep 17 00:00:00 2001 From: bkioshn Date: Tue, 15 Oct 2024 22:45:59 +0700 Subject: [PATCH 19/37] chore: remove license Signed-off-by: bkioshn --- catalyst-gateway/MIT-0 | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 catalyst-gateway/MIT-0 diff --git a/catalyst-gateway/MIT-0 b/catalyst-gateway/MIT-0 deleted file mode 100644 index e36a8bb6b17..00000000000 --- a/catalyst-gateway/MIT-0 +++ /dev/null @@ -1,16 +0,0 @@ -MIT No Attribution - -Copyright 2023 Input Output (IOG) - -Permission is hereby granted, free of charge, to any person obtaining a copy of this -software and associated documentation files (the "Software"), to deal in the Software -without restriction, including without limitation the rights to use, copy, modify, -merge, publish, distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, -INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file From cfa7e5595f0ce3bdc6b77da3ef990372fba12592 Mon Sep 17 00:00:00 2001 From: bkioshn Date: Wed, 16 Oct 2024 08:18:44 +0700 Subject: [PATCH 20/37] fix: add mit-0 license to deny.toml and test it Signed-off-by: bkioshn --- catalyst-gateway/Earthfile | 2 +- catalyst-gateway/deny.toml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/catalyst-gateway/Earthfile b/catalyst-gateway/Earthfile index 5acfdbaa22e..bdfa9b543eb 100644 --- a/catalyst-gateway/Earthfile +++ b/catalyst-gateway/Earthfile @@ -1,6 +1,6 @@ VERSION 0.8 -IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.2.16 AS rust-ci +IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:fix/mit-0-license AS rust-ci #cspell: words rustfmt toolsets USERARCH stdcfgs diff --git a/catalyst-gateway/deny.toml b/catalyst-gateway/deny.toml index 26ec8794bbf..8f5e39e8136 100644 --- a/catalyst-gateway/deny.toml +++ b/catalyst-gateway/deny.toml @@ -79,6 +79,7 @@ allow = [ "Unicode-3.0", "MPL-2.0", "Zlib", + "MIT-0", ] exceptions = [ #{ allow = ["Zlib"], crate = "tinyvec" }, @@ -120,4 +121,4 @@ license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }] #[[licenses.clarify]] #crate = "rustls-webpki" #expression = "ISC" -#license-files = [{ path = "LICENSE", hash = 0x001c7e6c }] +#license-files = [{ path = "LICENSE", hash = 0x001c7e6c }] \ No newline at end of file From 97d69adae74a094b90a5176ca7e54885293a6962 Mon Sep 17 00:00:00 2001 From: bkioshn Date: Wed, 16 Oct 2024 09:06:02 +0700 Subject: [PATCH 21/37] fix: update cat-gateway code gen Signed-off-by: bkioshn --- .../cat_gateway_api.swagger.chopper.dart | 29 +++++++++++++++++++ .../cat_gateway_api.swagger.dart | 29 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.chopper.dart b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.chopper.dart index 0bca4b80313..7509632ed03 100644 --- a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.chopper.dart +++ b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.chopper.dart @@ -192,6 +192,35 @@ final class _$CatGatewayApi extends CatGatewayApi { return client.send($request); } + @override + Future> _apiDraftConfigFrontendGet() { + final Uri $url = Uri.parse('/api/draft/config/frontend'); + final Request $request = Request( + 'GET', + $url, + client.baseUrl, + ); + return client.send($request); + } + + @override + Future> _apiDraftConfigFrontendPut({ + String? ip, + required Object? body, + }) { + final Uri $url = Uri.parse('/api/draft/config/frontend'); + final Map $params = {'IP': ip}; + final $body = body; + final Request $request = Request( + 'PUT', + $url, + client.baseUrl, + body: $body, + parameters: $params, + ); + return client.send($request); + } + @override Future> _apiRegistrationVoterVotingKeyGet({ required String? votingKey, diff --git a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.dart b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.dart index 88a85594d88..5d595518fb1 100644 --- a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.dart +++ b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.dart @@ -247,6 +247,35 @@ abstract class CatGatewayApi extends ChopperService { _apiCardanoCip36LatestRegistrationVoteKeyGet( {@Query('vote_key') required String? voteKey}); + ///Get the configuration for the frontend. + Future> apiDraftConfigFrontendGet() { + return _apiDraftConfigFrontendGet(); + } + + ///Get the configuration for the frontend. + @Get(path: '/api/draft/config/frontend') + Future> _apiDraftConfigFrontendGet(); + + ///Set the frontend configuration. + ///@param IP + Future> apiDraftConfigFrontendPut({ + String? ip, + required Object? body, + }) { + return _apiDraftConfigFrontendPut(ip: ip, body: body); + } + + ///Set the frontend configuration. + ///@param IP + @Put( + path: '/api/draft/config/frontend', + optionalBody: true, + ) + Future> _apiDraftConfigFrontendPut({ + @Query('IP') String? ip, + @Body() required Object? body, + }); + ///Voter's info ///@param voting_key A Voters Public ED25519 Key (as registered in their most recent valid [CIP-15](https://cips.cardano.org/cips/cip15) or [CIP-36](https://cips.cardano.org/cips/cip36) registration). ///@param event_id The Event ID to return results for. See [GET Events](Link to events endpoint) for details on retrieving all valid event IDs. From 5cc62ef16caec0939dc575c2ff999484d0c27cac Mon Sep 17 00:00:00 2001 From: bkioshn Date: Wed, 16 Oct 2024 11:00:11 +0700 Subject: [PATCH 22/37] fix: update cat-gateway rust-ci version Signed-off-by: bkioshn --- catalyst-gateway/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst-gateway/Earthfile b/catalyst-gateway/Earthfile index bdfa9b543eb..9eea78eadad 100644 --- a/catalyst-gateway/Earthfile +++ b/catalyst-gateway/Earthfile @@ -1,6 +1,6 @@ VERSION 0.8 -IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:fix/mit-0-license AS rust-ci +IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.2.17 AS rust-ci #cspell: words rustfmt toolsets USERARCH stdcfgs From fad3cfcf1e3e64908f837acfe78c5c1a87d1b301 Mon Sep 17 00:00:00 2001 From: bkioshn Date: Wed, 16 Oct 2024 14:54:31 +0700 Subject: [PATCH 23/37] fix: revert change Signed-off-by: bkioshn --- catalyst-gateway/bin/src/settings/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/catalyst-gateway/bin/src/settings/mod.rs b/catalyst-gateway/bin/src/settings/mod.rs index d46c6c7958a..6d86cddefcd 100644 --- a/catalyst-gateway/bin/src/settings/mod.rs +++ b/catalyst-gateway/bin/src/settings/mod.rs @@ -53,8 +53,7 @@ const CHECK_CONFIG_TICK_DEFAULT: &str = "5s"; /// Default Event DB URL. const EVENT_DB_URL_DEFAULT: &str = - // "postgresql://postgres:postgres@localhost/catalyst_events?sslmode=disable"; - "postgres://catalyst-event-dev:12341234@localhost:5432/CatalystEventDev?sslmode=disable"; + "postgresql://postgres:postgres@localhost/catalyst_events?sslmode=disable"; /// Hash the Public IPv4 and IPv6 address of the machine, and convert to a 128 bit V4 /// UUID. From 5fde921cf58440962f8222cc8fe04010eeb99aff Mon Sep 17 00:00:00 2001 From: bkioshn Date: Wed, 16 Oct 2024 14:54:54 +0700 Subject: [PATCH 24/37] fix: add new endpoint and fix validate json Signed-off-by: bkioshn --- .../db/event/config/default/frontend_ip.json | 1 + .../db/event/config/jsonschema/frontend.json | 1 + .../bin/src/db/event/config/key.rs | 42 ++++++++--- .../bin/src/db/event/config/mod.rs | 31 ++++++-- .../bin/src/service/api/config/mod.rs | 70 +++++++++++++++---- 5 files changed, 115 insertions(+), 30 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/event/config/default/frontend_ip.json diff --git a/catalyst-gateway/bin/src/db/event/config/default/frontend_ip.json b/catalyst-gateway/bin/src/db/event/config/default/frontend_ip.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/catalyst-gateway/bin/src/db/event/config/default/frontend_ip.json @@ -0,0 +1 @@ +{} diff --git a/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json b/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json index e8233bb95ae..cc521e747e1 100644 --- a/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json +++ b/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json @@ -1,6 +1,7 @@ { "$id": "https://www.stephenlewis.me/sentry-schema.json", "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Frontend JSON schema", "type": "object", "properties": { "sentry_dsn": { diff --git a/catalyst-gateway/bin/src/db/event/config/key.rs b/catalyst-gateway/bin/src/db/event/config/key.rs index ad84bd70cac..8c2e0a526f1 100644 --- a/catalyst-gateway/bin/src/db/event/config/key.rs +++ b/catalyst-gateway/bin/src/db/event/config/key.rs @@ -2,7 +2,7 @@ use std::{net::IpAddr, sync::LazyLock}; -use jsonschema::Validator; +use jsonschema::{BasicOutput, Validator}; use serde_json::{json, Value}; use tracing::error; @@ -15,14 +15,21 @@ pub(crate) enum ConfigKey { FrontendForIp(IpAddr), } +static FRONTEND_SCHEMA: LazyLock = + LazyLock::new(|| load_json_lazy(include_str!("jsonschema/frontend.json"))); + /// Frontend schema validator. static FRONTEND_SCHEMA_VALIDATOR: LazyLock = - LazyLock::new(|| schema_validator(&load_json_lazy(include_str!("jsonschema/frontend.json")))); + LazyLock::new(|| schema_validator(&FRONTEND_SCHEMA)); /// Frontend default configuration. static FRONTEND_DEFAULT: LazyLock = LazyLock::new(|| load_json_lazy(include_str!("default/frontend.json"))); +/// Frontend specific configuration. +static FRONTEND_IP_DEFAULT: LazyLock = + LazyLock::new(|| load_json_lazy(include_str!("default/frontend_ip.json"))); + /// Helper function to create a JSON validator from a JSON schema. /// If the schema is invalid, a default JSON validator is created. fn schema_validator(schema: &Value) -> Validator { @@ -60,25 +67,29 @@ impl ConfigKey { } /// Validate the provided value against the JSON schema. - pub(super) fn validate(&self, value: &Value) -> anyhow::Result<()> { + pub(super) fn validate(&self, value: &Value) -> BasicOutput<'static> { // Retrieve the validator based on ConfigKey let validator = match self { ConfigKey::Frontend | ConfigKey::FrontendForIp(_) => &*FRONTEND_SCHEMA_VALIDATOR, }; // Validate the value against the schema - anyhow::ensure!( - validator.is_valid(value), - "Invalid JSON, Failed schema validation" - ); - Ok(()) + validator.apply(value).basic() } /// Retrieve the default configuration value. pub(super) fn default(&self) -> Value { // Retrieve the default value based on the ConfigKey match self { - ConfigKey::Frontend | ConfigKey::FrontendForIp(_) => FRONTEND_DEFAULT.clone(), + ConfigKey::Frontend => FRONTEND_DEFAULT.clone(), + ConfigKey::FrontendForIp(_) => FRONTEND_IP_DEFAULT.clone(), + } + } + + /// Retrieve the JSON schema. + pub(crate) fn schema(&self) -> &Value { + match self { + ConfigKey::Frontend | ConfigKey::FrontendForIp(_) => &FRONTEND_SCHEMA, } } } @@ -90,12 +101,21 @@ mod tests { use super::*; #[test] - fn test_validate() { + fn test_valid_validate() { let value = json!({ "test": "test" }); let result = ConfigKey::Frontend.validate(&value); - assert!(result.is_ok()); + assert!(result.is_valid()); + println!("{:?}", serde_json::to_value(result).unwrap()); + } + + #[test] + fn test_invalid_validate() { + let value = json!([]); + let result = ConfigKey::Frontend.validate(&value); + assert!(!result.is_valid()); + println!("{:?}", serde_json::to_value(result).unwrap()); } #[test] diff --git a/catalyst-gateway/bin/src/db/event/config/mod.rs b/catalyst-gateway/bin/src/db/event/config/mod.rs index c931b4596dc..f7cda6cbab5 100644 --- a/catalyst-gateway/bin/src/db/event/config/mod.rs +++ b/catalyst-gateway/bin/src/db/event/config/mod.rs @@ -1,7 +1,9 @@ //! Configuration query +use jsonschema::BasicOutput; use key::ConfigKey; use serde_json::Value; +use tracing::error; use crate::db::event::EventDB; @@ -21,28 +23,43 @@ impl Config { /// # Returns /// /// - A JSON value of the configuration, if not found, returns the default value. + /// - Error if the query fails. pub(crate) async fn get(id: ConfigKey) -> anyhow::Result { let (id1, id2, id3) = id.to_id(); let rows = EventDB::query(GET_CONFIG, &[&id1, &id2, &id3]).await?; if let Some(row) = rows.first() { let value: Value = row.get(0); - id.validate(&value).map_err(|e| anyhow::anyhow!(e))?; - Ok(value) + match id.validate(&value) { + BasicOutput::Valid(_) => Ok(value), + BasicOutput::Invalid(errors) => { + // This should not happen; expecting the schema to be valid + error!("Validate schema failed: {:?}", errors); + Err(anyhow::anyhow!("Validate schema failed")) + }, + } } else { - // If data not found return default config value + // If data is not found, return the default config value Ok(id.default()) } } /// Set the configuration for the given `ConfigKey`. - pub(crate) async fn set(id: ConfigKey, value: Value) -> anyhow::Result<()> { - // Validate the value - id.validate(&value)?; + /// + /// # Returns + /// + /// - A `BasicOutput` of the validation result, which can be valid or invalid. + /// - Error if the query fails. + pub(crate) async fn set(id: ConfigKey, value: Value) -> anyhow::Result> { + let validate = id.validate(&value); + // Validate schema failed, return immediately with JSON schema error + if !validate.is_valid() { + return Ok(validate); + } let (id1, id2, id3) = id.to_id(); EventDB::query(UPSERT_CONFIG, &[&id1, &id2, &id3, &value]).await?; - Ok(()) + Ok(validate) } } diff --git a/catalyst-gateway/bin/src/service/api/config/mod.rs b/catalyst-gateway/bin/src/service/api/config/mod.rs index 46df7a58c30..0cc7f4e2957 100644 --- a/catalyst-gateway/bin/src/service/api/config/mod.rs +++ b/catalyst-gateway/bin/src/service/api/config/mod.rs @@ -2,9 +2,10 @@ use std::{net::IpAddr, str::FromStr}; +use jsonschema::BasicOutput; use poem::web::RealIp; use poem_openapi::{param::Query, payload::Json, ApiResponse, OpenApi}; -use serde_json::Value; +use serde_json::{json, Value}; use tracing::info; use crate::{ @@ -20,13 +21,13 @@ pub(crate) struct ConfigApi; enum Responses { /// Configuration result #[oai(status = 200)] - Ok(Json), - /// Configuration not found - #[oai(status = 404)] - NotFound(Json), + Ok(Json), /// Bad request #[oai(status = 400)] - BadRequest(Json), + BadRequest(Json), + /// Internal server error + #[oai(status = 500)] + ServerError(Json), } #[OpenApi(tag = "ApiTags::Config")] @@ -47,7 +48,12 @@ impl ConfigApi { // Attempt to fetch the IP configuration let ip_config = if let Some(ip) = ip_address.0 { - Config::get(ConfigKey::FrontendForIp(ip)).await.ok() + match Config::get(ConfigKey::FrontendForIp(ip)).await { + Ok(value) => Some(value), + Err(_) => { + return Responses::ServerError(Json("Failed to get configuration".to_string())) + }, + } } else { None }; @@ -63,9 +69,34 @@ impl ConfigApi { general }; - Responses::Ok(Json(response_config.to_string())) + Responses::Ok(Json(response_config)) + }, + Err(err) => Responses::ServerError(Json(err.to_string())), + } + } + + /// Get the frontend JSON schema. + #[oai( + path = "/draft/config/frontend/schema", + method = "get", + operation_id = "get_config_frontend_schema" + )] + #[allow(clippy::unused_async)] + async fn get_frontend_schema( + &self, #[oai(name = "IP")] ip_query: Query>, + ) -> Responses { + match ip_query.0 { + Some(ip) => { + match IpAddr::from_str(&ip) { + Ok(parsed_ip) => { + Responses::Ok(Json(ConfigKey::FrontendForIp(parsed_ip).schema().clone())) + }, + Err(err) => { + Responses::BadRequest(Json(json!(format!("Invalid IP address: {err}")))) + }, + } }, - Err(err) => Responses::NotFound(Json(err.to_string())), + None => Responses::Ok(Json(ConfigKey::Frontend.schema().clone())), } } @@ -84,7 +115,9 @@ impl ConfigApi { Some(ip) => { match IpAddr::from_str(&ip) { Ok(parsed_ip) => set(ConfigKey::FrontendForIp(parsed_ip), body_value).await, - Err(err) => Responses::BadRequest(Json(format!("Invalid IP address: {err}"))), + Err(err) => { + Responses::BadRequest(Json(json!(format!("Invalid IP address: {err}")))) + }, } }, None => set(ConfigKey::Frontend, body_value).await, @@ -114,7 +147,20 @@ fn merge_configs(general: &Value, ip_specific: &Value) -> Value { /// Helper function to handle set. async fn set(key: ConfigKey, value: Value) -> Responses { match Config::set(key, value).await { - Ok(()) => Responses::Ok(Json("Configuration successfully set.".to_string())), - Err(err) => Responses::BadRequest(Json(format!("Failed to set configuration: {err}"))), + Ok(validate) => { + match validate { + BasicOutput::Valid(_) => { + Responses::Ok(Json(json!("Configuration successfully set."))) + }, + BasicOutput::Invalid(errors) => { + let mut e = vec![]; + for error in errors { + e.push(error.error_description().to_string()); + } + Responses::BadRequest(Json(json!({"errors": e}))) + }, + } + }, + Err(err) => Responses::ServerError(Json(format!("Failed to set configuration: {err}"))), } } From 4e1a2b3978103e38d91fdc1f80b0b9097f4f10c9 Mon Sep 17 00:00:00 2001 From: bkioshn Date: Wed, 16 Oct 2024 15:16:35 +0700 Subject: [PATCH 25/37] fix: cat-gateway api code gen Signed-off-by: bkioshn --- .../cat_gateway_api.swagger.chopper.dart | 21 +++++++++++++++---- .../cat_gateway_api.swagger.dart | 20 ++++++++++++++---- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.chopper.dart b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.chopper.dart index 7509632ed03..a0d85cbd850 100644 --- a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.chopper.dart +++ b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.chopper.dart @@ -193,18 +193,18 @@ final class _$CatGatewayApi extends CatGatewayApi { } @override - Future> _apiDraftConfigFrontendGet() { + Future> _apiDraftConfigFrontendGet() { final Uri $url = Uri.parse('/api/draft/config/frontend'); final Request $request = Request( 'GET', $url, client.baseUrl, ); - return client.send($request); + return client.send($request); } @override - Future> _apiDraftConfigFrontendPut({ + Future> _apiDraftConfigFrontendPut({ String? ip, required Object? body, }) { @@ -218,7 +218,20 @@ final class _$CatGatewayApi extends CatGatewayApi { body: $body, parameters: $params, ); - return client.send($request); + return client.send($request); + } + + @override + Future> _apiDraftConfigFrontendSchemaGet({String? ip}) { + final Uri $url = Uri.parse('/api/draft/config/frontend/schema'); + final Map $params = {'IP': ip}; + final Request $request = Request( + 'GET', + $url, + client.baseUrl, + parameters: $params, + ); + return client.send($request); } @override diff --git a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.dart b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.dart index 5d595518fb1..e2325692f8c 100644 --- a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.dart +++ b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.dart @@ -248,17 +248,17 @@ abstract class CatGatewayApi extends ChopperService { {@Query('vote_key') required String? voteKey}); ///Get the configuration for the frontend. - Future> apiDraftConfigFrontendGet() { + Future apiDraftConfigFrontendGet() { return _apiDraftConfigFrontendGet(); } ///Get the configuration for the frontend. @Get(path: '/api/draft/config/frontend') - Future> _apiDraftConfigFrontendGet(); + Future _apiDraftConfigFrontendGet(); ///Set the frontend configuration. ///@param IP - Future> apiDraftConfigFrontendPut({ + Future apiDraftConfigFrontendPut({ String? ip, required Object? body, }) { @@ -271,11 +271,23 @@ abstract class CatGatewayApi extends ChopperService { path: '/api/draft/config/frontend', optionalBody: true, ) - Future> _apiDraftConfigFrontendPut({ + Future _apiDraftConfigFrontendPut({ @Query('IP') String? ip, @Body() required Object? body, }); + ///Get the frontend JSON schema. + ///@param IP + Future apiDraftConfigFrontendSchemaGet({String? ip}) { + return _apiDraftConfigFrontendSchemaGet(ip: ip); + } + + ///Get the frontend JSON schema. + ///@param IP + @Get(path: '/api/draft/config/frontend/schema') + Future _apiDraftConfigFrontendSchemaGet( + {@Query('IP') String? ip}); + ///Voter's info ///@param voting_key A Voters Public ED25519 Key (as registered in their most recent valid [CIP-15](https://cips.cardano.org/cips/cip15) or [CIP-36](https://cips.cardano.org/cips/cip36) registration). ///@param event_id The Event ID to return results for. See [GET Events](Link to events endpoint) for details on retrieving all valid event IDs. From e4afeb148354fde8b8e8163d7d3380dcca5f375b Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Wed, 16 Oct 2024 17:19:27 +0700 Subject: [PATCH 26/37] Update catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json --- .../bin/src/db/event/config/jsonschema/frontend.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json b/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json index cc521e747e1..8fd4a720575 100644 --- a/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json +++ b/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json @@ -5,7 +5,7 @@ "type": "object", "properties": { "sentry_dsn": { - "$ref": "#/definitions/saneUrl", + "$ref": "#/definitions/httpsUrl", "description": "The Data Source Name (DSN) for Sentry." }, "sentry_release": { From 65c587af113929693a8c23afc49736d3f81e228c Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Wed, 16 Oct 2024 17:19:57 +0700 Subject: [PATCH 27/37] Update catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json --- .../bin/src/db/event/config/jsonschema/frontend.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json b/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json index 8fd4a720575..184f3acf8a7 100644 --- a/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json +++ b/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json @@ -19,7 +19,7 @@ }, "additionalProperties": true, "definitions": { - "saneUrl": { + "httpsUrl": { "type": "string", "format": "uri", "pattern": "^https?://" From ef7870297b9c96e2696684d64ea9f77a864e352a Mon Sep 17 00:00:00 2001 From: bkioshn Date: Thu, 17 Oct 2024 11:28:24 +0700 Subject: [PATCH 28/37] fix: frontend default and json schema Signed-off-by: bkioshn --- .../src/db/event/config/default/frontend.json | 4 ++- .../db/event/config/jsonschema/frontend.json | 27 ++++++++++--------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/config/default/frontend.json b/catalyst-gateway/bin/src/db/event/config/default/frontend.json index bd8f2ba3856..9df58ed736f 100644 --- a/catalyst-gateway/bin/src/db/event/config/default/frontend.json +++ b/catalyst-gateway/bin/src/db/event/config/default/frontend.json @@ -1,3 +1,5 @@ { - "sentry_environment": "dev" + "sentry": { + "environment": "dev" + } } diff --git a/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json b/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json index 184f3acf8a7..d532bb3d1ad 100644 --- a/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json +++ b/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json @@ -4,22 +4,23 @@ "title": "Frontend JSON schema", "type": "object", "properties": { - "sentry_dsn": { - "$ref": "#/definitions/httpsUrl", - "description": "The Data Source Name (DSN) for Sentry." - }, - "sentry_release": { - "type": "string", - "description": "A version of the code deployed to an environment" - }, - "sentry_environment": { - "type": "string", - "description": "The environment in which the application is running, e.g., 'dev', 'qa'." + "sentry": { + "dsn": { + "$ref": "#/definitions/saneUrl", + "description": "The Data Source Name (DSN) for Sentry." + }, + "release": { + "type": "string", + "description": "A version of the code deployed to an environment" + }, + "environment": { + "type": "string", + "description": "The environment in which the application is running, e.g., 'dev', 'qa'." + } } }, - "additionalProperties": true, "definitions": { - "httpsUrl": { + "saneUrl": { "type": "string", "format": "uri", "pattern": "^https?://" From 80db594f805af148ab4c237ae5f50b921aec1d0b Mon Sep 17 00:00:00 2001 From: bkioshn Date: Thu, 17 Oct 2024 11:28:49 +0700 Subject: [PATCH 29/37] fix: error handling Signed-off-by: bkioshn --- .../bin/src/db/event/config/key.rs | 12 ++- .../bin/src/db/event/config/mod.rs | 15 ++-- .../bin/src/service/api/config/mod.rs | 87 ++++++++++--------- 3 files changed, 64 insertions(+), 50 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/config/key.rs b/catalyst-gateway/bin/src/db/event/config/key.rs index 8c2e0a526f1..4a1ec2bc4e4 100644 --- a/catalyst-gateway/bin/src/db/event/config/key.rs +++ b/catalyst-gateway/bin/src/db/event/config/key.rs @@ -1,6 +1,6 @@ //! Configuration Key -use std::{net::IpAddr, sync::LazyLock}; +use std::{fmt::Display, net::IpAddr, sync::LazyLock}; use jsonschema::{BasicOutput, Validator}; use serde_json::{json, Value}; @@ -15,6 +15,16 @@ pub(crate) enum ConfigKey { FrontendForIp(IpAddr), } +impl Display for ConfigKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ConfigKey::Frontend => write!(f, "Frontend configuration"), + ConfigKey::FrontendForIp(_) => write!(f, "Frontend configuration for IP"), + } + } +} + +/// Frontend schema. static FRONTEND_SCHEMA: LazyLock = LazyLock::new(|| load_json_lazy(include_str!("jsonschema/frontend.json"))); diff --git a/catalyst-gateway/bin/src/db/event/config/mod.rs b/catalyst-gateway/bin/src/db/event/config/mod.rs index f7cda6cbab5..6d67779c80b 100644 --- a/catalyst-gateway/bin/src/db/event/config/mod.rs +++ b/catalyst-gateway/bin/src/db/event/config/mod.rs @@ -22,7 +22,8 @@ impl Config { /// /// # Returns /// - /// - A JSON value of the configuration, if not found, returns the default value. + /// - A JSON value of the configuration, if not found or error, returns the default + /// value. /// - Error if the query fails. pub(crate) async fn get(id: ConfigKey) -> anyhow::Result { let (id1, id2, id3) = id.to_id(); @@ -31,17 +32,15 @@ impl Config { if let Some(row) = rows.first() { let value: Value = row.get(0); match id.validate(&value) { - BasicOutput::Valid(_) => Ok(value), + BasicOutput::Valid(_) => return Ok(value), BasicOutput::Invalid(errors) => { - // This should not happen; expecting the schema to be valid - error!("Validate schema failed: {:?}", errors); - Err(anyhow::anyhow!("Validate schema failed")) + // This should not happen, expecting the schema to be valid + error!(id=%id, errors=?errors, "Get Config, Schema validation failed, defaulting."); }, } - } else { - // If data is not found, return the default config value - Ok(id.default()) } + // Return the default config value as a fallback + Ok(id.default()) } /// Set the configuration for the given `ConfigKey`. diff --git a/catalyst-gateway/bin/src/service/api/config/mod.rs b/catalyst-gateway/bin/src/service/api/config/mod.rs index 0cc7f4e2957..25c248b55f5 100644 --- a/catalyst-gateway/bin/src/service/api/config/mod.rs +++ b/catalyst-gateway/bin/src/service/api/config/mod.rs @@ -4,13 +4,13 @@ use std::{net::IpAddr, str::FromStr}; use jsonschema::BasicOutput; use poem::web::RealIp; -use poem_openapi::{param::Query, payload::Json, ApiResponse, OpenApi}; +use poem_openapi::{param::Query, payload::Json, ApiResponse, Object, OpenApi}; use serde_json::{json, Value}; -use tracing::info; +use tracing::{error, info}; use crate::{ db::event::config::{key::ConfigKey, Config}, - service::common::tags::ApiTags, + service::common::{responses::WithErrorResponses, tags::ApiTags}, }; /// Configuration API struct @@ -24,12 +24,19 @@ enum Responses { Ok(Json), /// Bad request #[oai(status = 400)] - BadRequest(Json), - /// Internal server error - #[oai(status = 500)] - ServerError(Json), + BadRequest(Json), } +/// Bad request errors +#[derive(Object, Default)] +struct BadRequestError { + /// List of errors + errors: Vec, +} + +/// All responses. +type AllResponses = WithErrorResponses; + #[OpenApi(tag = "ApiTags::Config")] impl ConfigApi { /// Get the configuration for the frontend. @@ -40,8 +47,8 @@ impl ConfigApi { method = "get", operation_id = "get_config_frontend" )] - async fn get_frontend(&self, ip_address: RealIp) -> Responses { - info!("IP Address: {:?}", ip_address.0); + async fn get_frontend(&self, ip_address: RealIp) -> AllResponses { + info!(id = "get_config_frontend", "IP Address: {:?}", ip_address.0); // Fetch the general configuration let general_config = Config::get(ConfigKey::Frontend).await; @@ -50,8 +57,9 @@ impl ConfigApi { let ip_config = if let Some(ip) = ip_address.0 { match Config::get(ConfigKey::FrontendForIp(ip)).await { Ok(value) => Some(value), - Err(_) => { - return Responses::ServerError(Json("Failed to get configuration".to_string())) + Err(err) => { + error!(id="get_config_frontend", errors=?err, "Failed to get configuration for IP"); + return AllResponses::handle_error(&err); }, } } else { @@ -69,9 +77,12 @@ impl ConfigApi { general }; - Responses::Ok(Json(response_config)) + Responses::Ok(Json(response_config)).into() + }, + Err(err) => { + error!(id="get_config_frontend", errors=?err, "Failed to get general configuration"); + AllResponses::handle_error(&err) }, - Err(err) => Responses::ServerError(Json(err.to_string())), } } @@ -82,22 +93,9 @@ impl ConfigApi { operation_id = "get_config_frontend_schema" )] #[allow(clippy::unused_async)] - async fn get_frontend_schema( - &self, #[oai(name = "IP")] ip_query: Query>, - ) -> Responses { - match ip_query.0 { - Some(ip) => { - match IpAddr::from_str(&ip) { - Ok(parsed_ip) => { - Responses::Ok(Json(ConfigKey::FrontendForIp(parsed_ip).schema().clone())) - }, - Err(err) => { - Responses::BadRequest(Json(json!(format!("Invalid IP address: {err}")))) - }, - } - }, - None => Responses::Ok(Json(ConfigKey::Frontend.schema().clone())), - } + async fn get_frontend_schema(&self) -> AllResponses { + // Schema for both IP specific and general are identical + Responses::Ok(Json(ConfigKey::Frontend.schema().clone())).into() } /// Set the frontend configuration. @@ -108,7 +106,7 @@ impl ConfigApi { )] async fn put_frontend( &self, #[oai(name = "IP")] ip_query: Query>, body: Json, - ) -> Responses { + ) -> AllResponses { let body_value = body.0; match ip_query.0 { @@ -116,7 +114,10 @@ impl ConfigApi { match IpAddr::from_str(&ip) { Ok(parsed_ip) => set(ConfigKey::FrontendForIp(parsed_ip), body_value).await, Err(err) => { - Responses::BadRequest(Json(json!(format!("Invalid IP address: {err}")))) + Responses::BadRequest(Json(BadRequestError { + errors: vec![format!("Invalid IP address: {err}")], + })) + .into() }, } }, @@ -145,22 +146,26 @@ fn merge_configs(general: &Value, ip_specific: &Value) -> Value { } /// Helper function to handle set. -async fn set(key: ConfigKey, value: Value) -> Responses { +async fn set(key: ConfigKey, value: Value) -> AllResponses { match Config::set(key, value).await { Ok(validate) => { match validate { - BasicOutput::Valid(_) => { - Responses::Ok(Json(json!("Configuration successfully set."))) - }, + BasicOutput::Valid(_) => Responses::Ok(Json(json!(null))).into(), BasicOutput::Invalid(errors) => { - let mut e = vec![]; - for error in errors { - e.push(error.error_description().to_string()); - } - Responses::BadRequest(Json(json!({"errors": e}))) + let error_descriptions: Vec = errors + .iter() + .map(|error| error.error_description().clone().into_inner()) + .collect(); + Responses::BadRequest(Json(BadRequestError { + errors: error_descriptions, + })) + .into() }, } }, - Err(err) => Responses::ServerError(Json(format!("Failed to set configuration: {err}"))), + Err(err) => { + error!(id="put_config_frontend", errors=?err, "Failed to set configuration"); + AllResponses::handle_error(&err) + }, } } From 577aee6ba0e4e5ecf7bf832fecffa5618df8c20f Mon Sep 17 00:00:00 2001 From: bkioshn Date: Thu, 17 Oct 2024 11:41:50 +0700 Subject: [PATCH 30/37] fix: cat-gateway api code gen Signed-off-by: bkioshn --- .../cat_gateway_api.models.swagger.dart | 43 +++++++++++++++++++ .../cat_gateway_api.models.swagger.g.dart | 13 ++++++ .../cat_gateway_api.swagger.chopper.dart | 4 +- .../cat_gateway_api.swagger.dart | 9 ++-- 4 files changed, 60 insertions(+), 9 deletions(-) diff --git a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.models.swagger.dart b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.models.swagger.dart index 02e3d942f52..02c5ffafef3 100644 --- a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.models.swagger.dart +++ b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.models.swagger.dart @@ -62,6 +62,49 @@ extension $AccountVoteExtension on AccountVote { } } +@JsonSerializable(explicitToJson: true) +class BadRequestError { + const BadRequestError({ + required this.errors, + }); + + factory BadRequestError.fromJson(Map json) => + _$BadRequestErrorFromJson(json); + + static const toJsonFactory = _$BadRequestErrorToJson; + Map toJson() => _$BadRequestErrorToJson(this); + + @JsonKey(name: 'errors', defaultValue: []) + final List errors; + static const fromJsonFactory = _$BadRequestErrorFromJson; + + @override + bool operator ==(Object other) { + return identical(this, other) || + (other is BadRequestError && + (identical(other.errors, errors) || + const DeepCollectionEquality().equals(other.errors, errors))); + } + + @override + String toString() => jsonEncode(this); + + @override + int get hashCode => + const DeepCollectionEquality().hash(errors) ^ runtimeType.hashCode; +} + +extension $BadRequestErrorExtension on BadRequestError { + BadRequestError copyWith({List? errors}) { + return BadRequestError(errors: errors ?? this.errors); + } + + BadRequestError copyWithWrapped({Wrapped>? errors}) { + return BadRequestError( + errors: (errors != null ? errors.value : this.errors)); + } +} + @JsonSerializable(explicitToJson: true) class BlockDate { const BlockDate({ diff --git a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.models.swagger.g.dart b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.models.swagger.g.dart index b66b2e58a00..1734157e189 100644 --- a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.models.swagger.g.dart +++ b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.models.swagger.g.dart @@ -20,6 +20,19 @@ Map _$AccountVoteToJson(AccountVote instance) => 'votes': instance.votes, }; +BadRequestError _$BadRequestErrorFromJson(Map json) => + BadRequestError( + errors: (json['errors'] as List?) + ?.map((e) => e as String) + .toList() ?? + [], + ); + +Map _$BadRequestErrorToJson(BadRequestError instance) => + { + 'errors': instance.errors, + }; + BlockDate _$BlockDateFromJson(Map json) => BlockDate( epoch: (json['epoch'] as num).toInt(), slotId: (json['slot_id'] as num).toInt(), diff --git a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.chopper.dart b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.chopper.dart index a0d85cbd850..2b3298773be 100644 --- a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.chopper.dart +++ b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.chopper.dart @@ -222,14 +222,12 @@ final class _$CatGatewayApi extends CatGatewayApi { } @override - Future> _apiDraftConfigFrontendSchemaGet({String? ip}) { + Future> _apiDraftConfigFrontendSchemaGet() { final Uri $url = Uri.parse('/api/draft/config/frontend/schema'); - final Map $params = {'IP': ip}; final Request $request = Request( 'GET', $url, client.baseUrl, - parameters: $params, ); return client.send($request); } diff --git a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.dart b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.dart index e2325692f8c..da1eb940421 100644 --- a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.dart +++ b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.swagger.dart @@ -277,16 +277,13 @@ abstract class CatGatewayApi extends ChopperService { }); ///Get the frontend JSON schema. - ///@param IP - Future apiDraftConfigFrontendSchemaGet({String? ip}) { - return _apiDraftConfigFrontendSchemaGet(ip: ip); + Future apiDraftConfigFrontendSchemaGet() { + return _apiDraftConfigFrontendSchemaGet(); } ///Get the frontend JSON schema. - ///@param IP @Get(path: '/api/draft/config/frontend/schema') - Future _apiDraftConfigFrontendSchemaGet( - {@Query('IP') String? ip}); + Future _apiDraftConfigFrontendSchemaGet(); ///Voter's info ///@param voting_key A Voters Public ED25519 Key (as registered in their most recent valid [CIP-15](https://cips.cardano.org/cips/cip15) or [CIP-36](https://cips.cardano.org/cips/cip36) registration). From a0062c269c11caaecb35907c72529d11503d3bcf Mon Sep 17 00:00:00 2001 From: bkioshn Date: Thu, 17 Oct 2024 12:17:32 +0700 Subject: [PATCH 31/37] fix: openapi lint Signed-off-by: bkioshn --- catalyst-gateway/bin/src/service/api/config/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/catalyst-gateway/bin/src/service/api/config/mod.rs b/catalyst-gateway/bin/src/service/api/config/mod.rs index 25c248b55f5..a6931073187 100644 --- a/catalyst-gateway/bin/src/service/api/config/mod.rs +++ b/catalyst-gateway/bin/src/service/api/config/mod.rs @@ -31,6 +31,7 @@ enum Responses { #[derive(Object, Default)] struct BadRequestError { /// List of errors + #[oai(validator(max_items = "1000", max_length = "9999", pattern = "^[0-9a-zA-Z].*$"))] errors: Vec, } From ffa2dec5a838f7ea795f803047ed82739acb1501 Mon Sep 17 00:00:00 2001 From: bkioshn Date: Thu, 17 Oct 2024 14:42:05 +0700 Subject: [PATCH 32/37] fix: frontend json schema Signed-off-by: bkioshn --- .../bin/src/db/event/config/jsonschema/frontend.json | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json b/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json index d532bb3d1ad..36a5876eace 100644 --- a/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json +++ b/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json @@ -3,10 +3,12 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Frontend JSON schema", "type": "object", - "properties": { - "sentry": { + "sentry": { + "type": "object", + "description": "Configuration for Sentry.", + "properties": { "dsn": { - "$ref": "#/definitions/saneUrl", + "$ref": "#/definitions/httpsUrl", "description": "The Data Source Name (DSN) for Sentry." }, "release": { @@ -20,7 +22,7 @@ } }, "definitions": { - "saneUrl": { + "httpsUrl": { "type": "string", "format": "uri", "pattern": "^https?://" From 1d6fa8517014b295d1cbedcca18277f7d65dfebc Mon Sep 17 00:00:00 2001 From: bkioshn Date: Thu, 17 Oct 2024 14:45:27 +0700 Subject: [PATCH 33/37] fix: error handling Signed-off-by: bkioshn --- catalyst-gateway/bin/src/db/event/config/key.rs | 12 ++++++++---- .../bin/src/service/api/config/mod.rs | 15 ++++++++++----- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/config/key.rs b/catalyst-gateway/bin/src/db/event/config/key.rs index 4a1ec2bc4e4..cfdec51bf57 100644 --- a/catalyst-gateway/bin/src/db/event/config/key.rs +++ b/catalyst-gateway/bin/src/db/event/config/key.rs @@ -43,8 +43,12 @@ static FRONTEND_IP_DEFAULT: LazyLock = /// Helper function to create a JSON validator from a JSON schema. /// If the schema is invalid, a default JSON validator is created. fn schema_validator(schema: &Value) -> Validator { - jsonschema::validator_for(schema).unwrap_or_else(|e| { - error!("Error creating JSON validator: {}", e); + jsonschema::validator_for(schema).unwrap_or_else(|err| { + error!( + id = "schema_validator", + errors=?err, + "Error creating JSON validator" + ); // Create a default JSON validator as a fallback // This should not fail since it is hard coded @@ -59,8 +63,8 @@ fn schema_validator(schema: &Value) -> Validator { /// Helper function to convert a JSON string to a JSON value. fn load_json_lazy(data: &str) -> Value { - serde_json::from_str(data).unwrap_or_else(|e| { - error!("Error parsing JSON: {}", e); + serde_json::from_str(data).unwrap_or_else(|err| { + error!(id = "load_json_lazy", errors=?err, "Error parsing JSON"); json!({}) }) } diff --git a/catalyst-gateway/bin/src/service/api/config/mod.rs b/catalyst-gateway/bin/src/service/api/config/mod.rs index a6931073187..93618ce02a7 100644 --- a/catalyst-gateway/bin/src/service/api/config/mod.rs +++ b/catalyst-gateway/bin/src/service/api/config/mod.rs @@ -30,9 +30,12 @@ enum Responses { /// Bad request errors #[derive(Object, Default)] struct BadRequestError { - /// List of errors + /// Error messages. + #[oai(validator(max_length = "100", pattern = "^[0-9a-zA-Z].*$"))] + error: String, + /// Optional schema validation errors. #[oai(validator(max_items = "1000", max_length = "9999", pattern = "^[0-9a-zA-Z].*$"))] - errors: Vec, + schema_validation_errors: Option>, } /// All responses. @@ -116,7 +119,8 @@ impl ConfigApi { Ok(parsed_ip) => set(ConfigKey::FrontendForIp(parsed_ip), body_value).await, Err(err) => { Responses::BadRequest(Json(BadRequestError { - errors: vec![format!("Invalid IP address: {err}")], + error: format!("Invalid IP address: {err}"), + schema_validation_errors: None, })) .into() }, @@ -153,12 +157,13 @@ async fn set(key: ConfigKey, value: Value) -> AllResponses { match validate { BasicOutput::Valid(_) => Responses::Ok(Json(json!(null))).into(), BasicOutput::Invalid(errors) => { - let error_descriptions: Vec = errors + let schema_errors: Vec = errors .iter() .map(|error| error.error_description().clone().into_inner()) .collect(); Responses::BadRequest(Json(BadRequestError { - errors: error_descriptions, + error: "Invalid JSON data validating against JSON schema".to_string(), + schema_validation_errors: Some(schema_errors), })) .into() }, From 1a6ac401d72ddbbc6cdcc5fc6499741bbac62e8e Mon Sep 17 00:00:00 2001 From: bkioshn Date: Thu, 17 Oct 2024 14:52:47 +0700 Subject: [PATCH 34/37] fix: cat-gateway api code gen Signed-off-by: bkioshn --- .../cat_gateway_api.models.swagger.dart | 37 ++++++++++++++----- .../cat_gateway_api.models.swagger.g.dart | 13 ++++--- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.models.swagger.dart b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.models.swagger.dart index 02c5ffafef3..760d1fe957a 100644 --- a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.models.swagger.dart +++ b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.models.swagger.dart @@ -65,7 +65,8 @@ extension $AccountVoteExtension on AccountVote { @JsonSerializable(explicitToJson: true) class BadRequestError { const BadRequestError({ - required this.errors, + required this.error, + this.schemaValidationErrors, }); factory BadRequestError.fromJson(Map json) => @@ -74,16 +75,21 @@ class BadRequestError { static const toJsonFactory = _$BadRequestErrorToJson; Map toJson() => _$BadRequestErrorToJson(this); - @JsonKey(name: 'errors', defaultValue: []) - final List errors; + @JsonKey(name: 'error') + final String error; + @JsonKey(name: 'schema_validation_errors', defaultValue: []) + final List? schemaValidationErrors; static const fromJsonFactory = _$BadRequestErrorFromJson; @override bool operator ==(Object other) { return identical(this, other) || (other is BadRequestError && - (identical(other.errors, errors) || - const DeepCollectionEquality().equals(other.errors, errors))); + (identical(other.error, error) || + const DeepCollectionEquality().equals(other.error, error)) && + (identical(other.schemaValidationErrors, schemaValidationErrors) || + const DeepCollectionEquality().equals( + other.schemaValidationErrors, schemaValidationErrors))); } @override @@ -91,17 +97,28 @@ class BadRequestError { @override int get hashCode => - const DeepCollectionEquality().hash(errors) ^ runtimeType.hashCode; + const DeepCollectionEquality().hash(error) ^ + const DeepCollectionEquality().hash(schemaValidationErrors) ^ + runtimeType.hashCode; } extension $BadRequestErrorExtension on BadRequestError { - BadRequestError copyWith({List? errors}) { - return BadRequestError(errors: errors ?? this.errors); + BadRequestError copyWith( + {String? error, List? schemaValidationErrors}) { + return BadRequestError( + error: error ?? this.error, + schemaValidationErrors: + schemaValidationErrors ?? this.schemaValidationErrors); } - BadRequestError copyWithWrapped({Wrapped>? errors}) { + BadRequestError copyWithWrapped( + {Wrapped? error, + Wrapped?>? schemaValidationErrors}) { return BadRequestError( - errors: (errors != null ? errors.value : this.errors)); + error: (error != null ? error.value : this.error), + schemaValidationErrors: (schemaValidationErrors != null + ? schemaValidationErrors.value + : this.schemaValidationErrors)); } } diff --git a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.models.swagger.g.dart b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.models.swagger.g.dart index 1734157e189..3672b3cf353 100644 --- a/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.models.swagger.g.dart +++ b/catalyst_voices/packages/catalyst_voices_services/lib/generated/catalyst_gateway/cat_gateway_api.models.swagger.g.dart @@ -22,15 +22,18 @@ Map _$AccountVoteToJson(AccountVote instance) => BadRequestError _$BadRequestErrorFromJson(Map json) => BadRequestError( - errors: (json['errors'] as List?) - ?.map((e) => e as String) - .toList() ?? - [], + error: json['error'] as String, + schemaValidationErrors: + (json['schema_validation_errors'] as List?) + ?.map((e) => e as String) + .toList() ?? + [], ); Map _$BadRequestErrorToJson(BadRequestError instance) => { - 'errors': instance.errors, + 'error': instance.error, + 'schema_validation_errors': instance.schemaValidationErrors, }; BlockDate _$BlockDateFromJson(Map json) => BlockDate( From e0813d47267d2e55bbfde1c6cb0f80bcf8dcce0e Mon Sep 17 00:00:00 2001 From: bkioshn Date: Thu, 17 Oct 2024 17:40:38 +0700 Subject: [PATCH 35/37] fix: remove id Signed-off-by: bkioshn --- .../bin/src/db/event/config/jsonschema/frontend.json | 1 - 1 file changed, 1 deletion(-) diff --git a/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json b/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json index 36a5876eace..674c23dd099 100644 --- a/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json +++ b/catalyst-gateway/bin/src/db/event/config/jsonschema/frontend.json @@ -1,5 +1,4 @@ { - "$id": "https://www.stephenlewis.me/sentry-schema.json", "$schema": "http://json-schema.org/draft-07/schema#", "title": "Frontend JSON schema", "type": "object", From 55965fb747432bf42a554c3f68cfe8186b9ff614 Mon Sep 17 00:00:00 2001 From: bkioshn Date: Fri, 18 Oct 2024 10:48:35 +0700 Subject: [PATCH 36/37] fix: error log Signed-off-by: bkioshn --- catalyst-gateway/bin/src/db/event/config/key.rs | 8 ++++---- catalyst-gateway/bin/src/db/event/config/mod.rs | 2 +- catalyst-gateway/bin/src/service/api/config/mod.rs | 10 ++++------ 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/catalyst-gateway/bin/src/db/event/config/key.rs b/catalyst-gateway/bin/src/db/event/config/key.rs index cfdec51bf57..2d02be110b6 100644 --- a/catalyst-gateway/bin/src/db/event/config/key.rs +++ b/catalyst-gateway/bin/src/db/event/config/key.rs @@ -18,8 +18,8 @@ pub(crate) enum ConfigKey { impl Display for ConfigKey { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - ConfigKey::Frontend => write!(f, "Frontend configuration"), - ConfigKey::FrontendForIp(_) => write!(f, "Frontend configuration for IP"), + ConfigKey::Frontend => write!(f, "config_key_frontend"), + ConfigKey::FrontendForIp(_) => write!(f, "config_key_frontend_ip"), } } } @@ -46,7 +46,7 @@ fn schema_validator(schema: &Value) -> Validator { jsonschema::validator_for(schema).unwrap_or_else(|err| { error!( id = "schema_validator", - errors=?err, + error=?err, "Error creating JSON validator" ); @@ -64,7 +64,7 @@ fn schema_validator(schema: &Value) -> Validator { /// Helper function to convert a JSON string to a JSON value. fn load_json_lazy(data: &str) -> Value { serde_json::from_str(data).unwrap_or_else(|err| { - error!(id = "load_json_lazy", errors=?err, "Error parsing JSON"); + error!(id = "load_json_lazy", error=?err, "Error parsing JSON"); json!({}) }) } diff --git a/catalyst-gateway/bin/src/db/event/config/mod.rs b/catalyst-gateway/bin/src/db/event/config/mod.rs index 6d67779c80b..2ab81b6523e 100644 --- a/catalyst-gateway/bin/src/db/event/config/mod.rs +++ b/catalyst-gateway/bin/src/db/event/config/mod.rs @@ -35,7 +35,7 @@ impl Config { BasicOutput::Valid(_) => return Ok(value), BasicOutput::Invalid(errors) => { // This should not happen, expecting the schema to be valid - error!(id=%id, errors=?errors, "Get Config, Schema validation failed, defaulting."); + error!(id=%id, error=?errors, "Get Config, schema validation failed, defaulting."); }, } } diff --git a/catalyst-gateway/bin/src/service/api/config/mod.rs b/catalyst-gateway/bin/src/service/api/config/mod.rs index 93618ce02a7..421b1de9ed1 100644 --- a/catalyst-gateway/bin/src/service/api/config/mod.rs +++ b/catalyst-gateway/bin/src/service/api/config/mod.rs @@ -6,7 +6,7 @@ use jsonschema::BasicOutput; use poem::web::RealIp; use poem_openapi::{param::Query, payload::Json, ApiResponse, Object, OpenApi}; use serde_json::{json, Value}; -use tracing::{error, info}; +use tracing::error; use crate::{ db::event::config::{key::ConfigKey, Config}, @@ -52,8 +52,6 @@ impl ConfigApi { operation_id = "get_config_frontend" )] async fn get_frontend(&self, ip_address: RealIp) -> AllResponses { - info!(id = "get_config_frontend", "IP Address: {:?}", ip_address.0); - // Fetch the general configuration let general_config = Config::get(ConfigKey::Frontend).await; @@ -62,7 +60,7 @@ impl ConfigApi { match Config::get(ConfigKey::FrontendForIp(ip)).await { Ok(value) => Some(value), Err(err) => { - error!(id="get_config_frontend", errors=?err, "Failed to get configuration for IP"); + error!(id="get_config_frontend_ip", error=?err, "Failed to get frontend configuration for IP"); return AllResponses::handle_error(&err); }, } @@ -84,7 +82,7 @@ impl ConfigApi { Responses::Ok(Json(response_config)).into() }, Err(err) => { - error!(id="get_config_frontend", errors=?err, "Failed to get general configuration"); + error!(id="get_config_frontend_general", error=?err, "Failed to get general frontend configuration"); AllResponses::handle_error(&err) }, } @@ -170,7 +168,7 @@ async fn set(key: ConfigKey, value: Value) -> AllResponses { } }, Err(err) => { - error!(id="put_config_frontend", errors=?err, "Failed to set configuration"); + error!(id="put_config_frontend", error=?err, "Failed to set frontend configuration"); AllResponses::handle_error(&err) }, } From 5ea12b9aab107e50c943d278c9858bdb33e2e93a Mon Sep 17 00:00:00 2001 From: bkioshn Date: Fri, 18 Oct 2024 13:30:04 +0700 Subject: [PATCH 37/37] fix: bump ci to v3.2.18 Signed-off-by: bkioshn --- Earthfile | 6 +++--- catalyst-gateway/Earthfile | 2 +- catalyst-gateway/event-db/Earthfile | 2 +- catalyst-gateway/tests/Earthfile | 2 +- catalyst-gateway/tests/api_tests/Earthfile | 2 +- catalyst_voices/Earthfile | 2 +- catalyst_voices/uikit_example/Earthfile | 2 +- .../catalyst_cardano/wallet-automation/Earthfile | 4 ++-- docs/Earthfile | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Earthfile b/Earthfile index abf0dbcb87e..10f2aff2b19 100644 --- a/Earthfile +++ b/Earthfile @@ -1,8 +1,8 @@ VERSION 0.8 -IMPORT github.com/input-output-hk/catalyst-ci/earthly/mdlint:v3.2.16 AS mdlint-ci -IMPORT github.com/input-output-hk/catalyst-ci/earthly/cspell:v3.2.16 AS cspell-ci -IMPORT github.com/input-output-hk/catalyst-ci/earthly/postgresql:v3.2.16 AS postgresql-ci +IMPORT github.com/input-output-hk/catalyst-ci/earthly/mdlint:v3.2.18 AS mdlint-ci +IMPORT github.com/input-output-hk/catalyst-ci/earthly/cspell:v3.2.18 AS cspell-ci +IMPORT github.com/input-output-hk/catalyst-ci/earthly/postgresql:v3.2.18 AS postgresql-ci FROM debian:stable-slim diff --git a/catalyst-gateway/Earthfile b/catalyst-gateway/Earthfile index 9eea78eadad..df800dad673 100644 --- a/catalyst-gateway/Earthfile +++ b/catalyst-gateway/Earthfile @@ -1,6 +1,6 @@ VERSION 0.8 -IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.2.17 AS rust-ci +IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.2.18 AS rust-ci #cspell: words rustfmt toolsets USERARCH stdcfgs diff --git a/catalyst-gateway/event-db/Earthfile b/catalyst-gateway/event-db/Earthfile index 27c8cf8c7e3..e5558970e90 100644 --- a/catalyst-gateway/event-db/Earthfile +++ b/catalyst-gateway/event-db/Earthfile @@ -3,7 +3,7 @@ # the database and its associated software. VERSION 0.8 -IMPORT github.com/input-output-hk/catalyst-ci/earthly/postgresql:v3.2.16 AS postgresql-ci +IMPORT github.com/input-output-hk/catalyst-ci/earthly/postgresql:v3.2.18 AS postgresql-ci # cspell: words diff --git a/catalyst-gateway/tests/Earthfile b/catalyst-gateway/tests/Earthfile index bfe2e1c591b..bc13dce3070 100644 --- a/catalyst-gateway/tests/Earthfile +++ b/catalyst-gateway/tests/Earthfile @@ -1,5 +1,5 @@ VERSION 0.8 -IMPORT github.com/input-output-hk/catalyst-ci/earthly/spectral:v3.2.16 AS spectral-ci +IMPORT github.com/input-output-hk/catalyst-ci/earthly/spectral:v3.2.18 AS spectral-ci # test-lint-openapi - OpenAPI linting from an artifact # testing whether the OpenAPI generated during build stage follows good practice. diff --git a/catalyst-gateway/tests/api_tests/Earthfile b/catalyst-gateway/tests/api_tests/Earthfile index dcc7a3e812f..5c3f9467c82 100644 --- a/catalyst-gateway/tests/api_tests/Earthfile +++ b/catalyst-gateway/tests/api_tests/Earthfile @@ -1,6 +1,6 @@ VERSION 0.8 -IMPORT github.com/input-output-hk/catalyst-ci/earthly/python:v3.2.16 AS python-ci +IMPORT github.com/input-output-hk/catalyst-ci/earthly/python:v3.2.18 AS python-ci builder: FROM python-ci+python-base diff --git a/catalyst_voices/Earthfile b/catalyst_voices/Earthfile index ed43d957179..e0b4b906339 100644 --- a/catalyst_voices/Earthfile +++ b/catalyst_voices/Earthfile @@ -1,7 +1,7 @@ VERSION 0.8 IMPORT ../catalyst-gateway AS catalyst-gateway -IMPORT github.com/input-output-hk/catalyst-ci/earthly/flutter:v3.2.16 AS flutter-ci +IMPORT github.com/input-output-hk/catalyst-ci/earthly/flutter:v3.2.18 AS flutter-ci # Copy all the necessary files and running bootstrap builder: diff --git a/catalyst_voices/uikit_example/Earthfile b/catalyst_voices/uikit_example/Earthfile index 59626cd2ade..46394953940 100644 --- a/catalyst_voices/uikit_example/Earthfile +++ b/catalyst_voices/uikit_example/Earthfile @@ -1,7 +1,7 @@ VERSION 0.8 IMPORT ../ AS catalyst-voices -IMPORT github.com/input-output-hk/catalyst-ci/earthly/flutter:v3.2.16 AS flutter-ci +IMPORT github.com/input-output-hk/catalyst-ci/earthly/flutter:v3.2.18 AS flutter-ci # local-build-web - build web version of UIKit example. # Prefixed by "local" to make sure it's not auto triggered, the target was diff --git a/catalyst_voices_packages/catalyst_cardano/catalyst_cardano/wallet-automation/Earthfile b/catalyst_voices_packages/catalyst_cardano/catalyst_cardano/wallet-automation/Earthfile index 57986f8d89a..cfd8644552b 100644 --- a/catalyst_voices_packages/catalyst_cardano/catalyst_cardano/wallet-automation/Earthfile +++ b/catalyst_voices_packages/catalyst_cardano/catalyst_cardano/wallet-automation/Earthfile @@ -1,6 +1,6 @@ VERSION 0.8 -IMPORT github.com/input-output-hk/catalyst-ci/earthly/flutter:v3.2.16 AS flutter-ci -IMPORT github.com/input-output-hk/catalyst-ci/earthly/playwright:v3.2.16 AS playwright-ci +IMPORT github.com/input-output-hk/catalyst-ci/earthly/flutter:v3.2.18 AS flutter-ci +IMPORT github.com/input-output-hk/catalyst-ci/earthly/playwright:v3.2.18 AS playwright-ci deps: DO playwright-ci+SETUP --workdir=/wallet-automation diff --git a/docs/Earthfile b/docs/Earthfile index 1bf657793d7..ef720f8a7ee 100644 --- a/docs/Earthfile +++ b/docs/Earthfile @@ -1,6 +1,6 @@ VERSION 0.8 -IMPORT github.com/input-output-hk/catalyst-ci/earthly/docs:v3.2.16 AS docs-ci +IMPORT github.com/input-output-hk/catalyst-ci/earthly/docs:v3.2.18 AS docs-ci IMPORT .. AS repo IMPORT ../catalyst-gateway AS catalyst-gateway