From 50a2d8a8797e1c5a1aa2efc80b0745cf1bb84151 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 17 Dec 2024 17:45:14 +0100 Subject: [PATCH 01/24] chore: rename 'digest' to 'digests' in 'ArtifactsLocations' entities --- mithril-aggregator/src/artifact_builder/cardano_database.rs | 4 ++-- mithril-common/src/entities/cardano_database.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mithril-aggregator/src/artifact_builder/cardano_database.rs b/mithril-aggregator/src/artifact_builder/cardano_database.rs index fd3590842c..7f6cd98869 100644 --- a/mithril-aggregator/src/artifact_builder/cardano_database.rs +++ b/mithril-aggregator/src/artifact_builder/cardano_database.rs @@ -64,7 +64,7 @@ impl ArtifactBuilder for CardanoDataba let locations = ArtifactsLocations { ancillary: vec![], - digest: vec![], + digests: vec![], immutables: vec![], }; @@ -193,7 +193,7 @@ mod tests { expected_total_size, ArtifactsLocations { ancillary: vec![], - digest: vec![], + digests: vec![], immutables: vec![], }, CompressionAlgorithm::Zstandard, diff --git a/mithril-common/src/entities/cardano_database.rs b/mithril-common/src/entities/cardano_database.rs index cbaf541837..f6ee441f5c 100644 --- a/mithril-common/src/entities/cardano_database.rs +++ b/mithril-common/src/entities/cardano_database.rs @@ -93,8 +93,8 @@ pub enum AncillaryLocation { /// Locations of the Cardano database related files. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct ArtifactsLocations { - /// Locations of the immutable file digests. - pub digest: Vec, + /// Locations of the immutable files digests. + pub digests: Vec, /// Locations of the immutable files. pub immutables: Vec, From cc0ae52dfdce0bb721ceca2fec2cc3420d2dda0b Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 17 Dec 2024 17:21:56 +0100 Subject: [PATCH 02/24] feat: add 'CardanoDatabaseSnapshotMessage' message --- .../http_server/routes/artifact_routes/mod.rs | 1 + .../src/messages/cardano_database.rs | 186 ++++++++++++++++++ mithril-common/src/messages/mod.rs | 2 + 3 files changed, 189 insertions(+) create mode 100644 mithril-common/src/messages/cardano_database.rs diff --git a/mithril-aggregator/src/http_server/routes/artifact_routes/mod.rs b/mithril-aggregator/src/http_server/routes/artifact_routes/mod.rs index 2d6b7b8a3c..8a20cec327 100644 --- a/mithril-aggregator/src/http_server/routes/artifact_routes/mod.rs +++ b/mithril-aggregator/src/http_server/routes/artifact_routes/mod.rs @@ -1,3 +1,4 @@ +pub mod cardano_database; pub mod cardano_stake_distribution; pub mod cardano_transaction; pub mod mithril_stake_distribution; diff --git a/mithril-common/src/messages/cardano_database.rs b/mithril-common/src/messages/cardano_database.rs new file mode 100644 index 0000000000..3fc56770e5 --- /dev/null +++ b/mithril-common/src/messages/cardano_database.rs @@ -0,0 +1,186 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::{ + entities::{ + AncillaryLocation, ArtifactsLocations, CompressionAlgorithm, DigestLocation, Epoch, + ImmutablesLocation, + }, + messages::CardanoDbBeaconMessagePart, +}; + +/// Locations of the Cardano database related files. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ArtifactsLocationsMessagePart { + /// Locations of the the immutable file digests. + pub digests: Vec, + /// Locations of the immutable files. + pub immutables: Vec, + /// Locations of the ancillary files. + pub ancillary: Vec, +} + +impl From for ArtifactsLocationsMessagePart { + fn from(part: ArtifactsLocations) -> Self { + Self { + digests: part.digests, + immutables: part.immutables, + ancillary: part.ancillary, + } + } +} + +/// Cardano database snapshot. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct CardanoDatabaseSnapshotMessage { + /// Merkle root of the Cardano database snapshot. + pub merkle_root: String, + + /// Mithril beacon on the Cardano chain. + pub beacon: CardanoDbBeaconMessagePart, + + /// Hash of the associated certificate + pub certificate_hash: String, + + /// Size of the uncompressed Cardano database files. + pub total_db_size_uncompressed: u64, + + /// Locations of the Cardano database artifacts. + pub locations: ArtifactsLocationsMessagePart, + + /// Compression algorithm of the Cardano database artifacts. + pub compression_algorithm: CompressionAlgorithm, + + /// Version of the Cardano node used to create the snapshot. + pub cardano_node_version: String, + + /// Date and time at which the snapshot was created + pub created_at: DateTime, +} + +impl CardanoDatabaseSnapshotMessage { + /// Return a dummy test entity (test-only). + pub fn dummy() -> Self { + Self { + merkle_root: "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6" + .to_string(), + beacon: CardanoDbBeaconMessagePart { + network: Some("preview".to_string()), + epoch: Epoch(123), + immutable_file_number: 2345, + }, + certificate_hash: "f6c01b373bafc4e039844071d5da3ace4a9c0745b9e9560e3e2af01823e9abfb" + .to_string(), + total_db_size_uncompressed: 800796318, + created_at: DateTime::parse_from_rfc3339("2023-01-19T13:43:05.618857482Z") + .unwrap() + .with_timezone(&Utc), + locations: ArtifactsLocationsMessagePart { + digests: vec![DigestLocation::Aggregator { + uri: "https://host-1/digest-1".to_string(), + }], + immutables: vec![ + ImmutablesLocation::CloudStorage { + uri: "https://host-1/immutables-2".to_string(), + }, + ImmutablesLocation::CloudStorage { + uri: "https://host-2/immutables-2".to_string(), + }, + ], + ancillary: vec![AncillaryLocation::CloudStorage { + uri: "https://host-1/ancillary-3".to_string(), + }], + }, + compression_algorithm: CompressionAlgorithm::Gzip, + cardano_node_version: "0.0.1".to_string(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn golden_message() -> CardanoDatabaseSnapshotMessage { + CardanoDatabaseSnapshotMessage { + merkle_root: "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6" + .to_string(), + beacon: CardanoDbBeaconMessagePart { + network: Some("preview".to_string()), + epoch: Epoch(123), + immutable_file_number: 2345, + }, + certificate_hash: "f6c01b373bafc4e039844071d5da3ace4a9c0745b9e9560e3e2af01823e9abfb" + .to_string(), + total_db_size_uncompressed: 800796318, + created_at: DateTime::parse_from_rfc3339("2023-01-19T13:43:05.618857482Z") + .unwrap() + .with_timezone(&Utc), + locations: ArtifactsLocationsMessagePart { + digests: vec![DigestLocation::Aggregator { + uri: "https://host-1/digest-1".to_string(), + }], + immutables: vec![ + ImmutablesLocation::CloudStorage { + uri: "https://host-1/immutables-2".to_string(), + }, + ImmutablesLocation::CloudStorage { + uri: "https://host-2/immutables-2".to_string(), + }, + ], + ancillary: vec![AncillaryLocation::CloudStorage { + uri: "https://host-1/ancillary-3".to_string(), + }], + }, + compression_algorithm: CompressionAlgorithm::Gzip, + cardano_node_version: "0.0.1".to_string(), + } + } + + // Test the backward compatibility with possible future upgrades. + #[test] + fn test_v1() { + let json = r#"{ + "merkle_root": "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6", + "beacon": { + "network": "preview", + "epoch": 123, + "immutable_file_number": 2345 + }, + "certificate_hash": "f6c01b373bafc4e039844071d5da3ace4a9c0745b9e9560e3e2af01823e9abfb", + "total_db_size_uncompressed": 800796318, + "locations": { + "digests": [ + { + "type": "aggregator", + "uri": "https://host-1/digest-1" + } + ], + "immutables": [ + { + "type": "cloud_storage", + "uri": "https://host-1/immutables-2" + }, + { + "type": "cloud_storage", + "uri": "https://host-2/immutables-2" + } + ], + "ancillary": [ + { + "type": "cloud_storage", + "uri": "https://host-1/ancillary-3" + } + ] + }, + "compression_algorithm": "gzip", + "cardano_node_version": "0.0.1", + "created_at": "2023-01-19T13:43:05.618857482Z" + }"#; + let message: CardanoDatabaseSnapshotMessage = serde_json::from_str(json).expect( + "This JSON is expected to be successfully parsed into a CardanoDatabaseSnapshotMessage instance.", + ); + + assert_eq!(golden_message(), message); + } +} diff --git a/mithril-common/src/messages/mod.rs b/mithril-common/src/messages/mod.rs index 286c61cfac..047a116bf3 100644 --- a/mithril-common/src/messages/mod.rs +++ b/mithril-common/src/messages/mod.rs @@ -2,6 +2,7 @@ //! This module aims at providing shared structures for API communications. mod aggregator_features; mod aggregator_status; +mod cardano_database; mod cardano_stake_distribution; mod cardano_stake_distribution_list; mod cardano_transaction_snapshot; @@ -25,6 +26,7 @@ pub use aggregator_features::{ AggregatorCapabilities, AggregatorFeaturesMessage, CardanoTransactionsProverCapabilities, }; pub use aggregator_status::AggregatorStatusMessage; +pub use cardano_database::CardanoDatabaseSnapshotMessage; pub use cardano_stake_distribution::CardanoStakeDistributionMessage; pub use cardano_stake_distribution_list::{ CardanoStakeDistributionListItemMessage, CardanoStakeDistributionListMessage, From 4902ce4bce2166f2ab663ff82cfd171b43f13545 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 17 Dec 2024 17:36:32 +0100 Subject: [PATCH 03/24] feat: add 'CardanoDatabaseSnapshotListMessage' message --- .../src/messages/cardano_database_list.rs | 106 ++++++++++++++++++ mithril-common/src/messages/mod.rs | 4 + 2 files changed, 110 insertions(+) create mode 100644 mithril-common/src/messages/cardano_database_list.rs diff --git a/mithril-common/src/messages/cardano_database_list.rs b/mithril-common/src/messages/cardano_database_list.rs new file mode 100644 index 0000000000..e35ba6f1b1 --- /dev/null +++ b/mithril-common/src/messages/cardano_database_list.rs @@ -0,0 +1,106 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::entities::{CompressionAlgorithm, Epoch}; +use crate::messages::CardanoDbBeaconMessagePart; + +/// Message structure of a snapshot list +pub type CardanoDatabaseSnapshotListMessage = Vec; + +/// Message structure of a snapshot list item +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct CardanoDatabaseSnapshotListItemMessage { + /// Merkle root of the Cardano database snapshot + pub merkle_root: String, + + /// Mithril beacon on the Cardano chain + pub beacon: CardanoDbBeaconMessagePart, + + /// Hash of the associated certificate + pub certificate_hash: String, + + /// Size of the uncompressed Cardano database files. + pub total_db_size_uncompressed: u64, + + /// Compression algorithm of the Cardano database artifacts. + pub compression_algorithm: CompressionAlgorithm, + + /// Version of the Cardano node used to create the snapshot. + pub cardano_node_version: String, + + /// Date and time at which the snapshot was created + pub created_at: DateTime, +} + +impl CardanoDatabaseSnapshotListItemMessage { + /// Return a dummy test entity (test-only). + pub fn dummy() -> Self { + Self { + merkle_root: "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6" + .to_string(), + beacon: CardanoDbBeaconMessagePart { + network: Some("preview".to_string()), + epoch: Epoch(123), + immutable_file_number: 2345, + }, + certificate_hash: "f6c01b373bafc4e039844071d5da3ace4a9c0745b9e9560e3e2af01823e9abfb" + .to_string(), + total_db_size_uncompressed: 800796318, + created_at: DateTime::parse_from_rfc3339("2023-01-19T13:43:05.618857482Z") + .unwrap() + .with_timezone(&Utc), + compression_algorithm: CompressionAlgorithm::default(), + cardano_node_version: "0.0.1".to_string(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn golden_message() -> CardanoDatabaseSnapshotListMessage { + vec![CardanoDatabaseSnapshotListItemMessage { + merkle_root: "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6" + .to_string(), + beacon: CardanoDbBeaconMessagePart { + network: Some("preview".to_string()), + epoch: Epoch(123), + immutable_file_number: 2345, + }, + certificate_hash: "f6c01b373bafc4e039844071d5da3ace4a9c0745b9e9560e3e2af01823e9abfb" + .to_string(), + total_db_size_uncompressed: 800796318, + created_at: DateTime::parse_from_rfc3339("2023-01-19T13:43:05.618857482Z") + .unwrap() + .with_timezone(&Utc), + compression_algorithm: CompressionAlgorithm::default(), + cardano_node_version: "0.0.1".to_string(), + }] + } + + // Test the backward compatibility with possible future upgrades. + #[test] + fn test_v1() { + let json = r#"[ + { + "merkle_root": "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6", + "beacon": { + "network": "preview", + "epoch": 123, + "immutable_file_number": 2345 + }, + "certificate_hash": "f6c01b373bafc4e039844071d5da3ace4a9c0745b9e9560e3e2af01823e9abfb", + "total_db_size_uncompressed": 800796318, + "compression_algorithm": "gzip", + "cardano_node_version": "0.0.1", + "created_at": "2023-01-19T13:43:05.618857482Z" + } + ]"#; + let message: CardanoDatabaseSnapshotListMessage = serde_json::from_str(json).expect( + "This JSON is expected to be successfully parsed into a CardanoDatabaseSnapshotListMessage instance.", + ); + + assert_eq!(golden_message(), message); + } +} diff --git a/mithril-common/src/messages/mod.rs b/mithril-common/src/messages/mod.rs index 047a116bf3..5354a6bf63 100644 --- a/mithril-common/src/messages/mod.rs +++ b/mithril-common/src/messages/mod.rs @@ -3,6 +3,7 @@ mod aggregator_features; mod aggregator_status; mod cardano_database; +mod cardano_database_list; mod cardano_stake_distribution; mod cardano_stake_distribution_list; mod cardano_transaction_snapshot; @@ -27,6 +28,9 @@ pub use aggregator_features::{ }; pub use aggregator_status::AggregatorStatusMessage; pub use cardano_database::CardanoDatabaseSnapshotMessage; +pub use cardano_database_list::{ + CardanoDatabaseSnapshotListItemMessage, CardanoDatabaseSnapshotListMessage, +}; pub use cardano_stake_distribution::CardanoStakeDistributionMessage; pub use cardano_stake_distribution_list::{ CardanoStakeDistributionListItemMessage, CardanoStakeDistributionListMessage, From 379a54fabe790b3b7534949070604d6df30d04e8 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 17 Dec 2024 18:09:02 +0100 Subject: [PATCH 04/24] chore: make artifact location types public --- mithril-common/src/entities/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mithril-common/src/entities/mod.rs b/mithril-common/src/entities/mod.rs index a0b36e9678..e98b700347 100644 --- a/mithril-common/src/entities/mod.rs +++ b/mithril-common/src/entities/mod.rs @@ -35,6 +35,7 @@ pub use block_range::{BlockRange, BlockRangeLength, BlockRangesSequence}; pub use cardano_chain_point::{BlockHash, ChainPoint}; pub use cardano_database::{ AncillaryLocation, AncillaryLocationDiscriminants, ArtifactsLocations, CardanoDatabaseSnapshot, + DigestLocation, ImmutablesLocation, }; pub use cardano_db_beacon::CardanoDbBeacon; pub use cardano_network::CardanoNetwork; From 5972c29b9eed3f508c705676a764ed39a157dbd1 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 17 Dec 2024 18:39:01 +0100 Subject: [PATCH 05/24] feat: add conversion from 'CardanoDBBeacon' to 'CardanoDBBeaconMessagePart' --- .../messages/message_parts/signed_entity_type_message.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mithril-common/src/messages/message_parts/signed_entity_type_message.rs b/mithril-common/src/messages/message_parts/signed_entity_type_message.rs index 070491c568..e0b051eaf4 100644 --- a/mithril-common/src/messages/message_parts/signed_entity_type_message.rs +++ b/mithril-common/src/messages/message_parts/signed_entity_type_message.rs @@ -80,6 +80,12 @@ impl From for CardanoDbBeacon { } } +impl From for CardanoDbBeaconMessagePart { + fn from(beacon: CardanoDbBeacon) -> Self { + CardanoDbBeaconMessagePart::new_without_network(beacon.epoch, beacon.immutable_file_number) + } +} + impl> From<(CardanoDbBeacon, N)> for CardanoDbBeaconMessagePart { fn from((beacon, network): (CardanoDbBeacon, N)) -> Self { CardanoDbBeaconMessagePart::new(network.into(), beacon.epoch, beacon.immutable_file_number) From 07825c7453336f60db8c0aef5e42f64f72b31ca0 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 17 Dec 2024 18:40:09 +0100 Subject: [PATCH 06/24] feat: add conversion from 'SignedEntityRecord' to 'CardanoDatabaseSnapshotMessage' and 'CardanoDatabaseSnapshotListItemMessage' --- .../src/database/record/signed_entity.rs | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/mithril-aggregator/src/database/record/signed_entity.rs b/mithril-aggregator/src/database/record/signed_entity.rs index eee7bf85c1..04c2f680b4 100644 --- a/mithril-aggregator/src/database/record/signed_entity.rs +++ b/mithril-aggregator/src/database/record/signed_entity.rs @@ -3,11 +3,13 @@ use serde::{Deserialize, Serialize}; use mithril_common::crypto_helper::ProtocolParameters; use mithril_common::entities::{ - BlockNumber, Epoch, SignedEntity, SignedEntityType, Snapshot, StakeDistribution, + BlockNumber, CardanoDatabaseSnapshot, Epoch, SignedEntity, SignedEntityType, Snapshot, + StakeDistribution, }; #[cfg(test)] use mithril_common::entities::{CardanoStakeDistribution, MithrilStakeDistribution}; use mithril_common::messages::{ + CardanoDatabaseSnapshotListItemMessage, CardanoDatabaseSnapshotMessage, CardanoStakeDistributionListItemMessage, CardanoStakeDistributionMessage, CardanoTransactionSnapshotListItemMessage, CardanoTransactionSnapshotMessage, MithrilStakeDistributionListItemMessage, MithrilStakeDistributionMessage, @@ -169,6 +171,45 @@ impl TryFrom for SnapshotMessage { } } +impl TryFrom for CardanoDatabaseSnapshotMessage { + type Error = StdError; + + fn try_from(value: SignedEntityRecord) -> Result { + let artifact = serde_json::from_str::(&value.artifact)?; + let cardano_database_snapshot_message = CardanoDatabaseSnapshotMessage { + merkle_root: artifact.merkle_root, + beacon: artifact.beacon.into(), + certificate_hash: value.certificate_id, + total_db_size_uncompressed: artifact.total_db_size_uncompressed, + created_at: value.created_at, + locations: artifact.locations.into(), + compression_algorithm: artifact.compression_algorithm, + cardano_node_version: artifact.cardano_node_version, + }; + + Ok(cardano_database_snapshot_message) + } +} + +impl TryFrom for CardanoDatabaseSnapshotListItemMessage { + type Error = StdError; + + fn try_from(value: SignedEntityRecord) -> Result { + let artifact = serde_json::from_str::(&value.artifact)?; + let cardano_database_snapshot_list_item_message = CardanoDatabaseSnapshotListItemMessage { + merkle_root: artifact.merkle_root, + beacon: artifact.beacon.into(), + certificate_hash: value.certificate_id, + total_db_size_uncompressed: artifact.total_db_size_uncompressed, + created_at: value.created_at, + compression_algorithm: artifact.compression_algorithm, + cardano_node_version: artifact.cardano_node_version, + }; + + Ok(cardano_database_snapshot_list_item_message) + } +} + impl TryFrom for MithrilStakeDistributionMessage { type Error = StdError; From a39e76dd2714724f9d4a7d0e9c5e46c93f68d673 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 17 Dec 2024 18:43:17 +0100 Subject: [PATCH 07/24] feat: compute Cardano database messages in message service --- .../http_server/routes/artifact_routes/mod.rs | 1 - mithril-aggregator/src/services/message.rs | 111 +++++++++++++++++- 2 files changed, 105 insertions(+), 7 deletions(-) diff --git a/mithril-aggregator/src/http_server/routes/artifact_routes/mod.rs b/mithril-aggregator/src/http_server/routes/artifact_routes/mod.rs index 8a20cec327..2d6b7b8a3c 100644 --- a/mithril-aggregator/src/http_server/routes/artifact_routes/mod.rs +++ b/mithril-aggregator/src/http_server/routes/artifact_routes/mod.rs @@ -1,4 +1,3 @@ -pub mod cardano_database; pub mod cardano_stake_distribution; pub mod cardano_transaction; pub mod mithril_stake_distribution; diff --git a/mithril-aggregator/src/services/message.rs b/mithril-aggregator/src/services/message.rs index 00c0b8a37c..d6c610d436 100644 --- a/mithril-aggregator/src/services/message.rs +++ b/mithril-aggregator/src/services/message.rs @@ -8,6 +8,7 @@ use thiserror::Error; use mithril_common::{ entities::{Epoch, SignedEntityTypeDiscriminants}, messages::{ + CardanoDatabaseSnapshotListMessage, CardanoDatabaseSnapshotMessage, CardanoStakeDistributionListMessage, CardanoStakeDistributionMessage, CardanoTransactionSnapshotListMessage, CardanoTransactionSnapshotMessage, CertificateListMessage, CertificateMessage, MithrilStakeDistributionListMessage, @@ -35,11 +36,11 @@ pub trait MessageService: Sync + Send { certificate_hash: &str, ) -> StdResult>; - /// Return the message representation of the last N certificates + /// Return the message representation of the last N certificates. async fn get_certificate_list_message(&self, limit: usize) -> StdResult; - /// Return the information regarding the given snapshot + /// Return the information regarding the given snapshot. async fn get_snapshot_message( &self, signed_entity_id: &str, @@ -49,13 +50,25 @@ pub trait MessageService: Sync + Send { /// passed as argument. async fn get_snapshot_list_message(&self, limit: usize) -> StdResult; - /// Return the information regarding the MSD for the given identifier. + /// Return the information regarding the Cardano database for the given identifier. + async fn get_cardano_database_message( + &self, + signed_entity_id: &str, + ) -> StdResult>; + + /// Return the list of the last Cardano database message. + async fn get_cardano_database_list_message( + &self, + limit: usize, + ) -> StdResult; + + /// Return the information regarding the Mithril stake distribution for the given identifier. async fn get_mithril_stake_distribution_message( &self, signed_entity_id: &str, ) -> StdResult>; - /// Return the list of the last Mithril stake distributions message + /// Return the list of the last Mithril stake distributions message. async fn get_mithril_stake_distribution_list_message( &self, limit: usize, @@ -67,7 +80,7 @@ pub trait MessageService: Sync + Send { signed_entity_id: &str, ) -> StdResult>; - /// Return the list of the last Cardano transactions set message + /// Return the list of the last Cardano transactions set message. async fn get_cardano_transaction_list_message( &self, limit: usize, @@ -85,7 +98,7 @@ pub trait MessageService: Sync + Send { epoch: Epoch, ) -> StdResult>; - /// Return the list of the last Cardano stake distributions message + /// Return the list of the last Cardano stake distributions message. async fn get_cardano_stake_distribution_list_message( &self, limit: usize, @@ -153,6 +166,31 @@ impl MessageService for MithrilMessageService { entities.into_iter().map(|i| i.try_into()).collect() } + async fn get_cardano_database_message( + &self, + signed_entity_id: &str, + ) -> StdResult> { + let signed_entity = self + .signed_entity_storer + .get_signed_entity(signed_entity_id) + .await?; + + signed_entity.map(|v| v.try_into()).transpose() + } + + async fn get_cardano_database_list_message( + &self, + limit: usize, + ) -> StdResult { + let signed_entity_type_id = SignedEntityTypeDiscriminants::CardanoDatabase; + let entities = self + .signed_entity_storer + .get_last_signed_entities_by_type(&signed_entity_type_id, limit) + .await?; + + entities.into_iter().map(|i| i.try_into()).collect() + } + async fn get_mithril_stake_distribution_message( &self, signed_entity_id: &str, @@ -410,6 +448,67 @@ mod tests { } } + mod cardano_database { + use super::*; + + #[tokio::test] + async fn get_cardano_database_not_exist() { + let service = MessageServiceBuilder::new().build().await; + let snapshot = service.get_snapshot_message("whatever").await.unwrap(); + + assert!(snapshot.is_none()); + } + + #[tokio::test] + async fn get_cardano_database() { + let record = SignedEntityRecord { + signed_entity_id: "signed_entity_id".to_string(), + signed_entity_type: SignedEntityType::CardanoDatabase(fake_data::beacon()), + certificate_id: "cert_id".to_string(), + artifact: serde_json::to_string(&fake_data::cardano_database_snapshots(1)[0]) + .unwrap(), + created_at: Default::default(), + }; + let message: CardanoDatabaseSnapshotMessage = record.clone().try_into().unwrap(); + + let service = MessageServiceBuilder::new() + .with_signed_entity_records(&[record.clone()]) + .build() + .await; + + let response = service + .get_cardano_database_message(&record.signed_entity_id) + .await + .unwrap() + .expect("A CardanoDatabaseSnapshotMessage was expected."); + + assert_eq!(message, response); + } + + #[tokio::test] + async fn get_cardano_database_list_message() { + let record = SignedEntityRecord { + signed_entity_id: "signed_entity_id".to_string(), + signed_entity_type: SignedEntityType::CardanoDatabase(fake_data::beacon()), + certificate_id: "cert_id".to_string(), + artifact: serde_json::to_string(&fake_data::cardano_database_snapshots(1)[0]) + .unwrap(), + created_at: Default::default(), + }; + let message: CardanoDatabaseSnapshotListMessage = + vec![record.clone().try_into().unwrap()]; + + let service = MessageServiceBuilder::new() + .with_signed_entity_records(&[record]) + .build() + .await; + + let response = service.get_cardano_database_list_message(3).await.unwrap(); + + assert_eq!(message, response); + } + } + mod mithril_stake_distribution { use super::*; From 934fce7f122db52384fe29dfcb53757c5db411f6 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 18 Dec 2024 14:43:03 +0100 Subject: [PATCH 08/24] feat: add list and detail retrieval for 'CardanoDatabase' in signed entity service --- .../src/services/signed_entity.rs | 118 ++++++++++++------ 1 file changed, 83 insertions(+), 35 deletions(-) diff --git a/mithril-aggregator/src/services/signed_entity.rs b/mithril-aggregator/src/services/signed_entity.rs index 83dacd8d38..f447da8e4e 100644 --- a/mithril-aggregator/src/services/signed_entity.rs +++ b/mithril-aggregator/src/services/signed_entity.rs @@ -44,23 +44,30 @@ pub trait SignedEntityService: Send + Sync { total: usize, ) -> StdResult>>; - /// Return a list of signed Mithril stake distribution order by creation - /// date descending. - async fn get_last_signed_mithril_stake_distributions( + /// Return a signed snapshot + async fn get_signed_snapshot_by_id( &self, - total: usize, - ) -> StdResult>>; + signed_entity_id: &str, + ) -> StdResult>>; - /// Return the last signed Cardano Transaction Snapshot. - async fn get_last_cardano_transaction_snapshot( + /// Return a list of Cardano Database snapshots order by creation date descending. + async fn get_last_signed_cardano_database_snapshots( &self, - ) -> StdResult>>; + total: usize, + ) -> StdResult>>; - /// Return a signed snapshot - async fn get_signed_snapshot_by_id( + /// Return a Cardano Database snapshot + async fn get_signed_cardano_database_snapshot_by_id( &self, signed_entity_id: &str, - ) -> StdResult>>; + ) -> StdResult>>; + + /// Return a list of signed Mithril stake distribution ordered by creation + /// date descending. + async fn get_last_signed_mithril_stake_distributions( + &self, + total: usize, + ) -> StdResult>>; /// Return a signed Mithril stake distribution async fn get_signed_mithril_stake_distribution_by_id( @@ -68,7 +75,12 @@ pub trait SignedEntityService: Send + Sync { signed_entity_id: &str, ) -> StdResult>>; - /// Return a list of signed Cardano stake distribution order by creation + /// Return the last signed Cardano Transaction Snapshot. + async fn get_last_cardano_transaction_snapshot( + &self, + ) -> StdResult>>; + + /// Return a list of signed Cardano stake distribution ordered by creation /// date descending. async fn get_last_signed_cardano_stake_distributions( &self, @@ -376,17 +388,34 @@ impl SignedEntityService for MithrilSignedEntityService { Ok(signed_entities) } - async fn get_last_signed_mithril_stake_distributions( + async fn get_signed_snapshot_by_id( + &self, + signed_entity_id: &str, + ) -> StdResult>> { + let entity: Option> = match self + .signed_entity_storer + .get_signed_entity(signed_entity_id) + .await + .with_context(|| { + format!( + "Signed Entity Service can not get signed entity with id: '{signed_entity_id}'" + ) + })? { + Some(entity) => Some(entity.try_into()?), + None => None, + }; + + Ok(entity) + } + + async fn get_last_signed_cardano_database_snapshots( &self, total: usize, - ) -> StdResult>> { + ) -> StdResult>> { let signed_entities_records = self - .get_last_signed_entities( - total, - &SignedEntityTypeDiscriminants::MithrilStakeDistribution, - ) + .get_last_signed_entities(total, &SignedEntityTypeDiscriminants::CardanoDatabase) .await?; - let mut signed_entities: Vec> = Vec::new(); + let mut signed_entities: Vec> = Vec::new(); for record in signed_entities_records { signed_entities.push(record.try_into()?); @@ -395,24 +424,11 @@ impl SignedEntityService for MithrilSignedEntityService { Ok(signed_entities) } - async fn get_last_cardano_transaction_snapshot( - &self, - ) -> StdResult>> { - let mut signed_entities_records = self - .get_last_signed_entities(1, &SignedEntityTypeDiscriminants::CardanoTransactions) - .await?; - - match signed_entities_records.pop() { - Some(record) => Ok(Some(record.try_into()?)), - None => Ok(None), - } - } - - async fn get_signed_snapshot_by_id( + async fn get_signed_cardano_database_snapshot_by_id( &self, signed_entity_id: &str, - ) -> StdResult>> { - let entity: Option> = match self + ) -> StdResult>> { + let entity: Option> = match self .signed_entity_storer .get_signed_entity(signed_entity_id) .await @@ -428,6 +444,25 @@ impl SignedEntityService for MithrilSignedEntityService { Ok(entity) } + async fn get_last_signed_mithril_stake_distributions( + &self, + total: usize, + ) -> StdResult>> { + let signed_entities_records = self + .get_last_signed_entities( + total, + &SignedEntityTypeDiscriminants::MithrilStakeDistribution, + ) + .await?; + let mut signed_entities: Vec> = Vec::new(); + + for record in signed_entities_records { + signed_entities.push(record.try_into()?); + } + + Ok(signed_entities) + } + async fn get_signed_mithril_stake_distribution_by_id( &self, signed_entity_id: &str, @@ -448,6 +483,19 @@ impl SignedEntityService for MithrilSignedEntityService { Ok(entity) } + async fn get_last_cardano_transaction_snapshot( + &self, + ) -> StdResult>> { + let mut signed_entities_records = self + .get_last_signed_entities(1, &SignedEntityTypeDiscriminants::CardanoTransactions) + .await?; + + match signed_entities_records.pop() { + Some(record) => Ok(Some(record.try_into()?)), + None => Ok(None), + } + } + async fn get_last_signed_cardano_stake_distributions( &self, total: usize, From 892839357a3e466c5e85bd7a7b14b444130b2fba Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 18 Dec 2024 18:01:32 +0100 Subject: [PATCH 09/24] feat: update OpenAPI for 'CardanoDatabase' artifact routes --- openapi.yaml | 262 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 262 insertions(+) diff --git a/openapi.yaml b/openapi.yaml index b78d2ba54e..527a913661 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -276,6 +276,59 @@ paths: schema: $ref: "#/components/schemas/Error" + /artifact/cardano-database: + get: + summary: Get most recent Cardano database snapshots + description: | + Returns the list of the most recent Cardano database snapshots + responses: + "200": + description: Cardano database snapshots found + content: + application/json: + schema: + $ref: "#/components/schemas/CardanoDatabaseSnapshotListMessage" + "412": + description: API version mismatch + default: + description: Cardano database snapshots retrieval error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + + /artifact/cardano-database/{merkle_root}: + get: + summary: Get Cardano database snapshot information + description: | + Returns the information of a Cardano database snapshot and where to retrieve its binary contents + parameters: + - name: merkle_root + in: path + description: Merkle root of the Cardano database snapshot + required: true + schema: + type: string + format: bytes + examples: "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6" + responses: + "200": + description: Cardano database snapshot found + content: + application/json: + schema: + $ref: "#/components/schemas/CardanoDatabaseSnapshotMessage" + "404": + description: Cardano database snapshot not found + "412": + description: API version mismatch + default: + description: Cardano database snapshot retrieval error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + /artifact/mithril-stake-distributions: get: summary: Get most recent Mithril stake distributions @@ -1786,6 +1839,215 @@ components: "cardano_node_version": "1.0.0" } + CardanoDatabaseArtifactsLocationsMessagePart: + description: CardanoDatabaseArtifactsLocationsMessagePart represents the locations of the Cardano database artifacts + type: object + properties: + digests: + type: array + items: + $ref: "#/components/schemas/CardanoDatabaseArtifactLocationMessagePart" + immutables: + type: array + items: + $ref: "#/components/schemas/CardanoDatabaseArtifactLocationMessagePart" + ancillary: + type: array + items: + $ref: "#/components/schemas/CardanoDatabaseArtifactLocationMessagePart" + examples: + [ + { + "merkle_root": "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6", + "beacon": + { + "network": "preview", + "epoch": 123, + "immutable_file_number": 2345 + }, + "certificate_hash": "f6c01b373bafc4e039844071d5da3ace4a9c0745b9e9560e3e2af01823e9abfb", + "total_db_size_uncompressed": 800796318, + "compression_algorithm": "gzip", + "cardano_node_version": "0.0.1", + "created_at": "2023-01-19T13:43:05.618857482Z" + } + ] + + CardanoDatabaseArtifactLocationMessagePart: + description: CardanoDatabaseArtifactLocationMessagePart represents the location of a single Cardano database artifact + type: object + required: + - type + - uri + properties: + type: + description: Type of the artifact location + type: string + uri: + description: URI of the artifact location + type: string + examples: + { + "type": "cloud_storage", + "uri": "https://mithril-cdn-us.iohk.io/snapshot/digests-6367ee65d0d1272e6e70736a1ea2cae34015874517f6328364f6b73930966732.txt" + } + + CardanoDatabaseSnapshotListMessage: + description: CardanoDatabaseSnapshotListMessage represents a list of Cardano database snapshots + type: array + items: + $ref: "#/components/schemas/CardanoDatabaseSnapshotListItemMessage" + examples: + [ + { + "merkle_root": "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6", + "beacon": + { + "network": "preview", + "epoch": 123, + "immutable_file_number": 2345 + }, + "certificate_hash": "f6c01b373bafc4e039844071d5da3ace4a9c0745b9e9560e3e2af01823e9abfb", + "total_db_size_uncompressed": 800796318, + "compression_algorithm": "gzip", + "cardano_node_version": "0.0.1", + "created_at": "2023-01-19T13:43:05.618857482Z" + } + ] + + CardanoDatabaseSnapshotListItemMessage: + description: CardanoDatabaseSnapshotListItemMessage represents a list item of Cardano database snapshots + type: object + additionalProperties: false + required: + - merkle_root + - beacon + - certificate_hash + - total_db_size_uncompressed + - created_at + properties: + merkle_root: + description: Merkle root of the Cardano database snapshot + type: string + format: bytes + beacon: + $ref: "#/components/schemas/CardanoDbBeacon" + certificate_hash: + description: Hash of the associated certificate + type: string + format: bytes + total_db_size_uncompressed: + description: Size of the uncompressed Cardano database files in Bytes + type: integer + format: int64 + created_at: + description: Date and time at which the snapshot was created + type: string + format: date-time + compression_algorithm: + description: Compression algorithm of the Cardano database artifacts + type: string + cardano_node_version: + description: Version of the Cardano node which is used to create snapshot archives. + type: string + examples: + { + "merkle_root": "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6", + "beacon": + { + "network": "preview", + "epoch": 123, + "immutable_file_number": 2345 + }, + "certificate_hash": "f6c01b373bafc4e039844071d5da3ace4a9c0745b9e9560e3e2af01823e9abfb", + "total_db_size_uncompressed": 800796318, + "compression_algorithm": "zstandard", + "cardano_node_version": "0.0.1", + "created_at": "2023-01-19T13:43:05.618857482Z" + } + + CardanoDatabaseSnapshotMessage: + description: CardanoDatabaseSnapshot represents a Cardano database snapshot and its metadata + type: object + additionalProperties: false + required: + - merkle_root + - beacon + - certificate_hash + - total_db_size_uncompressed + - created_at + - locations + properties: + merkle_root: + description: Merkle root of the Cardano database snapshot + type: string + format: bytes + beacon: + $ref: "#/components/schemas/CardanoDbBeacon" + certificate_hash: + description: Hash of the associated certificate + type: string + format: bytes + total_db_size_uncompressed: + description: Size of the uncompressed Cardano database files in Bytes + type: integer + format: int64 + created_at: + description: Date and time at which the snapshot was created + type: string + format: date-time + locations: + $ref: "#/components/schemas/CardanoDatabaseArtifactsLocationsMessagePart" + compression_algorithm: + description: Compression algorithm of the Cardano database artifacts + type: string + cardano_node_version: + description: Version of the Cardano node which is used to create snapshot archives. + type: string + examples: + { + "merkle_root": "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6", + "beacon": + { + "network": "preview", + "epoch": 123, + "immutable_file_number": 2345 + }, + "certificate_hash": "f6c01b373bafc4e039844071d5da3ace4a9c0745b9e9560e3e2af01823e9abfb", + "total_db_size_uncompressed": 800796318, + "locations": + { + "digests": + [ + { + "type": "cloud_storage", + "uri": "https://mithril-cdn-us.iohk.io/snapshot/digests-6367ee65d0d1272e6e70736a1ea2cae34015874517f6328364f6b73930966732.txt" + } + ], + "immutables": + [ + { + "type": "cloud_storage", + "uri": "https://mithril-cdn-us.iohk.io/snapshot/immutables-6367ee65d0d1272e6e70736a1ea2cae34015874517f6328364f6b73930966732-{immutable-file_number}.tar.zst" + }, + { + "type": "cloud_storage", + "uri": "https://mithril-cdn-eu.iohk.io/snapshot/immutables-6367ee65d0d1272e6e70736a1ea2cae34015874517f6328364f6b73930966732-{immutable-file_number}.tar.zst" + } + ], + "ancillary": + [ + { + "type": "cloud_storage", + "uri": "https://mithril-cdn-us.iohk.io/snapshot/ancillary-6367ee65d0d1272e6e70736a1ea2cae34015874517f6328364f6b73930966732.tar.zst" + } + ] + }, + "compression_algorithm": "zstandard", + "cardano_node_version": "0.0.1", + "created_at": "2023-01-19T13:43:05.618857482Z" + } + MithrilStakeDistributionListMessage: description: MithrilStakeDistributionListMessage represents a list of Mithril stake distribution type: array From 84f516e0e932c410d5daa9d2237c4b362e1c1314 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 18 Dec 2024 18:01:47 +0100 Subject: [PATCH 10/24] feat: implement 'CardanoDatabase' artifact routes --- .../artifact_routes/cardano_database.rs | 358 ++++++++++++++++++ .../http_server/routes/artifact_routes/mod.rs | 1 + .../src/http_server/routes/router.rs | 1 + 3 files changed, 360 insertions(+) create mode 100644 mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs diff --git a/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs b/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs new file mode 100644 index 0000000000..fe84f6cb71 --- /dev/null +++ b/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs @@ -0,0 +1,358 @@ +use crate::http_server::routes::middlewares; +use crate::http_server::routes::router::RouterState; +use warp::Filter; + +pub fn routes( + router_state: &RouterState, +) -> impl Filter + Clone { + artifact_cardano_database_list(router_state) + .or(artifact_cardano_database_by_id(router_state)) + .or(serve_cardano_database_dir(router_state)) +} + +/// GET /artifact/cardano-database +fn artifact_cardano_database_list( + router_state: &RouterState, +) -> impl Filter + Clone { + warp::path!("artifact" / "cardano-database") + .and(warp::get()) + .and(middlewares::with_logger(router_state)) + .and(middlewares::with_http_message_service(router_state)) + .and_then(handlers::list_artifacts) +} + +/// GET /artifact/cardano-database/:id +fn artifact_cardano_database_by_id( + dependency_manager: &RouterState, +) -> impl Filter + Clone { + warp::path!("artifact" / "cardano-database" / String) + .and(warp::get()) + .and(middlewares::with_logger(dependency_manager)) + .and(middlewares::with_http_message_service(dependency_manager)) + .and(middlewares::with_metrics_service(dependency_manager)) + .and_then(handlers::get_artifact_by_signed_entity_id) +} + +fn serve_cardano_database_dir( + router_state: &RouterState, +) -> impl Filter + Clone { + warp::path("cardano_database_download") + .and(warp::fs::dir( + router_state.configuration.snapshot_directory.clone(), + )) + .and(middlewares::with_logger(router_state)) + .and_then(handlers::ensure_downloaded_file_is_a_cardano_database) +} + +mod handlers { + use crate::http_server::routes::reply; + use crate::services::MessageService; + use crate::MetricsService; + use slog::{debug, warn, Logger}; + use std::convert::Infallible; + use std::sync::Arc; + use warp::http::StatusCode; + + pub const LIST_MAX_ITEMS: usize = 20; + + /// List artifacts + pub async fn list_artifacts( + logger: Logger, + http_message_service: Arc, + ) -> Result { + match http_message_service + .get_cardano_database_list_message(LIST_MAX_ITEMS) + .await + { + Ok(message) => Ok(reply::json(&message, StatusCode::OK)), + Err(err) => { + warn!(logger,"list_artifacts_cardano_database"; "error" => ?err); + Ok(reply::server_error(err)) + } + } + } + + /// Get artifact by signed entity id + pub async fn get_artifact_by_signed_entity_id( + signed_entity_id: String, + logger: Logger, + http_message_service: Arc, + metrics_service: Arc, + ) -> Result { + metrics_service + .get_artifact_detail_cardano_db_total_served_since_startup() + .increment(); + + match http_message_service + .get_cardano_database_message(&signed_entity_id) + .await + { + Ok(Some(signed_entity)) => Ok(reply::json(&signed_entity, StatusCode::OK)), + Ok(None) => { + warn!(logger, "cardano_database_details::not_found"); + Ok(reply::empty(StatusCode::NOT_FOUND)) + } + Err(err) => { + warn!(logger,"cardano_database_details::error"; "error" => ?err); + Ok(reply::server_error(err)) + } + } + } + + /// Download a file if it's a Cardano_database archive file + pub async fn ensure_downloaded_file_is_a_cardano_database( + reply: warp::fs::File, + logger: Logger, + ) -> Result { + let filepath = reply.path().to_path_buf(); + debug!( + logger, + ">> ensure_downloaded_file_is_a_cardano_database / file: `{}`", + filepath.display() + ); + + // TODO: enhance this check with a regular expression once the file naming convention is defined + let file_is_a_cardano_database_archive = filepath.to_string_lossy().contains("ancillary") + || filepath.to_string_lossy().contains("immutable"); + match file_is_a_cardano_database_archive { + true => Ok(Box::new(warp::reply::with_header( + reply, + "Content-Disposition", + format!( + "attachment; filename=\"{}\"", + filepath.file_name().unwrap().to_str().unwrap() + ), + )) as Box), + false => { + warn!(logger,"ensure_downloaded_file_is_a_cardano_database::error"; "error" => "file is not a Cardano database archive"); + Ok(reply::empty(StatusCode::NOT_FOUND)) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + http_server::SERVER_BASE_PATH, initialize_dependencies, services::MockMessageService, + }; + use mithril_common::messages::{ + CardanoDatabaseSnapshotListItemMessage, CardanoDatabaseSnapshotMessage, + }; + use mithril_common::test_utils::apispec::APISpec; + use mithril_persistence::sqlite::HydrationError; + use serde_json::Value::Null; + use std::sync::Arc; + use warp::{ + http::{Method, StatusCode}, + test::request, + }; + + fn setup_router( + state: RouterState, + ) -> impl Filter + Clone { + let cors = warp::cors() + .allow_any_origin() + .allow_headers(vec!["content-type"]) + .allow_methods(vec![Method::GET, Method::POST, Method::OPTIONS]); + + warp::any() + .and(warp::path(SERVER_BASE_PATH)) + .and(routes(&state).with(cors)) + } + + #[tokio::test] + async fn test_cardano_database_get_ok() { + let mut mock_http_message_service = MockMessageService::new(); + mock_http_message_service + .expect_get_cardano_database_list_message() + .return_once(|_| Ok(vec![CardanoDatabaseSnapshotListItemMessage::dummy()])) + .once(); + let mut dependency_manager = initialize_dependencies().await; + dependency_manager.message_service = Arc::new(mock_http_message_service); + + let method = Method::GET.as_str(); + let path = "/artifact/cardano-database"; + + let response = request() + .method(method) + .path(&format!("/{SERVER_BASE_PATH}{path}")) + .reply(&setup_router(RouterState::new_with_dummy_config(Arc::new( + dependency_manager, + )))) + .await; + + APISpec::verify_conformity( + APISpec::get_all_spec_files(), + method, + path, + "application/json", + &Null, + &response, + &StatusCode::OK, + ) + .unwrap(); + } + + #[tokio::test] + async fn test_cardano_database_get_ko() { + let mut mock_http_message_service = MockMessageService::new(); + mock_http_message_service + .expect_get_cardano_database_list_message() + .return_once(|_| Err(HydrationError::InvalidData("invalid data".to_string()).into())) + .once(); + let mut dependency_manager = initialize_dependencies().await; + dependency_manager.message_service = Arc::new(mock_http_message_service); + + let method = Method::GET.as_str(); + let path = "/artifact/cardano-database"; + + let response = request() + .method(method) + .path(&format!("/{SERVER_BASE_PATH}{path}")) + .reply(&setup_router(RouterState::new_with_dummy_config(Arc::new( + dependency_manager, + )))) + .await; + + APISpec::verify_conformity( + APISpec::get_all_spec_files(), + method, + path, + "application/json", + &Null, + &response, + &StatusCode::INTERNAL_SERVER_ERROR, + ) + .unwrap(); + } + + #[tokio::test] + async fn test_cardano_database_detail_increments_artifact_detail_total_served_since_startup_metric( + ) { + let method = Method::GET.as_str(); + let path = "/artifact/cardano-database/{merkle_root}"; + let dependency_manager = Arc::new(initialize_dependencies().await); + let initial_counter_value = dependency_manager + .metrics_service + .get_artifact_detail_cardano_db_total_served_since_startup() + .get(); + + request() + .method(method) + .path(&format!("/{SERVER_BASE_PATH}{path}")) + .reply(&setup_router(RouterState::new_with_dummy_config( + dependency_manager.clone(), + ))) + .await; + + assert_eq!( + initial_counter_value + 1, + dependency_manager + .metrics_service + .get_artifact_detail_cardano_db_total_served_since_startup() + .get() + ); + } + + #[tokio::test] + async fn test_cardano_database_detail_get_ok() { + let mut mock_http_message_service = MockMessageService::new(); + mock_http_message_service + .expect_get_cardano_database_message() + .return_once(|_| Ok(Some(CardanoDatabaseSnapshotMessage::dummy()))) + .once(); + let mut dependency_manager = initialize_dependencies().await; + dependency_manager.message_service = Arc::new(mock_http_message_service); + + let method = Method::GET.as_str(); + let path = "/artifact/cardano-database/{merkle_root}"; + + let response = request() + .method(method) + .path(&format!("/{SERVER_BASE_PATH}{path}")) + .reply(&setup_router(RouterState::new_with_dummy_config(Arc::new( + dependency_manager, + )))) + .await; + + APISpec::verify_conformity( + APISpec::get_all_spec_files(), + method, + path, + "application/json", + &Null, + &response, + &StatusCode::OK, + ) + .unwrap(); + } + + #[tokio::test] + async fn test_cardano_database_detail_returns_404_not_found_when_no_cardano_database_snapshot() + { + let mut mock_http_message_service = MockMessageService::new(); + mock_http_message_service + .expect_get_cardano_database_message() + .return_once(|_| Ok(None)) + .once(); + let mut dependency_manager = initialize_dependencies().await; + dependency_manager.message_service = Arc::new(mock_http_message_service); + + let method = Method::GET.as_str(); + let path = "/artifact/cardano-database/{merkle_root}"; + + let response = request() + .method(method) + .path(&format!("/{SERVER_BASE_PATH}{path}")) + .reply(&setup_router(RouterState::new_with_dummy_config(Arc::new( + dependency_manager, + )))) + .await; + + APISpec::verify_conformity( + APISpec::get_all_spec_files(), + method, + path, + "application/json", + &Null, + &response, + &StatusCode::NOT_FOUND, + ) + .unwrap(); + } + + #[tokio::test] + async fn test_cardano_database_digest_get_ko() { + let mut mock_http_message_service = MockMessageService::new(); + mock_http_message_service + .expect_get_cardano_database_message() + .return_once(|_| Err(HydrationError::InvalidData("invalid data".to_string()).into())) + .once(); + let mut dependency_manager = initialize_dependencies().await; + dependency_manager.message_service = Arc::new(mock_http_message_service); + + let method = Method::GET.as_str(); + let path = "/artifact/cardano-database/{merkle_root}"; + + let response = request() + .method(method) + .path(&format!("/{SERVER_BASE_PATH}{path}")) + .reply(&setup_router(RouterState::new_with_dummy_config(Arc::new( + dependency_manager, + )))) + .await; + + APISpec::verify_conformity( + APISpec::get_all_spec_files(), + method, + path, + "application/json", + &Null, + &response, + &StatusCode::INTERNAL_SERVER_ERROR, + ) + .unwrap(); + } +} diff --git a/mithril-aggregator/src/http_server/routes/artifact_routes/mod.rs b/mithril-aggregator/src/http_server/routes/artifact_routes/mod.rs index 2d6b7b8a3c..8a20cec327 100644 --- a/mithril-aggregator/src/http_server/routes/artifact_routes/mod.rs +++ b/mithril-aggregator/src/http_server/routes/artifact_routes/mod.rs @@ -1,3 +1,4 @@ +pub mod cardano_database; pub mod cardano_stake_distribution; pub mod cardano_transaction; pub mod mithril_stake_distribution; diff --git a/mithril-aggregator/src/http_server/routes/router.rs b/mithril-aggregator/src/http_server/routes/router.rs index 71e2803b9d..4f79c9dc39 100644 --- a/mithril-aggregator/src/http_server/routes/router.rs +++ b/mithril-aggregator/src/http_server/routes/router.rs @@ -103,6 +103,7 @@ pub fn routes( .and( certificate_routes::routes(&state) .or(artifact_routes::snapshot::routes(&state)) + .or(artifact_routes::cardano_database::routes(&state)) .or(artifact_routes::mithril_stake_distribution::routes(&state)) .or(artifact_routes::cardano_stake_distribution::routes(&state)) .or(artifact_routes::cardano_transaction::routes(&state)) From 99f7312145ee98acfbb389f2b8449facc6afaab7 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 18 Dec 2024 18:26:31 +0100 Subject: [PATCH 11/24] feat: check 'CardanoDatabse' artifacts production in the e2e test --- .../artifact_routes/cardano_database.rs | 9 ++- .../src/assertions/check.rs | 79 +++++++++++++++++++ .../mithril-end-to-end/src/end_to_end_spec.rs | 24 ++++++ 3 files changed, 109 insertions(+), 3 deletions(-) diff --git a/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs b/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs index fe84f6cb71..c13ab01864 100644 --- a/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs +++ b/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs @@ -41,7 +41,10 @@ fn serve_cardano_database_dir( router_state.configuration.snapshot_directory.clone(), )) .and(middlewares::with_logger(router_state)) - .and_then(handlers::ensure_downloaded_file_is_a_cardano_database) + .and(middlewares::extract_config(router_state, |config| { + config.allow_http_serve_directory + })) + .and_then(handlers::ensure_downloaded_file_is_a_cardano_database_artifact) } mod handlers { @@ -99,8 +102,8 @@ mod handlers { } } - /// Download a file if it's a Cardano_database archive file - pub async fn ensure_downloaded_file_is_a_cardano_database( + /// Download a file if it's a Cardano_database artifact file + pub async fn ensure_downloaded_file_is_a_cardano_database_artifact( reply: warp::fs::File, logger: Logger, ) -> Result { diff --git a/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs b/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs index 168500d436..7d426faf3a 100644 --- a/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs +++ b/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs @@ -6,6 +6,7 @@ use anyhow::{anyhow, Context}; use mithril_common::{ entities::{Epoch, TransactionHash}, messages::{ + CardanoDatabaseSnapshotListMessage, CardanoDatabaseSnapshotMessage, CardanoStakeDistributionListMessage, CardanoStakeDistributionMessage, CardanoTransactionSnapshotListMessage, CardanoTransactionSnapshotMessage, CertificateMessage, MithrilStakeDistributionListMessage, MithrilStakeDistributionMessage, @@ -161,6 +162,84 @@ pub async fn assert_signer_is_signing_snapshot( } } +pub async fn assert_node_producing_cardano_database_snapshot( + aggregator_endpoint: &str, +) -> StdResult { + let url = format!("{aggregator_endpoint}/artifact/cardano-database"); + info!("Waiting for the aggregator to produce a Cardano database snapshot"); + + // todo: reduce the number of attempts if we can reduce the delay between two immutables + match attempt!(45, Duration::from_millis(2000), { + match reqwest::get(url.clone()).await { + Ok(response) => match response.status() { + StatusCode::OK => match response + .json::() + .await + .as_deref() + { + Ok([cardano_database_snapshot, ..]) => { + Ok(Some(cardano_database_snapshot.merkle_root.clone())) + } + Ok(&[]) => Ok(None), + Err(err) => Err(anyhow!("Invalid Cardano database snapshot body : {err}",)), + }, + s => Err(anyhow!("Unexpected status code from Aggregator: {s}")), + }, + Err(err) => Err(anyhow!(err).context(format!("Request to `{url}` failed"))), + } + }) { + AttemptResult::Ok(merkle_root) => { + info!("Aggregator produced a Cardano database snapshot"; "merkle_root" => &merkle_root); + Ok(merkle_root) + } + AttemptResult::Err(error) => Err(error), + AttemptResult::Timeout() => Err(anyhow!( + "Timeout exhausted assert_node_producing_snapshot, no response from `{url}`" + )), + } +} + +pub async fn assert_signer_is_signing_cardano_database_snapshot( + aggregator_endpoint: &str, + merkle_root: &str, + expected_epoch_min: Epoch, +) -> StdResult { + let url = format!("{aggregator_endpoint}/artifact/cardano-database/{merkle_root}"); + info!( + "Asserting the aggregator is signing the Cardano database snapshot message `{}` with an expected min epoch of `{}`", + merkle_root, + expected_epoch_min + ); + + match attempt!(10, Duration::from_millis(1000), { + match reqwest::get(url.clone()).await { + Ok(response) => match response.status() { + StatusCode::OK => match response.json::().await { + Ok(cardano_database_snapshot) => match cardano_database_snapshot.beacon.epoch { + epoch if epoch >= expected_epoch_min => Ok(Some(cardano_database_snapshot)), + epoch => Err(anyhow!( + "Minimum expected Cardano database snapshot epoch not reached : {epoch} < {expected_epoch_min}" + )), + }, + Err(err) => Err(anyhow!(err).context("Invalid Cardano database snapshot body")), + }, + StatusCode::NOT_FOUND => Ok(None), + s => Err(anyhow!("Unexpected status code from Aggregator: {s}")), + }, + Err(err) => Err(anyhow!(err).context(format!("Request to `{url}` failed"))), + } + }) { + AttemptResult::Ok(snapshot) => { + info!("Signer signed a snapshot"; "certificate_hash" => &snapshot.certificate_hash); + Ok(snapshot.certificate_hash) + } + AttemptResult::Err(error) => Err(error), + AttemptResult::Timeout() => Err(anyhow!( + "Timeout exhausted assert_signer_is_signing_snapshot, no response from `{url}`" + )), + } +} + pub async fn assert_node_producing_cardano_transactions( aggregator_endpoint: &str, ) -> StdResult { diff --git a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs index b2f524a3f7..27f872750b 100644 --- a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs @@ -180,6 +180,30 @@ impl<'a> Spec<'a> { assertions::assert_client_can_verify_snapshot(&mut client, &digest).await?; } + // Verify that Cardano database snapshot artifacts are produced and signed correctly + { + let merkle_root = + assertions::assert_node_producing_cardano_database_snapshot(&aggregator_endpoint) + .await?; + let certificate_hash = assertions::assert_signer_is_signing_cardano_database_snapshot( + &aggregator_endpoint, + &merkle_root, + expected_epoch_min, + ) + .await?; + + assertions::assert_is_creating_certificate_with_enough_signers( + &aggregator_endpoint, + &certificate_hash, + self.infrastructure.signers().len(), + ) + .await?; + + // TODO: uncomment when the client can verify Cardano database snapshots + //let mut client = self.infrastructure.build_client()?; + //assertions::assert_client_can_verify_cardano_database_snapshot(&mut client, &digest).await?; + } + // Verify that Cardano transactions artifacts are produced and signed correctly if self.is_signing_cardano_transactions { let hash = assertions::assert_node_producing_cardano_transactions(&aggregator_endpoint) From 06cf6bfacf9ec7d5493fa90616c9fe45f49ee1fe Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 19 Dec 2024 11:26:58 +0100 Subject: [PATCH 12/24] chore: remove unused 'URL_SNAPSHOT_MANIFEST' aggregator configuration option --- mithril-test-lab/mithril-devnet/mkfiles/mkfiles-docker.sh | 2 -- mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs | 1 - 2 files changed, 3 deletions(-) diff --git a/mithril-test-lab/mithril-devnet/mkfiles/mkfiles-docker.sh b/mithril-test-lab/mithril-devnet/mkfiles/mkfiles-docker.sh index 615f448863..2bec6725e4 100644 --- a/mithril-test-lab/mithril-devnet/mkfiles/mkfiles-docker.sh +++ b/mithril-test-lab/mithril-devnet/mkfiles/mkfiles-docker.sh @@ -125,7 +125,6 @@ cat >> docker-compose.yaml <> docker-compose.yaml < Date: Thu, 19 Dec 2024 13:09:21 +0100 Subject: [PATCH 13/24] feat: add an option to allow http directory serving in aggregator --- mithril-aggregator/src/configuration.rs | 9 +++++++++ mithril-aggregator/src/dependency_injection/builder.rs | 1 + .../routes/artifact_routes/cardano_database.rs | 6 ++++++ .../src/http_server/routes/artifact_routes/snapshot.rs | 9 +++++++++ mithril-aggregator/src/http_server/routes/router.rs | 2 ++ .../mithril-end-to-end/src/mithril/aggregator.rs | 1 + 6 files changed, 28 insertions(+) diff --git a/mithril-aggregator/src/configuration.rs b/mithril-aggregator/src/configuration.rs index 94837c4117..653d808322 100644 --- a/mithril-aggregator/src/configuration.rs +++ b/mithril-aggregator/src/configuration.rs @@ -186,6 +186,9 @@ pub struct Configuration { /// Time interval at which usage metrics are persisted in event database (in seconds). pub persist_usage_report_interval_in_seconds: u64, + + /// If set to true, the HTTP server can serve static directories. + pub allow_http_serve_directory: bool, } /// Uploader needed to copy the snapshot once computed. @@ -270,6 +273,7 @@ impl Configuration { metrics_server_ip: "0.0.0.0".to_string(), metrics_server_port: 9090, persist_usage_report_interval_in_seconds: 10, + allow_http_serve_directory: false, } } @@ -411,6 +415,9 @@ pub struct DefaultConfiguration { /// Time interval at which metrics are persisted in event database (in seconds). pub persist_usage_report_interval_in_seconds: u64, + + /// If set to true, the HTTP server can serve static directories. + pub allow_http_serve_directory: bool, } impl Default for DefaultConfiguration { @@ -443,6 +450,7 @@ impl Default for DefaultConfiguration { metrics_server_ip: "0.0.0.0".to_string(), metrics_server_port: 9090, persist_usage_report_interval_in_seconds: 10, + allow_http_serve_directory: false, } } } @@ -530,6 +538,7 @@ impl Source for DefaultConfiguration { ), ])), ); + insert_default_configuration!(result, myself.allow_http_serve_directory); Ok(result) } } diff --git a/mithril-aggregator/src/dependency_injection/builder.rs b/mithril-aggregator/src/dependency_injection/builder.rs index bc55d2a199..2243416a8c 100644 --- a/mithril-aggregator/src/dependency_injection/builder.rs +++ b/mithril-aggregator/src/dependency_injection/builder.rs @@ -1556,6 +1556,7 @@ impl DependenciesBuilder { .clone(), snapshot_directory: self.configuration.get_snapshot_dir()?, cardano_node_version: self.configuration.cardano_node_version.clone(), + allow_http_serve_directory: self.configuration.allow_http_serve_directory, }, ); diff --git a/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs b/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs index c13ab01864..0bf737c660 100644 --- a/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs +++ b/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs @@ -106,6 +106,7 @@ mod handlers { pub async fn ensure_downloaded_file_is_a_cardano_database_artifact( reply: warp::fs::File, logger: Logger, + allow_http_serve_directory: bool, ) -> Result { let filepath = reply.path().to_path_buf(); debug!( @@ -114,6 +115,11 @@ mod handlers { filepath.display() ); + if !allow_http_serve_directory { + warn!(logger, "ensure_downloaded_file_is_a_cardano_database::error"; "error" => "http serve directory is disabled"); + return Ok(reply::empty(StatusCode::FORBIDDEN)); + } + // TODO: enhance this check with a regular expression once the file naming convention is defined let file_is_a_cardano_database_archive = filepath.to_string_lossy().contains("ancillary") || filepath.to_string_lossy().contains("immutable"); diff --git a/mithril-aggregator/src/http_server/routes/artifact_routes/snapshot.rs b/mithril-aggregator/src/http_server/routes/artifact_routes/snapshot.rs index 7e0962338a..1305df6d6e 100644 --- a/mithril-aggregator/src/http_server/routes/artifact_routes/snapshot.rs +++ b/mithril-aggregator/src/http_server/routes/artifact_routes/snapshot.rs @@ -61,6 +61,9 @@ fn serve_snapshots_dir( )) .and(middlewares::with_logger(router_state)) .and(middlewares::with_signed_entity_service(router_state)) + .and(middlewares::extract_config(router_state, |config| { + config.allow_http_serve_directory + })) .and_then(handlers::ensure_downloaded_file_is_a_snapshot) } @@ -155,6 +158,7 @@ mod handlers { reply: warp::fs::File, logger: Logger, signed_entity_service: Arc, + allow_http_serve_directory: bool, ) -> Result { let filepath = reply.path().to_path_buf(); debug!( @@ -163,6 +167,11 @@ mod handlers { filepath.display() ); + if !allow_http_serve_directory { + warn!(logger, "ensure_downloaded_file_is_a_cardano_database::error"; "error" => "http serve directory is disabled"); + return Ok(reply::empty(StatusCode::FORBIDDEN)); + } + match crate::tools::extract_digest_from_path(&filepath) { Ok(digest) => match signed_entity_service .get_signed_snapshot_by_id(&digest) diff --git a/mithril-aggregator/src/http_server/routes/router.rs b/mithril-aggregator/src/http_server/routes/router.rs index 4f79c9dc39..7bcc362da1 100644 --- a/mithril-aggregator/src/http_server/routes/router.rs +++ b/mithril-aggregator/src/http_server/routes/router.rs @@ -39,6 +39,7 @@ pub struct RouterConfig { pub cardano_transactions_signing_config: CardanoTransactionsSigningConfig, pub snapshot_directory: PathBuf, pub cardano_node_version: String, + pub allow_http_serve_directory: bool, } #[cfg(test)] @@ -55,6 +56,7 @@ impl RouterConfig { cardano_transactions_signing_config: CardanoTransactionsSigningConfig::dummy(), snapshot_directory: PathBuf::from("/dummy/snapshot/directory"), cardano_node_version: "1.2.3".to_string(), + allow_http_serve_directory: false, } } } diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs index 989da98c31..ab90912680 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs @@ -99,6 +99,7 @@ impl Aggregator { ), ("CARDANO_TRANSACTIONS_SIGNING_CONFIG__STEP", "15"), ("PERSIST_USAGE_REPORT_INTERVAL_IN_SECONDS", "3"), + ("ALLOW_HTTP_SERVE_DIRECTORY", "true"), ]); let args = vec![ "--db-directory", From 0fbd0e63d347eb4d018c79e005c1658a3d860545 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 20 Dec 2024 11:37:14 +0100 Subject: [PATCH 14/24] chore: apply style review comments --- .../http_server/routes/artifact_routes/cardano_database.rs | 4 ++-- .../src/http_server/routes/artifact_routes/snapshot.rs | 2 +- mithril-aggregator/src/services/message.rs | 7 +++++-- mithril-common/src/messages/cardano_database_list.rs | 4 ++-- openapi.yaml | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs b/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs index 0bf737c660..690dbb3f2f 100644 --- a/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs +++ b/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs @@ -39,7 +39,7 @@ fn serve_cardano_database_dir( warp::path("cardano_database_download") .and(warp::fs::dir( router_state.configuration.snapshot_directory.clone(), - )) + )) //TODO: the directory opened here should be narrowed to the final directory where the artifacts are stored .and(middlewares::with_logger(router_state)) .and(middlewares::extract_config(router_state, |config| { config.allow_http_serve_directory @@ -333,7 +333,7 @@ mod tests { } #[tokio::test] - async fn test_cardano_database_digest_get_ko() { + async fn test_cardano_database_detail_get_ko() { let mut mock_http_message_service = MockMessageService::new(); mock_http_message_service .expect_get_cardano_database_message() diff --git a/mithril-aggregator/src/http_server/routes/artifact_routes/snapshot.rs b/mithril-aggregator/src/http_server/routes/artifact_routes/snapshot.rs index 1305df6d6e..4152171c65 100644 --- a/mithril-aggregator/src/http_server/routes/artifact_routes/snapshot.rs +++ b/mithril-aggregator/src/http_server/routes/artifact_routes/snapshot.rs @@ -168,7 +168,7 @@ mod handlers { ); if !allow_http_serve_directory { - warn!(logger, "ensure_downloaded_file_is_a_cardano_database::error"; "error" => "http serve directory is disabled"); + warn!(logger, "ensure_downloaded_file_is_a_snapshot::error"; "error" => "http serve directory is disabled"); return Ok(reply::empty(StatusCode::FORBIDDEN)); } diff --git a/mithril-aggregator/src/services/message.rs b/mithril-aggregator/src/services/message.rs index d6c610d436..ad5116ae5a 100644 --- a/mithril-aggregator/src/services/message.rs +++ b/mithril-aggregator/src/services/message.rs @@ -452,9 +452,12 @@ mod tests { use super::*; #[tokio::test] - async fn get_cardano_database_not_exist() { + async fn get_cardano_database_when_record_does_not_exist() { let service = MessageServiceBuilder::new().build().await; - let snapshot = service.get_snapshot_message("whatever").await.unwrap(); + let snapshot = service + .get_cardano_database_message("whatever") + .await + .unwrap(); assert!(snapshot.is_none()); } diff --git a/mithril-common/src/messages/cardano_database_list.rs b/mithril-common/src/messages/cardano_database_list.rs index e35ba6f1b1..2a79527dfb 100644 --- a/mithril-common/src/messages/cardano_database_list.rs +++ b/mithril-common/src/messages/cardano_database_list.rs @@ -4,10 +4,10 @@ use serde::{Deserialize, Serialize}; use crate::entities::{CompressionAlgorithm, Epoch}; use crate::messages::CardanoDbBeaconMessagePart; -/// Message structure of a snapshot list +/// Message structure of a Cardano database snapshot list pub type CardanoDatabaseSnapshotListMessage = Vec; -/// Message structure of a snapshot list item +/// Message structure of a Cardano database snapshot list item #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct CardanoDatabaseSnapshotListItemMessage { /// Merkle root of the Cardano database snapshot diff --git a/openapi.yaml b/openapi.yaml index 527a913661..52aafb59ca 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -2028,7 +2028,7 @@ components: [ { "type": "cloud_storage", - "uri": "https://mithril-cdn-us.iohk.io/snapshot/immutables-6367ee65d0d1272e6e70736a1ea2cae34015874517f6328364f6b73930966732-{immutable-file_number}.tar.zst" + "uri": "https://mithril-cdn-us.iohk.io/snapshot/immutables-{immutable-file_number}.tar.zst" }, { "type": "cloud_storage", From 7b182e9f035f52d39d1edc307850a622c843b6b7 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 20 Dec 2024 11:50:56 +0100 Subject: [PATCH 15/24] refactor: better implementation of signed entities retrieval in signed entity service --- .../src/services/signed_entity.rs | 48 ++++++++----------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/mithril-aggregator/src/services/signed_entity.rs b/mithril-aggregator/src/services/signed_entity.rs index f447da8e4e..ccf23bfbd6 100644 --- a/mithril-aggregator/src/services/signed_entity.rs +++ b/mithril-aggregator/src/services/signed_entity.rs @@ -373,17 +373,15 @@ impl SignedEntityService for MithrilSignedEntityService { &self, total: usize, ) -> StdResult>> { - let signed_entities_records = self + let signed_entities = self .get_last_signed_entities( total, &SignedEntityTypeDiscriminants::CardanoImmutableFilesFull, ) - .await?; - let mut signed_entities: Vec> = Vec::new(); - - for record in signed_entities_records { - signed_entities.push(record.try_into()?); - } + .await? + .into_iter() + .map(|record| record.try_into()) + .collect::, _>>()?; Ok(signed_entities) } @@ -412,14 +410,12 @@ impl SignedEntityService for MithrilSignedEntityService { &self, total: usize, ) -> StdResult>> { - let signed_entities_records = self + let signed_entities = self .get_last_signed_entities(total, &SignedEntityTypeDiscriminants::CardanoDatabase) - .await?; - let mut signed_entities: Vec> = Vec::new(); - - for record in signed_entities_records { - signed_entities.push(record.try_into()?); - } + .await? + .into_iter() + .map(|record| record.try_into()) + .collect::, _>>()?; Ok(signed_entities) } @@ -448,17 +444,15 @@ impl SignedEntityService for MithrilSignedEntityService { &self, total: usize, ) -> StdResult>> { - let signed_entities_records = self + let signed_entities = self .get_last_signed_entities( total, &SignedEntityTypeDiscriminants::MithrilStakeDistribution, ) - .await?; - let mut signed_entities: Vec> = Vec::new(); - - for record in signed_entities_records { - signed_entities.push(record.try_into()?); - } + .await? + .into_iter() + .map(|record| record.try_into()) + .collect::, _>>()?; Ok(signed_entities) } @@ -500,17 +494,15 @@ impl SignedEntityService for MithrilSignedEntityService { &self, total: usize, ) -> StdResult>> { - let signed_entities_records = self + let signed_entities = self .get_last_signed_entities( total, &SignedEntityTypeDiscriminants::CardanoStakeDistribution, ) - .await?; - let mut signed_entities: Vec> = Vec::new(); - - for record in signed_entities_records { - signed_entities.push(record.try_into()?); - } + .await? + .into_iter() + .map(|record| record.try_into()) + .collect::, _>>()?; Ok(signed_entities) } From 3570580d1f21a2e04a864f6fb88dae362c03c7fb Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 20 Dec 2024 12:02:34 +0100 Subject: [PATCH 16/24] refactor: create 'add_content_disposition_header' function in 'reply' module of HTTP server --- .../routes/artifact_routes/cardano_database.rs | 9 +-------- .../routes/artifact_routes/snapshot.rs | 9 +-------- .../src/http_server/routes/reply.rs | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs b/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs index 690dbb3f2f..1cc9f95b64 100644 --- a/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs +++ b/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs @@ -124,14 +124,7 @@ mod handlers { let file_is_a_cardano_database_archive = filepath.to_string_lossy().contains("ancillary") || filepath.to_string_lossy().contains("immutable"); match file_is_a_cardano_database_archive { - true => Ok(Box::new(warp::reply::with_header( - reply, - "Content-Disposition", - format!( - "attachment; filename=\"{}\"", - filepath.file_name().unwrap().to_str().unwrap() - ), - )) as Box), + true => Ok(reply::add_content_disposition_header(reply, &filepath)), false => { warn!(logger,"ensure_downloaded_file_is_a_cardano_database::error"; "error" => "file is not a Cardano database archive"); Ok(reply::empty(StatusCode::NOT_FOUND)) diff --git a/mithril-aggregator/src/http_server/routes/artifact_routes/snapshot.rs b/mithril-aggregator/src/http_server/routes/artifact_routes/snapshot.rs index 4152171c65..55b7538bfd 100644 --- a/mithril-aggregator/src/http_server/routes/artifact_routes/snapshot.rs +++ b/mithril-aggregator/src/http_server/routes/artifact_routes/snapshot.rs @@ -177,14 +177,7 @@ mod handlers { .get_signed_snapshot_by_id(&digest) .await { - Ok(Some(_)) => Ok(Box::new(warp::reply::with_header( - reply, - "Content-Disposition", - format!( - "attachment; filename=\"{}\"", - filepath.file_name().unwrap().to_str().unwrap() - ), - )) as Box), + Ok(Some(_)) => Ok(reply::add_content_disposition_header(reply, &filepath)), _ => Ok(reply::empty(StatusCode::NOT_FOUND)), }, Err(err) => { diff --git a/mithril-aggregator/src/http_server/routes/reply.rs b/mithril-aggregator/src/http_server/routes/reply.rs index 7cc91d8855..ed40ce7363 100644 --- a/mithril-aggregator/src/http_server/routes/reply.rs +++ b/mithril-aggregator/src/http_server/routes/reply.rs @@ -1,3 +1,5 @@ +use std::path::Path; + use serde::Serialize; use warp::http::StatusCode; @@ -57,6 +59,20 @@ pub fn service_unavailable>(message: T) -> Box Box { + Box::new(warp::reply::with_header( + reply, + "Content-Disposition", + format!( + "attachment; filename=\"{}\"", + filepath.file_name().unwrap().to_str().unwrap() + ), + )) +} + #[cfg(test)] mod tests { use anyhow::anyhow; From c90c39c306fb17009c28f816b9a3fcd4c71271cc Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 20 Dec 2024 12:10:27 +0100 Subject: [PATCH 17/24] refactor: remove usage of 'CardanoDbBeaconMessagePart' in the Cardano database messages --- .../src/database/record/signed_entity.rs | 4 ++-- .../src/messages/cardano_database.rs | 18 ++++++------------ .../src/messages/cardano_database_list.rs | 16 ++++++---------- .../signed_entity_type_message.rs | 6 ------ 4 files changed, 14 insertions(+), 30 deletions(-) diff --git a/mithril-aggregator/src/database/record/signed_entity.rs b/mithril-aggregator/src/database/record/signed_entity.rs index 04c2f680b4..66441c0663 100644 --- a/mithril-aggregator/src/database/record/signed_entity.rs +++ b/mithril-aggregator/src/database/record/signed_entity.rs @@ -178,7 +178,7 @@ impl TryFrom for CardanoDatabaseSnapshotMessage { let artifact = serde_json::from_str::(&value.artifact)?; let cardano_database_snapshot_message = CardanoDatabaseSnapshotMessage { merkle_root: artifact.merkle_root, - beacon: artifact.beacon.into(), + beacon: artifact.beacon, certificate_hash: value.certificate_id, total_db_size_uncompressed: artifact.total_db_size_uncompressed, created_at: value.created_at, @@ -198,7 +198,7 @@ impl TryFrom for CardanoDatabaseSnapshotListItemMessage { let artifact = serde_json::from_str::(&value.artifact)?; let cardano_database_snapshot_list_item_message = CardanoDatabaseSnapshotListItemMessage { merkle_root: artifact.merkle_root, - beacon: artifact.beacon.into(), + beacon: artifact.beacon, certificate_hash: value.certificate_id, total_db_size_uncompressed: artifact.total_db_size_uncompressed, created_at: value.created_at, diff --git a/mithril-common/src/messages/cardano_database.rs b/mithril-common/src/messages/cardano_database.rs index 3fc56770e5..d1980d90be 100644 --- a/mithril-common/src/messages/cardano_database.rs +++ b/mithril-common/src/messages/cardano_database.rs @@ -1,12 +1,9 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use crate::{ - entities::{ - AncillaryLocation, ArtifactsLocations, CompressionAlgorithm, DigestLocation, Epoch, - ImmutablesLocation, - }, - messages::CardanoDbBeaconMessagePart, +use crate::entities::{ + AncillaryLocation, ArtifactsLocations, CardanoDbBeacon, CompressionAlgorithm, DigestLocation, + Epoch, ImmutablesLocation, }; /// Locations of the Cardano database related files. @@ -37,7 +34,7 @@ pub struct CardanoDatabaseSnapshotMessage { pub merkle_root: String, /// Mithril beacon on the Cardano chain. - pub beacon: CardanoDbBeaconMessagePart, + pub beacon: CardanoDbBeacon, /// Hash of the associated certificate pub certificate_hash: String, @@ -64,8 +61,7 @@ impl CardanoDatabaseSnapshotMessage { Self { merkle_root: "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6" .to_string(), - beacon: CardanoDbBeaconMessagePart { - network: Some("preview".to_string()), + beacon: CardanoDbBeacon { epoch: Epoch(123), immutable_file_number: 2345, }, @@ -105,8 +101,7 @@ mod tests { CardanoDatabaseSnapshotMessage { merkle_root: "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6" .to_string(), - beacon: CardanoDbBeaconMessagePart { - network: Some("preview".to_string()), + beacon: CardanoDbBeacon { epoch: Epoch(123), immutable_file_number: 2345, }, @@ -143,7 +138,6 @@ mod tests { let json = r#"{ "merkle_root": "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6", "beacon": { - "network": "preview", "epoch": 123, "immutable_file_number": 2345 }, diff --git a/mithril-common/src/messages/cardano_database_list.rs b/mithril-common/src/messages/cardano_database_list.rs index 2a79527dfb..0d1c8cbf27 100644 --- a/mithril-common/src/messages/cardano_database_list.rs +++ b/mithril-common/src/messages/cardano_database_list.rs @@ -1,8 +1,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use crate::entities::{CompressionAlgorithm, Epoch}; -use crate::messages::CardanoDbBeaconMessagePart; +use crate::entities::{CardanoDbBeacon, CompressionAlgorithm, Epoch}; /// Message structure of a Cardano database snapshot list pub type CardanoDatabaseSnapshotListMessage = Vec; @@ -14,7 +13,7 @@ pub struct CardanoDatabaseSnapshotListItemMessage { pub merkle_root: String, /// Mithril beacon on the Cardano chain - pub beacon: CardanoDbBeaconMessagePart, + pub beacon: CardanoDbBeacon, /// Hash of the associated certificate pub certificate_hash: String, @@ -38,8 +37,7 @@ impl CardanoDatabaseSnapshotListItemMessage { Self { merkle_root: "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6" .to_string(), - beacon: CardanoDbBeaconMessagePart { - network: Some("preview".to_string()), + beacon: CardanoDbBeacon { epoch: Epoch(123), immutable_file_number: 2345, }, @@ -63,8 +61,7 @@ mod tests { vec![CardanoDatabaseSnapshotListItemMessage { merkle_root: "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6" .to_string(), - beacon: CardanoDbBeaconMessagePart { - network: Some("preview".to_string()), + beacon: CardanoDbBeacon { epoch: Epoch(123), immutable_file_number: 2345, }, @@ -86,9 +83,8 @@ mod tests { { "merkle_root": "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6", "beacon": { - "network": "preview", - "epoch": 123, - "immutable_file_number": 2345 + "epoch": 123, + "immutable_file_number": 2345 }, "certificate_hash": "f6c01b373bafc4e039844071d5da3ace4a9c0745b9e9560e3e2af01823e9abfb", "total_db_size_uncompressed": 800796318, diff --git a/mithril-common/src/messages/message_parts/signed_entity_type_message.rs b/mithril-common/src/messages/message_parts/signed_entity_type_message.rs index e0b051eaf4..070491c568 100644 --- a/mithril-common/src/messages/message_parts/signed_entity_type_message.rs +++ b/mithril-common/src/messages/message_parts/signed_entity_type_message.rs @@ -80,12 +80,6 @@ impl From for CardanoDbBeacon { } } -impl From for CardanoDbBeaconMessagePart { - fn from(beacon: CardanoDbBeacon) -> Self { - CardanoDbBeaconMessagePart::new_without_network(beacon.epoch, beacon.immutable_file_number) - } -} - impl> From<(CardanoDbBeacon, N)> for CardanoDbBeaconMessagePart { fn from((beacon, network): (CardanoDbBeacon, N)) -> Self { CardanoDbBeaconMessagePart::new(network.into(), beacon.epoch, beacon.immutable_file_number) From c8e193845f3929d0222e6d0c13225846b9148708 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 20 Dec 2024 12:15:59 +0100 Subject: [PATCH 18/24] refactor: use latest golden tests structure for Cardano database messages --- .../src/messages/cardano_database.rs | 81 ++++++++++--------- .../src/messages/cardano_database_list.rs | 37 +++++---- 2 files changed, 62 insertions(+), 56 deletions(-) diff --git a/mithril-common/src/messages/cardano_database.rs b/mithril-common/src/messages/cardano_database.rs index d1980d90be..0882478ed9 100644 --- a/mithril-common/src/messages/cardano_database.rs +++ b/mithril-common/src/messages/cardano_database.rs @@ -97,7 +97,45 @@ impl CardanoDatabaseSnapshotMessage { mod tests { use super::*; - fn golden_message() -> CardanoDatabaseSnapshotMessage { + const ACTUAL_JSON: &str = r#" + { + "merkle_root": "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6", + "beacon": { + "epoch": 123, + "immutable_file_number": 2345 + }, + "certificate_hash": "f6c01b373bafc4e039844071d5da3ace4a9c0745b9e9560e3e2af01823e9abfb", + "total_db_size_uncompressed": 800796318, + "locations": { + "digests": [ + { + "type": "aggregator", + "uri": "https://host-1/digest-1" + } + ], + "immutables": [ + { + "type": "cloud_storage", + "uri": "https://host-1/immutables-2" + }, + { + "type": "cloud_storage", + "uri": "https://host-2/immutables-2" + } + ], + "ancillary": [ + { + "type": "cloud_storage", + "uri": "https://host-1/ancillary-3" + } + ] + }, + "compression_algorithm": "gzip", + "cardano_node_version": "0.0.1", + "created_at": "2023-01-19T13:43:05.618857482Z" + }"#; + + fn golden_actual_message() -> CardanoDatabaseSnapshotMessage { CardanoDatabaseSnapshotMessage { merkle_root: "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6" .to_string(), @@ -134,47 +172,12 @@ mod tests { // Test the backward compatibility with possible future upgrades. #[test] - fn test_v1() { - let json = r#"{ - "merkle_root": "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6", - "beacon": { - "epoch": 123, - "immutable_file_number": 2345 - }, - "certificate_hash": "f6c01b373bafc4e039844071d5da3ace4a9c0745b9e9560e3e2af01823e9abfb", - "total_db_size_uncompressed": 800796318, - "locations": { - "digests": [ - { - "type": "aggregator", - "uri": "https://host-1/digest-1" - } - ], - "immutables": [ - { - "type": "cloud_storage", - "uri": "https://host-1/immutables-2" - }, - { - "type": "cloud_storage", - "uri": "https://host-2/immutables-2" - } - ], - "ancillary": [ - { - "type": "cloud_storage", - "uri": "https://host-1/ancillary-3" - } - ] - }, - "compression_algorithm": "gzip", - "cardano_node_version": "0.0.1", - "created_at": "2023-01-19T13:43:05.618857482Z" - }"#; + fn test_actual_json_deserialized_into_actual_message() { + let json = ACTUAL_JSON; let message: CardanoDatabaseSnapshotMessage = serde_json::from_str(json).expect( "This JSON is expected to be successfully parsed into a CardanoDatabaseSnapshotMessage instance.", ); - assert_eq!(golden_message(), message); + assert_eq!(golden_actual_message(), message); } } diff --git a/mithril-common/src/messages/cardano_database_list.rs b/mithril-common/src/messages/cardano_database_list.rs index 0d1c8cbf27..d9237295ae 100644 --- a/mithril-common/src/messages/cardano_database_list.rs +++ b/mithril-common/src/messages/cardano_database_list.rs @@ -57,7 +57,23 @@ impl CardanoDatabaseSnapshotListItemMessage { mod tests { use super::*; - fn golden_message() -> CardanoDatabaseSnapshotListMessage { + const ACTUAL_JSON: &str = r#" + [ + { + "merkle_root": "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6", + "beacon": { + "epoch": 123, + "immutable_file_number": 2345 + }, + "certificate_hash": "f6c01b373bafc4e039844071d5da3ace4a9c0745b9e9560e3e2af01823e9abfb", + "total_db_size_uncompressed": 800796318, + "compression_algorithm": "gzip", + "cardano_node_version": "0.0.1", + "created_at": "2023-01-19T13:43:05.618857482Z" + } + ]"#; + + fn golden_actual_message() -> CardanoDatabaseSnapshotListMessage { vec![CardanoDatabaseSnapshotListItemMessage { merkle_root: "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6" .to_string(), @@ -78,25 +94,12 @@ mod tests { // Test the backward compatibility with possible future upgrades. #[test] - fn test_v1() { - let json = r#"[ - { - "merkle_root": "c8224920b9f5ad7377594eb8a15f34f08eb3103cc5241d57cafc5638403ec7c6", - "beacon": { - "epoch": 123, - "immutable_file_number": 2345 - }, - "certificate_hash": "f6c01b373bafc4e039844071d5da3ace4a9c0745b9e9560e3e2af01823e9abfb", - "total_db_size_uncompressed": 800796318, - "compression_algorithm": "gzip", - "cardano_node_version": "0.0.1", - "created_at": "2023-01-19T13:43:05.618857482Z" - } - ]"#; + fn test_actual_json_deserialized_into_actual_message() { + let json = ACTUAL_JSON; let message: CardanoDatabaseSnapshotListMessage = serde_json::from_str(json).expect( "This JSON is expected to be successfully parsed into a CardanoDatabaseSnapshotListMessage instance.", ); - assert_eq!(golden_message(), message); + assert_eq!(golden_actual_message(), message); } } From 0e9a254fe49eb89577154134ca619997d33486af Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 20 Dec 2024 12:16:57 +0100 Subject: [PATCH 19/24] chore: fix typos in golden tests of messages --- mithril-common/src/messages/aggregator_features.rs | 2 +- mithril-common/src/messages/cardano_transaction_snapshot.rs | 2 +- .../src/messages/cardano_transaction_snapshot_list.rs | 2 +- mithril-common/src/messages/mithril_stake_distribution.rs | 2 +- mithril-common/src/messages/mithril_stake_distribution_list.rs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mithril-common/src/messages/aggregator_features.rs b/mithril-common/src/messages/aggregator_features.rs index 870044b3f7..55bdc695b6 100644 --- a/mithril-common/src/messages/aggregator_features.rs +++ b/mithril-common/src/messages/aggregator_features.rs @@ -125,7 +125,7 @@ mod tests { } }"#; - // Test the retro compatibility with possible future upgrades. + // Test the backward compatibility with possible future upgrades. #[test] fn test_actual_json_deserialized_into_previous_message() { let json = ACTUAL_JSON; diff --git a/mithril-common/src/messages/cardano_transaction_snapshot.rs b/mithril-common/src/messages/cardano_transaction_snapshot.rs index ee496f18f7..3a39a60dc7 100644 --- a/mithril-common/src/messages/cardano_transaction_snapshot.rs +++ b/mithril-common/src/messages/cardano_transaction_snapshot.rs @@ -61,7 +61,7 @@ mod tests { } } - // Test the retro compatibility with possible future upgrades. + // Test the backward compatibility with possible future upgrades. #[test] fn test_v1() { let json = r#"{ diff --git a/mithril-common/src/messages/cardano_transaction_snapshot_list.rs b/mithril-common/src/messages/cardano_transaction_snapshot_list.rs index 827779f968..3911b761d5 100644 --- a/mithril-common/src/messages/cardano_transaction_snapshot_list.rs +++ b/mithril-common/src/messages/cardano_transaction_snapshot_list.rs @@ -63,7 +63,7 @@ mod tests { }] } - // Test the retro compatibility with possible future upgrades. + // Test the backward compatibility with possible future upgrades. #[test] fn test_v1() { let json = r#"[{ diff --git a/mithril-common/src/messages/mithril_stake_distribution.rs b/mithril-common/src/messages/mithril_stake_distribution.rs index 02b7056d78..437ecc2d04 100644 --- a/mithril-common/src/messages/mithril_stake_distribution.rs +++ b/mithril-common/src/messages/mithril_stake_distribution.rs @@ -75,7 +75,7 @@ mod tests { } } - // Test the retro compatibility with possible future upgrades. + // Test the backward compatibility with possible future upgrades. #[test] fn test_v1() { let json = r#"{ diff --git a/mithril-common/src/messages/mithril_stake_distribution_list.rs b/mithril-common/src/messages/mithril_stake_distribution_list.rs index 66a68b8c6e..263c876d3a 100644 --- a/mithril-common/src/messages/mithril_stake_distribution_list.rs +++ b/mithril-common/src/messages/mithril_stake_distribution_list.rs @@ -51,7 +51,7 @@ mod tests { }] } - // Test the retro compatibility with possible future upgrades. + // Test the backward compatibility with possible future upgrades. #[test] fn test_v1() { let json = r#"[{ From 6971f6c4454869e6b198887f4ebce26d26d53ae4 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 20 Dec 2024 12:55:52 +0100 Subject: [PATCH 20/24] refactor: enhance message service tests --- mithril-aggregator/src/services/message.rs | 186 ++++++++++++++------- 1 file changed, 130 insertions(+), 56 deletions(-) diff --git a/mithril-aggregator/src/services/message.rs b/mithril-aggregator/src/services/message.rs index ad5116ae5a..6f97d85aaf 100644 --- a/mithril-aggregator/src/services/message.rs +++ b/mithril-aggregator/src/services/message.rs @@ -428,22 +428,36 @@ mod tests { #[tokio::test] async fn get_snapshot_list_message() { - let record = SignedEntityRecord { - signed_entity_id: "signed_entity_id".to_string(), - signed_entity_type: SignedEntityType::CardanoImmutableFilesFull(fake_data::beacon()), - certificate_id: "cert_id".to_string(), - artifact: serde_json::to_string(&fake_data::snapshots(1)[0]).unwrap(), - created_at: Default::default(), - }; - let message: SnapshotListMessage = vec![record.clone().try_into().unwrap()]; + let records = vec![ + SignedEntityRecord { + signed_entity_id: "signed_entity_id-1".to_string(), + signed_entity_type: SignedEntityType::CardanoImmutableFilesFull( + fake_data::beacon(), + ), + certificate_id: "cert_id-1".to_string(), + artifact: serde_json::to_string(&fake_data::snapshots(1)[0]).unwrap(), + created_at: Default::default(), + }, + SignedEntityRecord { + signed_entity_id: "signed_entity_id-2".to_string(), + signed_entity_type: SignedEntityType::CardanoDatabase(fake_data::beacon()), + certificate_id: "cert_id-2".to_string(), + artifact: serde_json::to_string(&fake_data::cardano_database_snapshots(1)[0]) + .unwrap(), + created_at: Default::default(), + }, + ]; + let message: SnapshotListMessage = vec![records[0].clone().try_into().unwrap()]; let service = MessageServiceBuilder::new() - .with_signed_entity_records(&[record]) + .with_signed_entity_records(&records) .build() .await; - let response = service.get_snapshot_list_message(3).await.unwrap(); + let response = service.get_snapshot_list_message(0).await.unwrap(); + assert!(response.is_empty()); + let response = service.get_snapshot_list_message(3).await.unwrap(); assert_eq!(message, response); } } @@ -490,24 +504,37 @@ mod tests { #[tokio::test] async fn get_cardano_database_list_message() { - let record = SignedEntityRecord { - signed_entity_id: "signed_entity_id".to_string(), - signed_entity_type: SignedEntityType::CardanoDatabase(fake_data::beacon()), - certificate_id: "cert_id".to_string(), - artifact: serde_json::to_string(&fake_data::cardano_database_snapshots(1)[0]) - .unwrap(), - created_at: Default::default(), - }; + let records = vec![ + SignedEntityRecord { + signed_entity_id: "signed_entity_id-1".to_string(), + signed_entity_type: SignedEntityType::CardanoDatabase(fake_data::beacon()), + certificate_id: "cert_id-1".to_string(), + artifact: serde_json::to_string(&fake_data::cardano_database_snapshots(1)[0]) + .unwrap(), + created_at: Default::default(), + }, + SignedEntityRecord { + signed_entity_id: "signed_entity_id-2".to_string(), + signed_entity_type: SignedEntityType::CardanoImmutableFilesFull( + fake_data::beacon(), + ), + certificate_id: "cert_id-2".to_string(), + artifact: serde_json::to_string(&fake_data::snapshots(1)[0]).unwrap(), + created_at: Default::default(), + }, + ]; let message: CardanoDatabaseSnapshotListMessage = - vec![record.clone().try_into().unwrap()]; + vec![records[0].clone().try_into().unwrap()]; let service = MessageServiceBuilder::new() - .with_signed_entity_records(&[record]) + .with_signed_entity_records(&records) .build() .await; - let response = service.get_cardano_database_list_message(3).await.unwrap(); + let response = service.get_cardano_database_list_message(0).await.unwrap(); + assert!(response.is_empty()); + let response = service.get_cardano_database_list_message(3).await.unwrap(); assert_eq!(message, response); } } @@ -555,27 +582,42 @@ mod tests { #[tokio::test] async fn get_mithril_stake_distribution_list_message() { - let record = SignedEntityRecord { - signed_entity_id: "signed_entity_id".to_string(), - signed_entity_type: SignedEntityType::MithrilStakeDistribution(Epoch(18)), - certificate_id: "cert_id".to_string(), - artifact: serde_json::to_string(&fake_data::mithril_stake_distributions(1)[0]) - .unwrap(), - created_at: Default::default(), - }; + let records = vec![ + SignedEntityRecord { + signed_entity_id: "signed_entity_id-1".to_string(), + signed_entity_type: SignedEntityType::MithrilStakeDistribution(Epoch(18)), + certificate_id: "cert_id-1".to_string(), + artifact: serde_json::to_string(&fake_data::mithril_stake_distributions(1)[0]) + .unwrap(), + created_at: Default::default(), + }, + SignedEntityRecord { + signed_entity_id: "signed_entity_id-2".to_string(), + signed_entity_type: SignedEntityType::CardanoDatabase(fake_data::beacon()), + certificate_id: "cert_id-2".to_string(), + artifact: serde_json::to_string(&fake_data::cardano_database_snapshots(1)[0]) + .unwrap(), + created_at: Default::default(), + }, + ]; let message: MithrilStakeDistributionListMessage = - vec![record.clone().try_into().unwrap()]; + vec![records[0].clone().try_into().unwrap()]; let service = MessageServiceBuilder::new() - .with_signed_entity_records(&[record]) + .with_signed_entity_records(&records) .build() .await; let response = service - .get_mithril_stake_distribution_list_message(10) + .get_mithril_stake_distribution_list_message(0) .await .unwrap(); + assert!(response.is_empty()); + let response = service + .get_mithril_stake_distribution_list_message(3) + .await + .unwrap(); assert_eq!(message, response); } } @@ -626,30 +668,47 @@ mod tests { #[tokio::test] async fn get_cardano_transaction_list_message() { - let record = SignedEntityRecord { - signed_entity_id: "signed_entity_id".to_string(), - signed_entity_type: SignedEntityType::CardanoTransactions( - Epoch(18), - BlockNumber(120), - ), - certificate_id: "cert_id".to_string(), - artifact: serde_json::to_string(&fake_data::cardano_transactions_snapshot(1)[0]) + let records = vec![ + SignedEntityRecord { + signed_entity_id: "signed_entity_id-1".to_string(), + signed_entity_type: SignedEntityType::CardanoTransactions( + Epoch(18), + BlockNumber(120), + ), + certificate_id: "cert_id-1".to_string(), + artifact: serde_json::to_string( + &fake_data::cardano_transactions_snapshot(1)[0], + ) .unwrap(), - created_at: Default::default(), - }; + created_at: Default::default(), + }, + SignedEntityRecord { + signed_entity_id: "signed_entity_id-2".to_string(), + signed_entity_type: SignedEntityType::CardanoDatabase(fake_data::beacon()), + certificate_id: "cert_id-2".to_string(), + artifact: serde_json::to_string(&fake_data::cardano_database_snapshots(1)[0]) + .unwrap(), + created_at: Default::default(), + }, + ]; let message: CardanoTransactionSnapshotListMessage = - vec![record.clone().try_into().unwrap()]; + vec![records[0].clone().try_into().unwrap()]; let service = MessageServiceBuilder::new() - .with_signed_entity_records(&[record]) + .with_signed_entity_records(&records) .build() .await; let response = service - .get_cardano_transaction_list_message(10) + .get_cardano_transaction_list_message(0) .await .unwrap(); + assert!(response.is_empty()); + let response = service + .get_cardano_transaction_list_message(3) + .await + .unwrap(); assert_eq!(message, response); } } @@ -737,27 +796,42 @@ mod tests { #[tokio::test] async fn get_cardano_stake_distribution_list_message() { - let record = SignedEntityRecord { - signed_entity_id: "signed_entity_id".to_string(), - signed_entity_type: SignedEntityType::CardanoStakeDistribution(Epoch(18)), - certificate_id: "cert_id".to_string(), - artifact: serde_json::to_string(&fake_data::cardano_stake_distributions(1)[0]) - .unwrap(), - created_at: Default::default(), - }; + let records = vec![ + SignedEntityRecord { + signed_entity_id: "signed_entity_id-1".to_string(), + signed_entity_type: SignedEntityType::CardanoStakeDistribution(Epoch(18)), + certificate_id: "cert_id-1".to_string(), + artifact: serde_json::to_string(&fake_data::cardano_stake_distributions(1)[0]) + .unwrap(), + created_at: Default::default(), + }, + SignedEntityRecord { + signed_entity_id: "signed_entity_id-2".to_string(), + signed_entity_type: SignedEntityType::CardanoDatabase(fake_data::beacon()), + certificate_id: "cert_id-2".to_string(), + artifact: serde_json::to_string(&fake_data::cardano_database_snapshots(1)[0]) + .unwrap(), + created_at: Default::default(), + }, + ]; let message: CardanoStakeDistributionListMessage = - vec![record.clone().try_into().unwrap()]; + vec![records[0].clone().try_into().unwrap()]; let service = MessageServiceBuilder::new() - .with_signed_entity_records(&[record]) + .with_signed_entity_records(&records) .build() .await; let response = service - .get_cardano_stake_distribution_list_message(10) + .get_cardano_stake_distribution_list_message(0) .await .unwrap(); + assert!(response.is_empty()); + let response = service + .get_cardano_stake_distribution_list_message(3) + .await + .unwrap(); assert_eq!(message, response); } } From 92470a76c158a43bdc5c54597c662f0cc271fd84 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 20 Dec 2024 13:12:21 +0100 Subject: [PATCH 21/24] refactor: make configuration automatically compute if directory serving is enabled in the HTTP server --- mithril-aggregator/src/configuration.rs | 35 ++++++++++++++----- .../src/dependency_injection/builder.rs | 2 +- .../artifact_routes/cardano_database.rs | 1 + .../src/mithril/aggregator.rs | 1 - 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/mithril-aggregator/src/configuration.rs b/mithril-aggregator/src/configuration.rs index 653d808322..26a5b969f5 100644 --- a/mithril-aggregator/src/configuration.rs +++ b/mithril-aggregator/src/configuration.rs @@ -186,9 +186,6 @@ pub struct Configuration { /// Time interval at which usage metrics are persisted in event database (in seconds). pub persist_usage_report_interval_in_seconds: u64, - - /// If set to true, the HTTP server can serve static directories. - pub allow_http_serve_directory: bool, } /// Uploader needed to copy the snapshot once computed. @@ -273,7 +270,6 @@ impl Configuration { metrics_server_ip: "0.0.0.0".to_string(), metrics_server_port: 9090, persist_usage_report_interval_in_seconds: 10, - allow_http_serve_directory: false, } } @@ -336,6 +332,15 @@ impl Configuration { Ok(allowed_discriminants) } + + /// Check if the HTTP server can serve static directories. + // TODO: This function should be completed when the configuration of the uploaders for the Cardano database is done. + pub fn allow_http_serve_directory(&self) -> bool { + match self.snapshot_uploader_type { + SnapshotUploaderType::Local => true, + SnapshotUploaderType::Gcp => false, + } + } } /// Default configuration with all the default values for configurations. @@ -415,9 +420,6 @@ pub struct DefaultConfiguration { /// Time interval at which metrics are persisted in event database (in seconds). pub persist_usage_report_interval_in_seconds: u64, - - /// If set to true, the HTTP server can serve static directories. - pub allow_http_serve_directory: bool, } impl Default for DefaultConfiguration { @@ -450,7 +452,6 @@ impl Default for DefaultConfiguration { metrics_server_ip: "0.0.0.0".to_string(), metrics_server_port: 9090, persist_usage_report_interval_in_seconds: 10, - allow_http_serve_directory: false, } } } @@ -538,7 +539,6 @@ impl Source for DefaultConfiguration { ), ])), ); - insert_default_configuration!(result, myself.allow_http_serve_directory); Ok(result) } } @@ -608,4 +608,21 @@ mod test { BTreeSet::from(SignedEntityConfig::DEFAULT_ALLOWED_DISCRIMINANTS) ); } + + #[test] + fn allow_http_serve_directory() { + let config = Configuration { + snapshot_uploader_type: SnapshotUploaderType::Local, + ..Configuration::new_sample() + }; + + assert!(config.allow_http_serve_directory()); + + let config = Configuration { + snapshot_uploader_type: SnapshotUploaderType::Gcp, + ..Configuration::new_sample() + }; + + assert!(!config.allow_http_serve_directory()); + } } diff --git a/mithril-aggregator/src/dependency_injection/builder.rs b/mithril-aggregator/src/dependency_injection/builder.rs index 2243416a8c..49e008cadc 100644 --- a/mithril-aggregator/src/dependency_injection/builder.rs +++ b/mithril-aggregator/src/dependency_injection/builder.rs @@ -1556,7 +1556,7 @@ impl DependenciesBuilder { .clone(), snapshot_directory: self.configuration.get_snapshot_dir()?, cardano_node_version: self.configuration.cardano_node_version.clone(), - allow_http_serve_directory: self.configuration.allow_http_serve_directory, + allow_http_serve_directory: self.configuration.allow_http_serve_directory(), }, ); diff --git a/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs b/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs index 1cc9f95b64..bbf2aaa1cf 100644 --- a/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs +++ b/mithril-aggregator/src/http_server/routes/artifact_routes/cardano_database.rs @@ -103,6 +103,7 @@ mod handlers { } /// Download a file if it's a Cardano_database artifact file + // TODO: this function should probable be unit tested once the file naming convention is defined pub async fn ensure_downloaded_file_is_a_cardano_database_artifact( reply: warp::fs::File, logger: Logger, diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs index ab90912680..989da98c31 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs @@ -99,7 +99,6 @@ impl Aggregator { ), ("CARDANO_TRANSACTIONS_SIGNING_CONFIG__STEP", "15"), ("PERSIST_USAGE_REPORT_INTERVAL_IN_SECONDS", "3"), - ("ALLOW_HTTP_SERVE_DIRECTORY", "true"), ]); let args = vec![ "--db-directory", From 613290391d10fdc5e674bd95d1027a2fdc722e6a Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 20 Dec 2024 15:06:54 +0100 Subject: [PATCH 22/24] refactor: make e2e tests assertions more readable --- .../src/assertions/check.rs | 110 +++++++++++++++--- 1 file changed, 93 insertions(+), 17 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs b/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs index 7d426faf3a..10c5ab0969 100644 --- a/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs +++ b/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs @@ -24,11 +24,14 @@ pub async fn assert_node_producing_mithril_stake_distribution( let url = format!("{aggregator_endpoint}/artifact/mithril-stake-distributions"); info!("Waiting for the aggregator to produce a mithril stake distribution"); - // todo: reduce the number of attempts if we can reduce the delay between two immutables - match attempt!(45, Duration::from_millis(2000), { + async fn fetch_last_mithril_stake_distribution_hash(url: String) -> StdResult> { match reqwest::get(url.clone()).await { Ok(response) => match response.status() { - StatusCode::OK => match response.json::().await.as_deref() { + StatusCode::OK => match response + .json::() + .await + .as_deref() + { Ok([stake_distribution, ..]) => Ok(Some(stake_distribution.hash.clone())), Ok(&[]) => Ok(None), Err(err) => Err(anyhow!("Invalid mithril stake distribution body : {err}",)), @@ -37,6 +40,11 @@ pub async fn assert_node_producing_mithril_stake_distribution( }, Err(err) => Err(anyhow!(err).context(format!("Request to `{url}` failed"))), } + } + + // todo: reduce the number of attempts if we can reduce the delay between two immutables + match attempt!(45, Duration::from_millis(2000), { + fetch_last_mithril_stake_distribution_hash(url.clone()).await }) { AttemptResult::Ok(hash) => { info!("Aggregator produced a mithril stake distribution"; "hash" => &hash); @@ -61,7 +69,10 @@ pub async fn assert_signer_is_signing_mithril_stake_distribution( expected_epoch_min ); - match attempt!(10, Duration::from_millis(1000), { + async fn fetch_mithril_stake_distribution_message( + url: String, + expected_epoch_min: Epoch, + ) -> StdResult> { match reqwest::get(url.clone()).await { Ok(response) => match response.status() { StatusCode::OK => match response.json::().await { @@ -78,6 +89,10 @@ pub async fn assert_signer_is_signing_mithril_stake_distribution( }, Err(err) => Err(anyhow!(err).context(format!("Request to `{url}` failed"))), } + } + + match attempt!(10, Duration::from_millis(1000), { + fetch_mithril_stake_distribution_message(url.clone(), expected_epoch_min).await }) { AttemptResult::Ok(stake_distribution) => { // todo: assert that the mithril stake distribution is really signed @@ -95,8 +110,7 @@ pub async fn assert_node_producing_snapshot(aggregator_endpoint: &str) -> StdRes let url = format!("{aggregator_endpoint}/artifact/snapshots"); info!("Waiting for the aggregator to produce a snapshot"); - // todo: reduce the number of attempts if we can reduce the delay between two immutables - match attempt!(45, Duration::from_millis(2000), { + async fn fetch_last_snapshot_digest(url: String) -> StdResult> { match reqwest::get(url.clone()).await { Ok(response) => match response.status() { StatusCode::OK => match response.json::>().await.as_deref() { @@ -108,6 +122,11 @@ pub async fn assert_node_producing_snapshot(aggregator_endpoint: &str) -> StdRes }, Err(err) => Err(anyhow!(err).context(format!("Request to `{url}` failed"))), } + } + + // todo: reduce the number of attempts if we can reduce the delay between two immutables + match attempt!(45, Duration::from_millis(2000), { + fetch_last_snapshot_digest(url.clone()).await }) { AttemptResult::Ok(digest) => { info!("Aggregator produced a snapshot"; "digest" => &digest); @@ -132,7 +151,10 @@ pub async fn assert_signer_is_signing_snapshot( expected_epoch_min ); - match attempt!(10, Duration::from_millis(1000), { + async fn fetch_snapshot_message( + url: String, + expected_epoch_min: Epoch, + ) -> StdResult> { match reqwest::get(url.clone()).await { Ok(response) => match response.status() { StatusCode::OK => match response.json::().await { @@ -149,6 +171,10 @@ pub async fn assert_signer_is_signing_snapshot( }, Err(err) => Err(anyhow!(err).context(format!("Request to `{url}` failed"))), } + } + + match attempt!(10, Duration::from_millis(1000), { + fetch_snapshot_message(url.clone(), expected_epoch_min).await }) { AttemptResult::Ok(snapshot) => { // todo: assert that the snapshot is really signed @@ -168,8 +194,9 @@ pub async fn assert_node_producing_cardano_database_snapshot( let url = format!("{aggregator_endpoint}/artifact/cardano-database"); info!("Waiting for the aggregator to produce a Cardano database snapshot"); - // todo: reduce the number of attempts if we can reduce the delay between two immutables - match attempt!(45, Duration::from_millis(2000), { + async fn fetch_last_cardano_database_snapshot_merkle_root( + url: String, + ) -> StdResult> { match reqwest::get(url.clone()).await { Ok(response) => match response.status() { StatusCode::OK => match response @@ -187,6 +214,11 @@ pub async fn assert_node_producing_cardano_database_snapshot( }, Err(err) => Err(anyhow!(err).context(format!("Request to `{url}` failed"))), } + } + + // todo: reduce the number of attempts if we can reduce the delay between two immutables + match attempt!(45, Duration::from_millis(2000), { + fetch_last_cardano_database_snapshot_merkle_root(url.clone()).await }) { AttemptResult::Ok(merkle_root) => { info!("Aggregator produced a Cardano database snapshot"; "merkle_root" => &merkle_root); @@ -211,7 +243,10 @@ pub async fn assert_signer_is_signing_cardano_database_snapshot( expected_epoch_min ); - match attempt!(10, Duration::from_millis(1000), { + async fn fetch_cardano_database_snapshot_message( + url: String, + expected_epoch_min: Epoch, + ) -> StdResult> { match reqwest::get(url.clone()).await { Ok(response) => match response.status() { StatusCode::OK => match response.json::().await { @@ -228,6 +263,10 @@ pub async fn assert_signer_is_signing_cardano_database_snapshot( }, Err(err) => Err(anyhow!(err).context(format!("Request to `{url}` failed"))), } + } + + match attempt!(10, Duration::from_millis(1000), { + fetch_cardano_database_snapshot_message(url.clone(), expected_epoch_min).await }) { AttemptResult::Ok(snapshot) => { info!("Signer signed a snapshot"; "certificate_hash" => &snapshot.certificate_hash); @@ -246,7 +285,9 @@ pub async fn assert_node_producing_cardano_transactions( let url = format!("{aggregator_endpoint}/artifact/cardano-transactions"); info!("Waiting for the aggregator to produce a Cardano transactions artifact"); - match attempt!(45, Duration::from_millis(2000), { + async fn fetch_last_cardano_transaction_snapshot_hash( + url: String, + ) -> StdResult> { match reqwest::get(url.clone()).await { Ok(response) => match response.status() { StatusCode::OK => match response @@ -264,6 +305,10 @@ pub async fn assert_node_producing_cardano_transactions( }, Err(err) => Err(anyhow!(err).context(format!("Request to `{url}` failed"))), } + } + + match attempt!(45, Duration::from_millis(2000), { + fetch_last_cardano_transaction_snapshot_hash(url.clone()).await }) { AttemptResult::Ok(hash) => { info!("Aggregator produced a Cardano transactions artifact"; "hash" => &hash); @@ -288,7 +333,10 @@ pub async fn assert_signer_is_signing_cardano_transactions( expected_epoch_min ); - match attempt!(10, Duration::from_millis(1000), { + async fn fetch_cardano_transaction_snapshot_message( + url: String, + expected_epoch_min: Epoch, + ) -> StdResult> { match reqwest::get(url.clone()).await { Ok(response) => match response.status() { StatusCode::OK => match response.json::().await { @@ -305,6 +353,10 @@ pub async fn assert_signer_is_signing_cardano_transactions( }, Err(err) => Err(anyhow!(err).context(format!("Request to `{url}` failed"))), } + } + + match attempt!(10, Duration::from_millis(1000), { + fetch_cardano_transaction_snapshot_message(url.clone(), expected_epoch_min).await }) { AttemptResult::Ok(artifact) => { info!("Signer signed a Cardano transactions artifact"; "certificate_hash" => &artifact.certificate_hash); @@ -323,11 +375,20 @@ pub async fn assert_node_producing_cardano_stake_distribution( let url = format!("{aggregator_endpoint}/artifact/cardano-stake-distributions"); info!("Waiting for the aggregator to produce a Cardano stake distribution"); - match attempt!(45, Duration::from_millis(2000), { + async fn fetch_last_cardano_stake_distribution_message( + url: String, + ) -> StdResult> { match reqwest::get(url.clone()).await { Ok(response) => match response.status() { - StatusCode::OK => match response.json::().await.as_deref() { - Ok([stake_distribution, ..]) => Ok(Some((stake_distribution.hash.clone(), stake_distribution.epoch))), + StatusCode::OK => match response + .json::() + .await + .as_deref() + { + Ok([stake_distribution, ..]) => Ok(Some(( + stake_distribution.hash.clone(), + stake_distribution.epoch, + ))), Ok(&[]) => Ok(None), Err(err) => Err(anyhow!("Invalid Cardano stake distribution body : {err}",)), }, @@ -335,6 +396,10 @@ pub async fn assert_node_producing_cardano_stake_distribution( }, Err(err) => Err(anyhow!(err).context(format!("Request to `{url}` failed"))), } + } + + match attempt!(45, Duration::from_millis(2000), { + fetch_last_cardano_stake_distribution_message(url.clone()).await }) { AttemptResult::Ok((hash, epoch)) => { info!("Aggregator produced a Cardano stake distribution"; "hash" => &hash, "epoch" => #?epoch); @@ -359,7 +424,10 @@ pub async fn assert_signer_is_signing_cardano_stake_distribution( expected_epoch_min ); - match attempt!(10, Duration::from_millis(1000), { + async fn fetch_cardano_stake_distribution_message( + url: String, + expected_epoch_min: Epoch, + ) -> StdResult> { match reqwest::get(url.clone()).await { Ok(response) => match response.status() { StatusCode::OK => match response.json::().await { @@ -376,6 +444,10 @@ pub async fn assert_signer_is_signing_cardano_stake_distribution( }, Err(err) => Err(anyhow!(err).context(format!("Request to `{url}` failed"))), } + } + + match attempt!(10, Duration::from_millis(1000), { + fetch_cardano_stake_distribution_message(url.clone(), expected_epoch_min).await }) { AttemptResult::Ok(cardano_stake_distribution) => { info!("Signer signed a Cardano stake distribution"; "certificate_hash" => &cardano_stake_distribution.certificate_hash); @@ -395,7 +467,7 @@ pub async fn assert_is_creating_certificate_with_enough_signers( ) -> StdResult<()> { let url = format!("{aggregator_endpoint}/certificate/{certificate_hash}"); - match attempt!(10, Duration::from_millis(1000), { + async fn fetch_certificate_message(url: String) -> StdResult> { match reqwest::get(url.clone()).await { Ok(response) => match response.status() { StatusCode::OK => match response.json::().await { @@ -407,6 +479,10 @@ pub async fn assert_is_creating_certificate_with_enough_signers( }, Err(err) => Err(anyhow!(err).context(format!("Request to `{url}` failed"))), } + } + + match attempt!(10, Duration::from_millis(1000), { + fetch_certificate_message(url.clone()).await }) { AttemptResult::Ok(certificate) => { info!("Aggregator produced a certificate"; "certificate" => ?certificate); From f369ef3c994d258b701a1e214c1430a4a35b4c4c Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 18 Dec 2024 18:32:17 +0100 Subject: [PATCH 23/24] docs: update CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d1bbcee5d..f7ba18ac22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ As a minor extension, we have adopted a slightly different versioning convention - Build and publish both a `stable` version (for release networks) and an `unstable` version (for testing networks) of the explorer. +- **UNSTABLE** Cardano database incremental certification: + + - Implement the artifact routes of the aggregator for the signed entity type `CardanoDatabase`. + - Crates versions: | Crate | Version | From 57efde68a19df2289fd41e715893053a8168e5fd Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 20 Dec 2024 15:12:57 +0100 Subject: [PATCH 24/24] chore: upgrade crates versions and openapi.yaml version * mithril-aggregator from '0.6.4' to '0.6.5' * mithril-common from '0.4.97' to '0.4.98' * mithril-end-to-end from '0.4.55' to '0.4.56' * openapi.yaml from '0.1.38' to '0.1.39' --- Cargo.lock | 6 +++--- mithril-aggregator/Cargo.toml | 2 +- mithril-common/Cargo.toml | 2 +- mithril-test-lab/mithril-end-to-end/Cargo.toml | 2 +- openapi.yaml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ee49fef20c..dd03697bc8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3586,7 +3586,7 @@ dependencies = [ [[package]] name = "mithril-aggregator" -version = "0.6.4" +version = "0.6.5" dependencies = [ "anyhow", "async-trait", @@ -3743,7 +3743,7 @@ dependencies = [ [[package]] name = "mithril-common" -version = "0.4.97" +version = "0.4.98" dependencies = [ "anyhow", "async-trait", @@ -3814,7 +3814,7 @@ dependencies = [ [[package]] name = "mithril-end-to-end" -version = "0.4.55" +version = "0.4.56" dependencies = [ "anyhow", "async-recursion", diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml index b1dd921747..f6ccc5cf16 100644 --- a/mithril-aggregator/Cargo.toml +++ b/mithril-aggregator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-aggregator" -version = "0.6.4" +version = "0.6.5" description = "A Mithril Aggregator server" authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-common/Cargo.toml b/mithril-common/Cargo.toml index 460cfc4960..613e403a91 100644 --- a/mithril-common/Cargo.toml +++ b/mithril-common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-common" -version = "0.4.97" +version = "0.4.98" description = "Common types, interfaces, and utilities for Mithril nodes." authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-test-lab/mithril-end-to-end/Cargo.toml b/mithril-test-lab/mithril-end-to-end/Cargo.toml index a01917b289..783ca01a39 100644 --- a/mithril-test-lab/mithril-end-to-end/Cargo.toml +++ b/mithril-test-lab/mithril-end-to-end/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-end-to-end" -version = "0.4.55" +version = "0.4.56" authors = { workspace = true } edition = { workspace = true } documentation = { workspace = true } diff --git a/openapi.yaml b/openapi.yaml index 52aafb59ca..643407cb28 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -4,7 +4,7 @@ info: # `mithril-common/src/lib.rs` file. If you plan to update it # here to reflect changes in the API, please also update the constant in the # Rust file. - version: 0.1.38 + version: 0.1.39 title: Mithril Aggregator Server description: | The REST API provided by a Mithril Aggregator Node in a Mithril network.