diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 32e90bdb..0832ba6e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,15 +1,9 @@ name: Release on: - push: - tags: - - "v*" - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} + release: + types: [published] jobs: - # Compiles binaries using a bulder matrix binary_build: name: Build release binaries strategy: @@ -111,51 +105,6 @@ jobs: name: ${{ matrix.name }} path: ${{ matrix.name }} - # Creates a GitHub Container package using multi-arch docker builds - container_build: - runs-on: ubuntu-latest - - permissions: - contents: read - packages: write - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=semver,pattern=v{{major}} - type=semver,pattern=v{{major}}.{{minor}} - type=semver,pattern=v{{version}} - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push - uses: docker/build-push-action@v4 - with: - context: . - # temporarly removing arm platform until we resolve the ca-certificates issue - #platforms: linux/arm64,linux/amd64 - platforms: linux/amd64 - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - # Create GitHub release with Rust build targets and release notes github_release: name: Create GitHub Release needs: [binary_build, container_build] @@ -188,3 +137,13 @@ jobs: files: oura-*/oura-* body_path: RELEASE.md draft: true + + - name: Deploying to Amazon S3 + uses: dvelasquez/deploy-s3-action@main + with: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} + AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} + BUCKET_PATH: "/" + DIST_LOCATION_CODE: oura-*/oura-* diff --git a/Cargo.toml b/Cargo.toml index 8f1f5e6c..711c9a5e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "oura" description = "The tail of Cardano" -version = "2.0.0-alpha.2" +version = "1.8.1" edition = "2021" repository = "https://github.com/txpipe/oura" homepage = "https://github.com/txpipe/oura" @@ -12,22 +12,13 @@ authors = ["Santiago Carmuega "] [dependencies] -# pallas = "0.19.0-alpha.0" -# pallas = { path = "../pallas/pallas" } -pallas = { git = "https://github.com/txpipe/pallas" } - -gasket = { version = "^0.4", features = ["derive"] } -# gasket = { path = "../../construkts/gasket-rs/gasket", features = ["derive"] } -# gasket = { git = "https://github.com/construkts/gasket-rs.git", features = ["derive"] } - -utxorpc-spec-ledger = "1.0.0-alpha.0" -# utxorpc-spec-ledger = { path = "../../utxorpc/spec/ledger/gen/rust" } -# utxorpc-spec-ledger = { git = "https://github.com/utxorpc/spec.git" } - +pallas = { git = "https://github.com/Emurgo/pallas", tag = "conway_2" } +# pallas = { git = "https://github.com/txpipe/pallas" } +# pallas = { path = "../../pallas/pallas" } hex = "0.4.3" net2 = "0.2.37" bech32 = "0.9.1" -clap = { version = "4.2.7", features = ["derive"] } +clap = "3.2.22" log = "0.4.17" env_logger = "0.10.0" crossterm = "0.25" @@ -38,20 +29,17 @@ config = { version = "0.13.2", default-features = false, features = [ "json", ] } serde = { version = "1.0.152", features = ["derive"] } -serde_json = { version = "1.0.89" } +serde_json = { version = "1.0.89", features = ["arbitrary_precision"] } strum = "0.24" strum_macros = "0.24" prometheus_exporter = { version = "0.8.5", default-features = false } unicode-truncate = "0.2.0" -thiserror = "1.0.39" -indicatif = "0.17.3" -lazy_static = "1.4.0" -tracing = "0.1.37" -tracing-subscriber = "0.3.17" -file-rotate = "0.7.1" -tokio = { version = "1", features = ["rt"] } -async-trait = "0.1.68" -reqwest = { version = "0.11", features = ["json"] } + +# feature logs +file-rotate = { version = "0.7.1", optional = true } + +# feature: webhook +reqwest = { version = "0.11", optional = true, features = ["blocking", "json"] } # feature: kafkasink kafka = { version = "0.8.0", optional = true } @@ -68,11 +56,14 @@ aws-sdk-sqs = { version = "0.14.0", optional = true } aws-sdk-lambda = { version = "0.14.0", optional = true } aws-sdk-s3 = { version = "0.14.0", optional = true } +# features: elasticsearch || aws +tokio = { version = "1.24.2", optional = true, features = ["rt"] } + # required for CI to complete successfully openssl = { version = "0.10", optional = true, features = ["vendored"] } # redis support -redis = { version = "0.23.0", optional = true, features = ["tokio-comp"] } +redis = { version = "0.21.6", optional = true, features = ["tokio-comp"] } # features: gcp google-cloud-pubsub = { version = "0.12.0", optional = true } @@ -81,11 +72,15 @@ google-cloud-googleapis = { version = "0.7.0", optional = true } # features: rabbitmqsink lapin = { version = "2.1.1", optional = true } -# features: deno -deno_core = { version = "0.175.0", optional = true } -serde_v8 = { version = "0.86.0", optional = true } -deno_runtime = { version = "0.101.0", optional = true } - [features] -default = ["deno"] -deno = ["deno_core", "deno_runtime", "serde_v8"] +default = [] +web = ["reqwest"] +logs = ["file-rotate"] +webhook = ["web"] +kafkasink = ["kafka", "openssl"] +elasticsink = ["elasticsearch", "tokio"] +fingerprint = ["murmur3"] +aws = ["aws-config", "aws-sdk-sqs", "aws-sdk-lambda", "aws-sdk-s3", "tokio"] +redissink = ["redis", "tokio"] +gcp = ["google-cloud-pubsub", "google-cloud-googleapis", "tokio", "web"] +rabbitmqsink = ["lapin", "tokio"] diff --git a/README.md b/README.md index 6f5c37e9..40651b1f 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,6 @@
-> **Warning** -> `main` branch is now tracking **V2**. This new version is a complete overhaul of the processing pipeline, multiple breaking changes. If you're looking for **V1**, you can switch to the long-term support branch named `lts/v1` - ## Introduction We have tools to "explore" the Cardano blockchain, which are useful when you know what you're looking for. We argue that there's a different, complementary use-case which is to "observe" the blockchain and react to particular event patterns. @@ -35,7 +32,7 @@ In this terminal recording we get to see a few mins of live output from a testne All the heavy lifting required to communicate with the Cardano node is done by the [Pallas](https://github.com/txpipe/pallas) library, which provides an implementation of the Ouroboros multiplexer and a few of the required mini-protocol state-machines (ChainSync and LocalState in particular). -The data pipeline is implemented by the [Gasket](https://github.com/construkts/gasket-rs) library which provides a framework for building staged, event-driven applications. Under this abstraction, each component of the pipeline (aka: _Stage_) runs in its own thread and communicates with other stages by sending messages (very similar to the _Actor pattern_). +The data pipeline makes heavy use (maybe a bit too much) of multi-threading and mpsc channels provided by Rust's `std::sync` library. ## Use Cases @@ -49,7 +46,7 @@ Similar to the well-known db-sync tool provided by IOHK, _Oura_ can be used as a Given its small memory / cpu footprint, _Oura_ can be deployed side-by-side with your Cardano node even in resource-constrained environments, such as Raspberry PIs. -### As A Trigger Of Custom Actions +### As a Trigger of Custom Actions _Oura_ running in `daemon` mode can be configured to use custom filters to pinpoint particular transaction patterns and trigger actions whenever it finds a match. For example: send an email when a particular policy / asset combination appears in a transaction; call an AWS Lambda function when a wallet delegates to a particular pool; send a http-call to a webhook each time a metadata key appears in the TX payload; @@ -69,46 +66,53 @@ Oura is in its essence just a pipeline for processing events. Each stage of the ## Feature Status -- Data Types - - CBOR blocks - - CBOR txs - - Oura v1 model (for backward-compatibility) - - Parsed Txs (structured objects with all tx data) - - Generic JSON (any kind of JSON values) - Sources - - chain-sync from local node - - chain-sync + block-fetch from remote relay node - - S3 bucket with block data - - Kafka topic with block data + - [x] chain-sync full-block (node-to-client) + - [x] chain-sync + block-fetch (node-to-node) + - [x] Parsing of Shelley-compatible blocks (Shelley, Allegra, Mary, Alonzo) + - [x] Parsing of Byron blocks - Sinks - - Kafka topic - - Elasticsearch index / data stream - - Rotating log files with compression - - Redis streams - - AWS SQS queue - - AWS Lambda call - - AWS S3 objects - - GCP PubSub - - GCP Cloud Function - - Azure Sinks - - webhook (http post) - - terminal (append-only, tail-like) + - [x] Kafka topic + - [x] Elasticsearch index / data stream + - [x] Rotating log files with compression + - [x] Redis streams + - [x] AWS SQS queue + - [x] AWS Lambda call + - [x] AWS S3 objects + - [ ] GCP Sinks + - [ ] Azure Sinks + - [x] webhook (http post) + - [x] terminal (append-only, tail-like) + - [x] RabbitMQ +- Events / Parsers + - [x] block events (start, end) + - [x] transaction events (inputs, outputs, assets) + - [x] metadata events (labels, content) + - [x] mint events (policy, asset, quantity) + - [x] pool registrations events + - [x] delegation events + - [x] CIP-25 metadata parser (image, files) + - [ ] CIP-15 metadata parser - Filters - - Parse block / tx CBOR - - Split block into txs - - Select Txs by matching rules (address, metadata, policies, etc) - - Enrich tx data with related inputs - - Custom Typescript code (uses Deno) - - Custom WASM plugin - - Rollback buffer with compensating actions + - [x] cherry pick by event type (block, tx, mint, cert, etc) + - [x] cherry pick by asset subject (policy, name, etc) + - [x] cherry pick by metadata keys + - [ ] cherry pick by block property (size, tx count) + - [ ] cherry pick by tx property (fee, has native script, has plutus script, etc) + - [ ] cherry pick by utxo property (address, asset, amount range) + - [ ] enrich events with policy info from external metadata service + - [ ] enrich input tx info from Blockfrost API + - [ ] enrich addresses descriptions using ADAHandle - Other - - stateful chain cursor to recover from restarts - - buffer stage to hold blocks until they reach a certain depth - - pipeline metrics to track the progress and performance + - [x] stateful chain cursor to recover from restarts + - [x] buffer stage to hold blocks until they reach a certain depth + - [x] pipeline metrics to track the progress and performance ## Known Limitations -- Oura reads events from minted blocks / transactions. Support for querying the mempool is not yet implemented. +- ~~Oura only knows how to process blocks from the Shelley era. We are working on adding support for Byron in a future release.~~ (available since v1.2) +- Oura reads events from minted blocks / transactions. Support for querying the mempool is planned for a future release. +- ~~Oura will notify about chain rollbacks as a new event. The business logic for "undoing" the already processed events is a responsability of the consumer. We're working on adding support for a "buffer" filter stage which can hold blocks until they reach a configurable depth (number of confirmations).~~ (rollback buffer available since v1.2) ## Contributing diff --git a/assets/denopkgs/v2AlphaOuraUtils/mod.ts b/assets/denopkgs/v2AlphaOuraUtils/mod.ts deleted file mode 100644 index 691bb1cf..00000000 --- a/assets/denopkgs/v2AlphaOuraUtils/mod.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./ouraTypes.ts"; -export * from "./plutusData.ts"; diff --git a/assets/denopkgs/v2AlphaOuraUtils/ouraTypes.ts b/assets/denopkgs/v2AlphaOuraUtils/ouraTypes.ts deleted file mode 100644 index c38964d9..00000000 --- a/assets/denopkgs/v2AlphaOuraUtils/ouraTypes.ts +++ /dev/null @@ -1,254 +0,0 @@ -export type Era = - | "Undefined" - | "Unknown" - | "Byron" - | "Shelley" - | "Allegra" - | "Mary" - | "Alonzo" - | "Babbage"; - -export type GenericJson = Record; - -export type MetadatumRendition = { - map_json?: GenericJson; - array_json?: GenericJson; - int_scalar?: string; - text_scalar?: string; - bytes_hex?: string; -}; - -export type MetadataRecord = { - label: string; - content: MetadatumRendition; -}; - -export type CIP25AssetRecord = { - version: string; - policy: string; - asset: string; - name: string | null; - image: string | null; - media_type: string | null; - description: string | null; - raw_json: GenericJson; -}; - -export type CIP15AssetRecord = { - voting_key: string; - stake_pub: string; - reward_address: string; - nonce: number; - raw_json: GenericJson; -}; - -export type TxInputRecord = { - tx_id: string; - index: number; -}; - -export type OutputAssetRecord = { - policy: string; - asset: string; - asset_ascii: string | null; - amount: number; -}; - -export type TxOutputRecord = { - address: string; - amount: number; - assets: OutputAssetRecord[] | null; - datum_hash: string | null; - inline_datum: PlutusDatumRecord | null; -}; - -export type MintRecord = { - policy: string; - asset: string; - quantity: number; -}; - -export type WithdrawalRecord = { - reward_account: string; - coin: number; -}; - -export type TransactionRecord = { - hash: string; - fee: number; - ttl: number | null; - validity_interval_start: number | null; - network_id: number | null; - input_count: number; - collateral_input_count: number; - has_collateral_output: boolean; - output_count: number; - mint_count: number; - total_output: number; - - // include_details - metadata: MetadataRecord[] | null; - inputs: TxInputRecord[] | null; - outputs: TxOutputRecord[] | null; - collateral_inputs: TxInputRecord[] | null; - collateral_output: TxOutputRecord | null; - mint: MintRecord[] | null; - vkey_witnesses: VKeyWitnessRecord[] | null; - native_witnesses: NativeWitnessRecord[] | null; - plutus_witnesses: PlutusWitnessRecord[] | null; - plutus_redeemers: PlutusRedeemerRecord[] | null; - plutus_data: PlutusDatumRecord[] | null; - withdrawals: WithdrawalRecord[] | null; - size: number; -}; - -export type EventContext = { - block_hash: string | null; - block_number: number | null; - slot: number | null; - timestamp: number | null; - tx_idx: number | null; - tx_hash: string | null; - input_idx: number | null; - output_idx: number | null; - output_address: string | null; - certificate_idx: number | null; -}; - -export type StakeCredential = { - addr_keyhash?: string; - scripthash?: string; -}; - -export type VKeyWitnessRecord = { - vkey_hex: string; - signature_hex: string; -}; - -export type NativeWitnessRecord = { - policy_id: string; - script_json: GenericJson; -}; - -export type PlutusWitnessRecord = { - script_hash: string; - script_hex: string; -}; - -export type PlutusRedeemerRecord = { - purpose: string; - ex_units_mem: number; - ex_units_steps: number; - input_idx: number; - plutus_data: GenericJson; -}; - -export type PlutusDatumRecord = { - datum_hash: string; - plutus_data: GenericJson; -}; - -export type BlockRecord = { - era: Era; - epoch: number | null; - epoch_slot: number | null; - body_size: number; - issuer_vkey: string; - vrf_vkey: string; - tx_count: number; - slot: number; - hash: string; - number: number; - previous_hash: string; - cbor_hex: string | null; - transactions: TransactionRecord[] | null; -}; - -export type CollateralRecord = { - tx_id: string; - index: number; -}; - -export type PoolRegistrationRecord = { - operator: string; - vrf_keyhash: string; - pledge: number; - cost: number; - margin: number; - reward_account: string; - pool_owners: string[]; - relays: string[]; - pool_metadata: string | null; - pool_metadata_hash: string | null; -}; - -export type RollBackRecord = { - block_slot: number; - block_hash: string; -}; - -export type MoveInstantaneousRewardsCertRecord = { - from_reserves: boolean; - from_treasury: boolean; - to_stake_credentials: Array<[StakeCredential, number]> | null; - to_other_pot: number | null; -}; - -export type NativeScriptRecord = { - policy_id: string; - script: GenericJson; -}; - -export type PlutusScriptRecord = { - hash: string; - data: string; -}; - -export type StakeRegistrationRecord = { credential: StakeCredential }; - -export type StakeDeregistrationRecord = { credential: StakeCredential }; - -export type StakeDelegation = { - credential: StakeCredential; - pool_hash: string; -}; - -export type PoolRetirementRecord = { - pool: string; - epoch: number; -}; - -export type GenesisKeyDelegationRecord = {}; - -export type Event = { - context: EventContext; - fingerprint?: string; - - block?: BlockRecord; - block_end?: BlockRecord; - transaction?: TransactionRecord; - transaction_end?: TransactionRecord; - tx_input?: TxInputRecord; - tx_output?: TxOutputRecord; - output_asset?: OutputAssetRecord; - metadata?: MetadataRecord; - v_key_witness?: VKeyWitnessRecord; - native_witness?: NativeWitnessRecord; - plutus_witness?: PlutusWitnessRecord; - plutus_redeemer?: PlutusRedeemerRecord; - plutus_datum?: PlutusDatumRecord; - cip25_asset?: CIP25AssetRecord; - cip15_asset?: CIP15AssetRecord; - mint?: MintRecord; - collateral?: CollateralRecord; - native_script?: NativeScriptRecord; - plutus_script?: PlutusScriptRecord; - stake_registration?: StakeRegistrationRecord; - stake_deregistration?: StakeDeregistrationRecord; - stake_delegation?: StakeDelegation; - pool_registration?: PoolRegistrationRecord; - pool_retirement?: PoolRetirementRecord; - genesis_key_delegation?: GenesisKeyDelegationRecord; - move_instantaneous_rewards_cert?: MoveInstantaneousRewardsCertRecord; - roll_back?: RollBackRecord; -}; diff --git a/assets/denopkgs/v2AlphaOuraUtils/plutusData.ts b/assets/denopkgs/v2AlphaOuraUtils/plutusData.ts deleted file mode 100644 index ef9ec2b4..00000000 --- a/assets/denopkgs/v2AlphaOuraUtils/plutusData.ts +++ /dev/null @@ -1,32 +0,0 @@ -import * as hex from "https://deno.land/std/encoding/hex.ts"; -import * as oura from "./ouraTypes.ts"; - -export type PlutusMap = Array<{ - k: { bytes: string }; - v: { bytes: string }; -}>; - -const TEXT_ENCODER = new TextEncoder(); -const TEXT_DECODER = new TextDecoder(); - -function isReadableAscii(raw: string): boolean { - return raw.split("").every((char) => { - const charCode = char.charCodeAt(0); - return 0x20 <= charCode && charCode <= 0x7e; - }); -} - -function hexToText(hexString: string): string { - const hexBytes = TEXT_ENCODER.encode(hexString); - const utfBytes = hex.decode(hexBytes); - return TEXT_DECODER.decode(utfBytes); -} - -export function plutusMapToPlainJson(source: PlutusMap): oura.GenericJson { - return source.reduce((all, item) => { - const key = hexToText(item.k.bytes); - const maybeText = hexToText(item.v.bytes); - all[key] = isReadableAscii(maybeText) ? maybeText : item.v.bytes; - return all; - }, {}); -} diff --git a/book/src/sinks/aws_s3.md b/book/src/sinks/aws_s3.md index 5216d890..507c7cd5 100644 --- a/book/src/sinks/aws_s3.md +++ b/book/src/sinks/aws_s3.md @@ -29,7 +29,7 @@ max_retries = 5 - `bucket`: The name of the bucket to store the blocks. - `prefix`: A prefix to prepend on each object's key. - `naming`: One of the available naming conventions (see section below) -- `content`: Either `Cbor` for binary encoding or `CborHex` for plain text hex representation of the CBOR +- `content`: Either `Cbor` for binary encoding or `CborHex` for plain text hex representation of the CBOR. Or `Json` for json string representation. - `max_retries`: The max number of send retries before exiting the pipeline with an error. IMPORTANT: For this sink to work correctly, the `include_block_cbor` option should be enabled in the source sink configuration (see [mapper options](../advanced/mapper_options.md)). @@ -42,6 +42,7 @@ S3 Buckets allow the user to query by object prefix. It's important to use a nam - `Hash`: formats the key using `"{hash}"` - `SlotHash`: formats the key using `"{slot}.{hash}"` - `BlockHash`: formats the key using `"{block_num}.{hash}"` +- `BlockNumber`: formats the key using `"{block_num}"` - `EpochHash`: formats the key using `"{epoch}.{hash}"` - `EpochSlotHash`: formats the key using `"{epoch}.{slot}.{hash}"` - `EpochBlockHash`: formats the key using `"{epoch}.{block_num}.{hash}"` @@ -52,6 +53,7 @@ The sink provides two options for encoding the content of the object: - `Cbor`: the S3 object will contain the raw, unmodified CBOR value in binary format. The content type of the object in this case will be "application/cbor". - `CborHex`: the S3 object will contain the CBOR payload of the block encoded as a hex string. The content type of the object in this case will be "text/plain". +- `Json`: the S3 object will contain block encoded as a json string. The content type of the object in this case will be "application/json". ## Metadata diff --git a/src/bin/oura/console.rs b/src/bin/oura/console.rs deleted file mode 100644 index fcdb4f4c..00000000 --- a/src/bin/oura/console.rs +++ /dev/null @@ -1,208 +0,0 @@ -use std::{ - sync::Mutex, - time::{Duration, Instant}, -}; - -use gasket::{metrics::Reading, runtime::Tether}; -use lazy_static::{__Deref, lazy_static}; -use log::Log; - -#[derive(clap::ValueEnum, Clone, Default)] -pub enum Mode { - /// shows progress as a plain sequence of logs - #[default] - Plain, - /// shows aggregated progress and metrics - Tui, -} - -struct TuiConsole { - chainsync_progress: indicatif::ProgressBar, - fetched_blocks: indicatif::ProgressBar, - plexer_ops_count: indicatif::ProgressBar, - filter_ops_count: indicatif::ProgressBar, - mapper_ops_count: indicatif::ProgressBar, - sink_ops_count: indicatif::ProgressBar, -} - -impl TuiConsole { - fn build_counter_spinner( - name: &str, - container: &indicatif::MultiProgress, - ) -> indicatif::ProgressBar { - container.add( - indicatif::ProgressBar::new_spinner().with_style( - indicatif::ProgressStyle::default_spinner() - .template(&format!( - "{{spinner}} {name:<20} {{msg:<20}} {{pos:>8}} | {{per_sec}}" - )) - .unwrap(), - ), - ) - } - - fn new() -> Self { - let container = indicatif::MultiProgress::new(); - - Self { - chainsync_progress: container.add( - indicatif::ProgressBar::new(0).with_style( - indicatif::ProgressStyle::default_bar() - .template("chainsync progress: {bar} {pos}/{len} eta: {eta}\n{msg}") - .unwrap(), - ), - ), - fetched_blocks: Self::build_counter_spinner("fetched blocks", &container), - plexer_ops_count: Self::build_counter_spinner("plexer ops", &container), - filter_ops_count: Self::build_counter_spinner("filter ops", &container), - mapper_ops_count: Self::build_counter_spinner("mapper ops", &container), - sink_ops_count: Self::build_counter_spinner("sink ops", &container), - } - } - - fn refresh<'a>(&self, tethers: impl Iterator) { - for tether in tethers { - let state = match tether.check_state() { - gasket::runtime::TetherState::Dropped => "dropped!", - gasket::runtime::TetherState::Blocked(_) => "blocked!", - gasket::runtime::TetherState::Alive(x) => match x { - gasket::runtime::StagePhase::Bootstrap => "bootstrapping...", - gasket::runtime::StagePhase::Working => "working...", - gasket::runtime::StagePhase::Teardown => "tearing down...", - gasket::runtime::StagePhase::Ended => "ended", - }, - }; - - match tether.read_metrics() { - Ok(readings) => { - for (key, value) in readings { - match (tether.name(), key, value) { - (_, "chain_tip", Reading::Gauge(x)) => { - self.chainsync_progress.set_length(x as u64); - } - (_, "latest_block", Reading::Gauge(x)) => { - self.chainsync_progress.set_position(x as u64); - } - (_, "fetched_blocks", Reading::Count(x)) => { - self.fetched_blocks.set_position(x); - self.fetched_blocks.set_message(state); - } - ("plexer", "ops_count", Reading::Count(x)) => { - self.plexer_ops_count.set_position(x); - self.plexer_ops_count.set_message(state); - } - ("filter", "ops_count", Reading::Count(x)) => { - self.filter_ops_count.set_position(x); - self.filter_ops_count.set_message(state); - } - ("mapper", "ops_count", Reading::Count(x)) => { - self.mapper_ops_count.set_position(x); - self.mapper_ops_count.set_message(state); - } - ("sink", "ops_count", Reading::Count(x)) => { - self.sink_ops_count.set_position(x); - self.sink_ops_count.set_message(state); - } - _ => (), - } - } - } - Err(err) => { - println!("couldn't read metrics"); - dbg!(err); - } - } - } - } -} - -impl Log for TuiConsole { - fn enabled(&self, metadata: &log::Metadata) -> bool { - metadata.level() >= log::Level::Info - } - - fn log(&self, record: &log::Record) { - self.chainsync_progress - .set_message(format!("{}", record.args())) - } - - fn flush(&self) {} -} - -struct PlainConsole { - last_report: Mutex, -} - -impl PlainConsole { - fn new() -> Self { - Self { - last_report: Mutex::new(Instant::now()), - } - } - - fn refresh<'a>(&self, tethers: impl Iterator) { - let mut last_report = self.last_report.lock().unwrap(); - - if last_report.elapsed() <= Duration::from_secs(10) { - return; - } - - for tether in tethers { - match tether.check_state() { - gasket::runtime::TetherState::Dropped => { - log::error!("[{}] stage tether has been dropped", tether.name()); - } - gasket::runtime::TetherState::Blocked(_) => { - log::warn!( - "[{}] stage tehter is blocked or not reporting state", - tether.name() - ); - } - gasket::runtime::TetherState::Alive(state) => { - log::debug!("[{}] stage is alive with state: {:?}", tether.name(), state); - match tether.read_metrics() { - Ok(readings) => { - for (key, value) in readings { - log::debug!("[{}] metric `{}` = {:?}", tether.name(), key, value); - } - } - Err(err) => { - log::error!("[{}] error reading metrics: {}", tether.name(), err) - } - } - } - } - } - - *last_report = Instant::now(); - } -} - -lazy_static! { - static ref TUI_CONSOLE: TuiConsole = TuiConsole::new(); -} - -lazy_static! { - static ref PLAIN_CONSOLE: PlainConsole = PlainConsole::new(); -} - -pub fn initialize(mode: &Option) { - match mode { - Some(Mode::Tui) => log::set_logger(TUI_CONSOLE.deref()) - .map(|_| log::set_max_level(log::LevelFilter::Info)) - .unwrap(), - _ => tracing::subscriber::set_global_default( - tracing_subscriber::FmtSubscriber::builder() - .with_max_level(tracing::Level::TRACE) - .finish(), - ) - .unwrap(), - } -} - -pub fn refresh<'a>(mode: &Option, tethers: impl Iterator) { - match mode { - Some(Mode::Tui) => TUI_CONSOLE.refresh(tethers), - _ => PLAIN_CONSOLE.refresh(tethers), - } -} diff --git a/src/bin/oura/daemon.rs b/src/bin/oura/daemon.rs index cf540a57..e33d71a6 100644 --- a/src/bin/oura/daemon.rs +++ b/src/bin/oura/daemon.rs @@ -1,196 +1,348 @@ -use gasket::runtime::Tether; -use oura::{filters, framework::*, sinks, sources}; -use pallas::ledger::traverse::wellknown::GenesisValues; +use std::{sync::Arc, thread::JoinHandle}; + +use clap::ArgMatches; +use config::{Config, ConfigError, Environment, File}; +use log::debug; use serde::Deserialize; -use std::{collections::VecDeque, time::Duration}; -use crate::console; +use oura::{ + pipelining::{ + BootstrapResult, FilterProvider, PartialBootstrapResult, SinkProvider, SourceProvider, + StageReceiver, + }, + sources::{MagicArg, PointArg}, + utils::{cursor, metrics, ChainWellKnownInfo, Utils, WithUtils}, + Error, +}; + +use oura::filters::noop::Config as NoopFilterConfig; +use oura::filters::selection::Config as SelectionConfig; +use oura::sinks::assert::Config as AssertConfig; +use oura::sinks::stdout::Config as StdoutConfig; +use oura::sinks::terminal::Config as TerminalConfig; +use oura::sources::n2c::Config as N2CConfig; +use oura::sources::n2n::Config as N2NConfig; + +#[cfg(feature = "logs")] +use oura::sinks::logs::Config as WriterConfig; + +#[cfg(feature = "webhook")] +use oura::sinks::webhook::Config as WebhookConfig; + +#[cfg(feature = "kafkasink")] +use oura::sinks::kafka::Config as KafkaConfig; + +#[cfg(feature = "elasticsink")] +use oura::sinks::elastic::Config as ElasticConfig; + +#[cfg(feature = "aws")] +use oura::sinks::aws_sqs::Config as AwsSqsConfig; + +#[cfg(feature = "aws")] +use oura::sinks::aws_lambda::Config as AwsLambdaConfig; + +#[cfg(feature = "aws")] +use oura::sinks::aws_s3::Config as AwsS3Config; + +#[cfg(feature = "redissink")] +use oura::sinks::redis::Config as RedisConfig; + +#[cfg(feature = "gcp")] +use oura::sinks::gcp_pubsub::Config as GcpPubSubConfig; + +#[cfg(feature = "gcp")] +use oura::sinks::gcp_cloudfunction::Config as GcpCloudFunctionConfig; + +#[cfg(feature = "rabbitmqsink")] +use oura::sinks::rabbitmq::Config as RabbitmqConfig; + +#[cfg(feature = "fingerprint")] +use oura::filters::fingerprint::Config as FingerprintConfig; + +#[derive(Debug, Deserialize)] +#[serde(tag = "type")] +enum Source { + N2C(N2CConfig), + N2N(N2NConfig), +} + +fn bootstrap_source(config: Source, utils: Arc) -> PartialBootstrapResult { + match config { + Source::N2C(config) => WithUtils::new(config, utils).bootstrap(), + Source::N2N(config) => WithUtils::new(config, utils).bootstrap(), + } +} + +fn infer_magic_from_source(config: &Source) -> Option { + match config { + Source::N2C(config) => config.magic.clone(), + Source::N2N(config) => config.magic.clone(), + } +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "type")] +enum Filter { + Noop(NoopFilterConfig), + Selection(SelectionConfig), + + #[cfg(feature = "fingerprint")] + Fingerprint(FingerprintConfig), +} + +impl FilterProvider for Filter { + fn bootstrap(&self, input: StageReceiver) -> PartialBootstrapResult { + match self { + Filter::Noop(c) => c.bootstrap(input), + Filter::Selection(c) => c.bootstrap(input), + + #[cfg(feature = "fingerprint")] + Filter::Fingerprint(c) => c.bootstrap(input), + } + } +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "type")] +enum Sink { + Terminal(TerminalConfig), + Stdout(StdoutConfig), + Assert(AssertConfig), + + #[cfg(feature = "logs")] + Logs(WriterConfig), + + #[cfg(feature = "webhook")] + Webhook(WebhookConfig), + + #[cfg(feature = "kafkasink")] + Kafka(KafkaConfig), + + #[cfg(feature = "elasticsink")] + Elastic(ElasticConfig), + + #[cfg(feature = "aws")] + AwsSqs(AwsSqsConfig), + + #[cfg(feature = "aws")] + AwsLambda(AwsLambdaConfig), + + #[cfg(feature = "aws")] + AwsS3(AwsS3Config), + + #[cfg(feature = "redissink")] + Redis(RedisConfig), + + #[cfg(feature = "gcp")] + GcpPubSub(GcpPubSubConfig), + + #[cfg(feature = "gcp")] + GcpCloudFunction(GcpCloudFunctionConfig), + + #[cfg(feature = "rabbitmqsink")] + Rabbitmq(RabbitmqConfig), +} + +fn bootstrap_sink(config: Sink, input: StageReceiver, utils: Arc) -> BootstrapResult { + match config { + Sink::Terminal(c) => WithUtils::new(c, utils).bootstrap(input), + Sink::Stdout(c) => WithUtils::new(c, utils).bootstrap(input), + Sink::Assert(c) => WithUtils::new(c, utils).bootstrap(input), -#[derive(Deserialize)] + #[cfg(feature = "logs")] + Sink::Logs(c) => WithUtils::new(c, utils).bootstrap(input), + + #[cfg(feature = "webhook")] + Sink::Webhook(c) => WithUtils::new(c, utils).bootstrap(input), + + #[cfg(feature = "kafkasink")] + Sink::Kafka(c) => WithUtils::new(c, utils).bootstrap(input), + + #[cfg(feature = "elasticsink")] + Sink::Elastic(c) => WithUtils::new(c, utils).bootstrap(input), + + #[cfg(feature = "aws")] + Sink::AwsSqs(c) => WithUtils::new(c, utils).bootstrap(input), + + #[cfg(feature = "aws")] + Sink::AwsLambda(c) => WithUtils::new(c, utils).bootstrap(input), + + #[cfg(feature = "aws")] + Sink::AwsS3(c) => WithUtils::new(c, utils).bootstrap(input), + + #[cfg(feature = "redissink")] + Sink::Redis(c) => WithUtils::new(c, utils).bootstrap(input), + + #[cfg(feature = "gcp")] + Sink::GcpPubSub(c) => WithUtils::new(c, utils).bootstrap(input), + + #[cfg(feature = "gcp")] + Sink::GcpCloudFunction(c) => WithUtils::new(c, utils).bootstrap(input), + + #[cfg(feature = "rabbitmqsink")] + Sink::Rabbitmq(c) => WithUtils::new(c, utils).bootstrap(input), + } +} + +#[derive(Debug, Deserialize)] struct ConfigRoot { - source: sources::Config, - filters: Option>, - sink: sinks::Config, - intersect: IntersectConfig, - finalize: Option, - chain: Option, - retries: Option, + source: Source, + + #[serde(default)] + filters: Vec, + + sink: Sink, + + chain: Option, + + cursor: Option, + + metrics: Option, } impl ConfigRoot { - pub fn new(explicit_file: &Option) -> Result { - let mut s = config::Config::builder(); + pub fn new(explicit_file: Option) -> Result { + let mut s = Config::builder(); - // our base config will always be in /etc/scrolls - s = s.add_source(config::File::with_name("/etc/oura/daemon.toml").required(false)); + // our base config will always be in /etc/oura + s = s.add_source(File::with_name("/etc/oura/daemon.toml").required(false)); // but we can override it by having a file in the working dir - s = s.add_source(config::File::with_name("oura.toml").required(false)); + s = s.add_source(File::with_name("oura.toml").required(false)); // if an explicit file was passed, then we load it as mandatory - if let Some(explicit) = explicit_file.as_ref().and_then(|x| x.to_str()) { - s = s.add_source(config::File::with_name(explicit).required(true)); + if let Some(explicit) = explicit_file { + s = s.add_source(File::with_name(&explicit).required(true)); } // finally, we use env vars to make some last-step overrides - s = s.add_source(config::Environment::with_prefix("OURA").separator("_")); + s = s.add_source(Environment::with_prefix("OURA").separator("_")); s.build()?.try_deserialize() } } -struct Runtime { - source: Tether, - filters: Vec, - sink: Tether, -} - -impl Runtime { - fn all_tethers(&self) -> impl Iterator { - std::iter::once(&self.source) - .chain(self.filters.iter()) - .chain(std::iter::once(&self.sink)) - } - - fn should_stop(&self) -> bool { - self.all_tethers().any(|tether| match tether.check_state() { - gasket::runtime::TetherState::Alive(x) => { - matches!(x, gasket::runtime::StagePhase::Ended) - } - _ => true, - }) +fn define_chain_info( + explicit: Option, + magic: &MagicArg, +) -> Result { + match explicit { + Some(x) => Ok(x), + None => ChainWellKnownInfo::try_from_magic(**magic), } +} - fn shutdown(&self) { - for tether in self.all_tethers() { - let state = tether.check_state(); - log::warn!("dismissing stage: {} with state {:?}", tether.name(), state); - tether.dismiss_stage().expect("stage stops"); - - // Can't join the stage because there's a risk of deadlock, usually - // because a stage gets stuck sending into a port which depends on a - // different stage not yet dismissed. The solution is to either - // create a DAG of dependencies and dismiss in the - // correct order, or implement a 2-phase teardown where - // ports are disconnected and flushed before joining the - // stage. - - //tether.join_stage(); - } +fn define_cursor( + explicit: Option, + config: Option, +) -> Option { + match (explicit, config) { + (Some(x), _) => Some(cursor::Config::Memory(x)), + (_, x) => x, } } -fn define_gasket_policy(config: Option<&gasket::retries::Policy>) -> gasket::runtime::Policy { - gasket::runtime::Policy { - tick_timeout: std::time::Duration::from_secs(5).into(), - bootstrap_retry: config.cloned().unwrap_or_default(), - work_retry: config.cloned().unwrap_or_default(), - teardown_retry: config.cloned().unwrap_or_default(), +fn bootstrap_utils( + chain: ChainWellKnownInfo, + cursor: Option, + metrics: Option, +) -> Utils { + let mut utils = Utils::new(chain); + + if let Some(cursor) = cursor { + utils = utils.with_cursor(cursor); } -} -fn chain_stages<'a>( - source: &'a mut dyn StageBootstrapper, - filters: Vec<&'a mut dyn StageBootstrapper>, - sink: &'a mut dyn StageBootstrapper, -) { - let mut prev = source; - - for filter in filters { - let (to_next, from_prev) = gasket::messaging::tokio::channel(100); - prev.connect_output(to_next); - filter.connect_input(from_prev); - prev = filter; + if let Some(metrics) = metrics { + utils = utils.with_metrics(metrics); } - let (to_next, from_prev) = gasket::messaging::tokio::channel(100); - prev.connect_output(to_next); - sink.connect_input(from_prev); + utils } +/// Sets up the whole pipeline from configuration fn bootstrap( - mut source: sources::Bootstrapper, - mut filters: Vec, - mut sink: sinks::Bootstrapper, - policy: gasket::runtime::Policy, -) -> Result { - chain_stages( - &mut source, - filters - .iter_mut() - .map(|x| x as &mut dyn StageBootstrapper) - .collect::>(), - &mut sink, - ); - - let runtime = Runtime { - source: source.spawn(policy.clone()), - filters: filters - .into_iter() - .map(|x| x.spawn(policy.clone())) - .collect(), - sink: sink.spawn(policy), - }; + config: ConfigRoot, + explicit_cursor: Option, +) -> Result>, Error> { + let ConfigRoot { + source, + filters, + sink, + chain, + cursor, + metrics, + } = config; - Ok(runtime) -} + let magic = infer_magic_from_source(&source).unwrap_or_default(); -pub fn run(args: &Args) -> Result<(), Error> { - console::initialize(&args.console); + let chain = define_chain_info(chain, &magic)?; - let config = ConfigRoot::new(&args.config).map_err(Error::config)?; + let cursor = define_cursor(explicit_cursor, cursor); - let chain = config.chain.unwrap_or_default(); - let intersect = config.intersect; - let finalize = config.finalize; - let current_dir = std::env::current_dir().unwrap(); + let utils = Arc::new(bootstrap_utils(chain, cursor, metrics)); - // TODO: load from persistence mechanism - let cursor = Cursor::new(VecDeque::new()); + let mut threads = Vec::with_capacity(10); - let ctx = Context { - chain, - intersect, - finalize, - cursor, - current_dir, - }; + let (source_handle, source_rx) = bootstrap_source(source, utils.clone())?; + threads.push(source_handle); - let source = config.source.bootstrapper(&ctx)?; + let mut last_rx = source_rx; - let filters = config - .filters - .into_iter() - .flatten() - .map(|x| x.bootstrapper(&ctx)) - .collect::>()?; + for filter in filters.iter() { + let (filter_handle, filter_rx) = filter.bootstrap(last_rx)?; + threads.push(filter_handle); + last_rx = filter_rx; + } - let sink = config.sink.bootstrapper(&ctx)?; + let sink_handle = bootstrap_sink(sink, last_rx, utils)?; + threads.push(sink_handle); - let retries = define_gasket_policy(config.retries.as_ref()); - let runtime = bootstrap(source, filters, sink, retries)?; + Ok(threads) +} - log::info!("oura is running..."); +pub fn run(args: &ArgMatches) -> Result<(), Error> { + env_logger::init(); - while !runtime.should_stop() { - console::refresh(&args.console, runtime.all_tethers()); - std::thread::sleep(Duration::from_millis(1500)); - } + let explicit_config = match args.is_present("config") { + true => Some(args.value_of_t("config")?), + false => None, + }; + + let explicit_cursor = match args.is_present("cursor") { + true => Some(args.value_of_t("cursor")?), + false => None, + }; - log::info!("Oura is stopping..."); - runtime.shutdown(); + let root = ConfigRoot::new(explicit_config)?; + + debug!("daemon starting with this config: {:?}", root); + + let threads = bootstrap(root, explicit_cursor)?; + + // TODO: refactor into new loop that monitors thread health + for handle in threads { + handle.join().expect("error in pipeline thread"); + } Ok(()) } -#[derive(clap::Args)] -#[clap(author, version, about, long_about = None)] -pub struct Args { - #[clap(long, value_parser)] - //#[clap(description = "config file to load by the daemon")] - config: Option, - - #[clap(long, value_parser)] - //#[clap(description = "type of progress to display")], - console: Option, +/// Creates the clap definition for this sub-command +pub(crate) fn command_definition<'a>() -> clap::Command<'a> { + clap::Command::new("daemon") + .arg( + clap::Arg::new("config") + .long("config") + .takes_value(true) + .help("config file to load by the daemon"), + ) + .arg( + clap::Arg::new("cursor") + .long("cursor") + .takes_value(true) + .help( + "initial chain cursor, overrides configuration file, expects format `slot,hex-hash`", + ), + ) } diff --git a/src/bin/oura/dump.rs b/src/bin/oura/dump.rs new file mode 100644 index 00000000..b3d4ec05 --- /dev/null +++ b/src/bin/oura/dump.rs @@ -0,0 +1,195 @@ +use serde::Deserialize; +use std::{str::FromStr, sync::Arc}; + +use clap::ArgMatches; + +use oura::{ + mapper::Config as MapperConfig, + pipelining::{BootstrapResult, SinkProvider, SourceProvider, StageReceiver}, + sources::{AddressArg, BearerKind, IntersectArg, MagicArg}, + utils::{ChainWellKnownInfo, Utils, WithUtils}, +}; + +use oura::sinks::stdout::Config as StdoutConfig; +use oura::sources::n2c::Config as N2CConfig; +use oura::sources::n2n::Config as N2NConfig; + +#[cfg(feature = "logs")] +use oura::sinks::logs::Config as LogsConfig; + +use crate::Error; + +#[derive(Clone, Debug, Deserialize)] +pub enum PeerMode { + AsNode, + AsClient, +} + +impl FromStr for PeerMode { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_ref() { + "node" => Ok(PeerMode::AsNode), + "client" => Ok(PeerMode::AsClient), + _ => Err("can't parse peer mode (valid values: client|node)"), + } + } +} + +enum DumpSource { + N2C(N2CConfig), + N2N(N2NConfig), +} + +enum DumpSink { + Stdout(StdoutConfig), + #[cfg(feature = "logs")] + Logs(LogsConfig), +} + +fn bootstrap_sink(sink: DumpSink, input: StageReceiver, utils: Arc) -> BootstrapResult { + match sink { + DumpSink::Stdout(c) => WithUtils::new(c, utils).bootstrap(input), + + #[cfg(feature = "logs")] + DumpSink::Logs(c) => WithUtils::new(c, utils).bootstrap(input), + } +} + +pub fn run(args: &ArgMatches) -> Result<(), Error> { + env_logger::builder() + .filter_module("oura::dump", log::LevelFilter::Info) + .init(); + + let socket = args.value_of_t("socket")?; + + let bearer = match args.is_present("bearer") { + true => args.value_of_t("bearer")?, + #[cfg(target_family = "unix")] + false => BearerKind::Unix, + #[cfg(target_family = "windows")] + false => BearerKind::Tcp, + }; + + let magic = match args.is_present("magic") { + true => args.value_of_t("magic")?, + false => MagicArg::default(), + }; + + let intersect = match args.is_present("since") { + true => Some(IntersectArg::Point(args.value_of_t("since")?)), + false => None, + }; + + let mode = match (args.is_present("mode"), &bearer) { + (true, _) => args + .value_of_t("mode") + .expect("invalid value for 'mode' arg"), + (false, BearerKind::Tcp) => PeerMode::AsNode, + #[cfg(target_family = "unix")] + (false, BearerKind::Unix) => PeerMode::AsClient, + }; + + let output: Option = match args.is_present("output") { + #[cfg(feature = "logs")] + true => Some(args.value_of_t("output")?), + _ => None, + }; + + let mapper = MapperConfig { + include_block_end_events: true, + ..Default::default() + }; + + let well_known = ChainWellKnownInfo::try_from_magic(*magic)?; + + // TODO: map add cli arg to enable / disable cursor + + let utils = Arc::new(Utils::new(well_known)); + + #[allow(deprecated)] + let source_setup = match mode { + PeerMode::AsNode => DumpSource::N2N(N2NConfig { + address: AddressArg(bearer, socket), + magic: Some(magic), + well_known: None, + min_depth: 0, + mapper, + since: None, + intersect, + retry_policy: None, + finalize: None, + }), + PeerMode::AsClient => DumpSource::N2C(N2CConfig { + address: AddressArg(bearer, socket), + magic: Some(magic), + well_known: None, + min_depth: 0, + mapper, + since: None, + intersect, + retry_policy: None, + finalize: None, + }), + }; + + let sink_setup = match &output { + #[cfg(feature = "logs")] + Some(x) => DumpSink::Logs(LogsConfig { + output_path: Some(x.to_owned()), + ..Default::default() + }), + _ => DumpSink::Stdout(StdoutConfig { + ..Default::default() + }), + }; + + let (source_handle, source_output) = match source_setup { + DumpSource::N2C(c) => WithUtils::new(c, utils.clone()).bootstrap()?, + DumpSource::N2N(c) => WithUtils::new(c, utils.clone()).bootstrap()?, + }; + + let sink_handle = bootstrap_sink(sink_setup, source_output, utils)?; + + log::info!( + "Oura started dumping events to {}", + output.as_deref().unwrap_or("stdout") + ); + + sink_handle.join().map_err(|_| "error in sink thread")?; + source_handle.join().map_err(|_| "error in source thread")?; + + Ok(()) +} + +/// Creates the clap definition for this sub-command +pub(crate) fn command_definition<'a>() -> clap::Command<'a> { + clap::Command::new("dump") + .arg(clap::Arg::new("socket").required(true)) + .arg( + clap::Arg::new("bearer") + .long("bearer") + .takes_value(true) + .possible_values(["tcp", "unix"]), + ) + .arg(clap::Arg::new("magic").long("magic").takes_value(true)) + .arg( + clap::Arg::new("since") + .long("since") + .takes_value(true) + .help("point in the chain to start reading from, expects format `slot,hex-hash`"), + ) + .arg( + clap::Arg::new("mode") + .long("mode") + .takes_value(true) + .possible_values(["node", "client"]), + ) + .arg( + clap::Arg::new("output") + .long("output") + .takes_value(true) + .help("path-like prefix for the log files (fallbacks to stdout output)"), + ) +} diff --git a/src/bin/oura/main.rs b/src/bin/oura/main.rs index 24420043..6b26b7d7 100644 --- a/src/bin/oura/main.rs +++ b/src/bin/oura/main.rs @@ -1,22 +1,29 @@ -use clap::Parser; +mod daemon; +mod dump; +mod watch; + use std::process; -mod console; -mod daemon; +use clap::Command; -#[derive(Parser)] -#[clap(name = "Oura")] -#[clap(bin_name = "oura")] -#[clap(author, version, about, long_about = None)] -enum Oura { - Daemon(daemon::Args), -} +type Error = oura::Error; fn main() { - let args = Oura::parse(); + let args = Command::new("app") + .name("oura") + .about("the tail of cardano") + .version(env!("CARGO_PKG_VERSION")) + .subcommand(watch::command_definition()) + .subcommand(dump::command_definition()) + .subcommand(daemon::command_definition()) + .arg_required_else_help(true) + .get_matches(); - let result = match args { - Oura::Daemon(x) => daemon::run(&x), + let result = match args.subcommand() { + Some(("watch", args)) => watch::run(args), + Some(("dump", args)) => dump::run(args), + Some(("daemon", args)) => daemon::run(args), + _ => Err("nothing to do".into()), }; if let Err(err) = &result { diff --git a/src/bin/oura/watch.rs b/src/bin/oura/watch.rs new file mode 100644 index 00000000..6685ddc8 --- /dev/null +++ b/src/bin/oura/watch.rs @@ -0,0 +1,171 @@ +use std::{str::FromStr, sync::Arc}; + +use clap::ArgMatches; +use oura::{ + mapper::Config as MapperConfig, + pipelining::{SinkProvider, SourceProvider}, + sources::{AddressArg, BearerKind, IntersectArg, MagicArg}, + utils::{ChainWellKnownInfo, Utils, WithUtils}, +}; + +use serde::Deserialize; + +use oura::sources::n2c::Config as N2CConfig; +use oura::sources::n2n::Config as N2NConfig; + +use crate::Error; + +#[derive(Clone, Debug, Deserialize)] +pub enum PeerMode { + AsNode, + AsClient, +} + +impl FromStr for PeerMode { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_ref() { + "node" => Ok(PeerMode::AsNode), + "client" => Ok(PeerMode::AsClient), + _ => Err("can't parse peer mode (valid values: client|node)"), + } + } +} + +enum WatchSource { + N2C(N2CConfig), + N2N(N2NConfig), +} + +pub fn run(args: &ArgMatches) -> Result<(), Error> { + env_logger::builder() + .filter_level(log::LevelFilter::Error) + .init(); + + let socket = args.value_of_t("socket")?; + + let bearer = match args.is_present("bearer") { + true => args.value_of_t("bearer")?, + #[cfg(target_family = "unix")] + false => BearerKind::Unix, + #[cfg(target_family = "windows")] + false => BearerKind::Tcp, + }; + + let magic = match args.is_present("magic") { + true => args.value_of_t("magic")?, + false => MagicArg::default(), + }; + + let intersect = match args.is_present("since") { + true => Some(IntersectArg::Point(args.value_of_t("since")?)), + false => None, + }; + + let mode = match (args.is_present("mode"), &bearer) { + (true, _) => args + .value_of_t("mode") + .expect("invalid value for 'mode' arg"), + (false, BearerKind::Tcp) => PeerMode::AsNode, + #[cfg(target_family = "unix")] + (false, BearerKind::Unix) => PeerMode::AsClient, + }; + + let throttle = match args.is_present("throttle") { + true => Some(args.value_of_t("throttle")?), + false => None, + }; + + let wrap = args.is_present("wrap"); + + let mapper = MapperConfig { + include_block_end_events: true, + ..Default::default() + }; + + let well_known = ChainWellKnownInfo::try_from_magic(*magic)?; + + let utils = Arc::new(Utils::new(well_known)); + + #[allow(deprecated)] + let source_setup = match mode { + PeerMode::AsNode => WatchSource::N2N(N2NConfig { + address: AddressArg(bearer, socket), + magic: Some(magic), + well_known: None, + min_depth: 0, + mapper, + since: None, + intersect, + retry_policy: None, + finalize: None, + }), + PeerMode::AsClient => WatchSource::N2C(N2CConfig { + address: AddressArg(bearer, socket), + magic: Some(magic), + well_known: None, + min_depth: 0, + mapper, + since: None, + intersect, + retry_policy: None, + finalize: None, + }), + }; + + let sink_setup = oura::sinks::terminal::Config { + throttle_min_span_millis: throttle, + wrap: Some(wrap), + }; + + let (source_handle, source_output) = match source_setup { + WatchSource::N2C(c) => WithUtils::new(c, utils.clone()).bootstrap()?, + WatchSource::N2N(c) => WithUtils::new(c, utils.clone()).bootstrap()?, + }; + + let sink_handle = WithUtils::new(sink_setup, utils).bootstrap(source_output)?; + + sink_handle.join().map_err(|_| "error in sink thread")?; + source_handle.join().map_err(|_| "error in source thread")?; + + Ok(()) +} + +/// Creates the clap definition for this sub-command +pub(crate) fn command_definition<'a>() -> clap::Command<'a> { + clap::Command::new("watch") + .arg(clap::Arg::new("socket").required(true)) + .arg( + clap::Arg::new("bearer") + .long("bearer") + .takes_value(true) + .possible_values(["tcp", "unix"]), + ) + .arg(clap::Arg::new("magic").long("magic").takes_value(true)) + .arg( + clap::Arg::new("since") + .long("since") + .takes_value(true) + .help("point in the chain to start reading from, expects format `slot,hex-hash`"), + ) + .arg( + clap::Arg::new("throttle") + .long("throttle") + .takes_value(true) + .help("milliseconds to wait between output lines (for easier reading)"), + ) + .arg( + clap::Arg::new("wrap") + .long("wrap") + .short('w') + .takes_value(false) + .help("long text output should break and continue in the following line"), + ) + .arg( + clap::Arg::new("mode") + .long("mode") + .takes_value(true) + .possible_values(["node", "client"]), + ) +} diff --git a/src/filters/deno/mod.rs b/src/filters/deno/mod.rs deleted file mode 100644 index 72c0e661..00000000 --- a/src/filters/deno/mod.rs +++ /dev/null @@ -1,216 +0,0 @@ -//! A mapper with custom logic from using the Deno runtime - -use deno_core::{op, Extension, ModuleSpecifier, OpState}; -use deno_runtime::permissions::PermissionsContainer; -use deno_runtime::worker::{MainWorker as DenoWorker, WorkerOptions}; -use deno_runtime::BootstrapOptions; -use gasket::framework::*; -use pallas::network::miniprotocols::Point; -use serde::Deserialize; -use std::path::PathBuf; -use tracing::trace; - -use crate::framework::*; - -//pub struct WrappedRuntime(DenoWorker); -//unsafe impl Send for WrappedRuntime {} - -pub type WrappedRuntime = DenoWorker; - -#[op] -fn op_pop_record(state: &mut OpState) -> Result { - let r: Record = state.take(); - let j = serde_json::Value::from(r); - - Ok(j) -} - -#[op] -fn op_put_record( - state: &mut OpState, - value: serde_json::Value, -) -> Result<(), deno_core::error::AnyError> { - match value { - serde_json::Value::Null => (), - _ => state.put(value), - }; - - Ok(()) -} - -async fn setup_deno(main_module: &PathBuf) -> DenoWorker { - let ext = Extension::builder("oura") - .ops(vec![op_pop_record::decl(), op_put_record::decl()]) - .build(); - - let empty_module = deno_core::ModuleSpecifier::parse("data:text/javascript;base64,").unwrap(); - - let mut deno = DenoWorker::bootstrap_from_options( - empty_module, - PermissionsContainer::allow_all(), - WorkerOptions { - extensions: vec![ext], - bootstrap: BootstrapOptions { - ..Default::default() - }, - ..Default::default() - }, - ); - - let code = std::fs::read_to_string(main_module).unwrap(); - - deno.js_runtime - .load_side_module(&ModuleSpecifier::parse("oura:mapper").unwrap(), Some(code)) - .await - .unwrap(); - - let res = deno.execute_script("[oura:runtime.js]", include_str!("./runtime.js")); - deno.run_event_loop(false).await.unwrap(); - res.unwrap(); - - deno -} - -pub struct Worker { - runtime: WrappedRuntime, -} - -const SYNC_CALL_SNIPPET: &'static str = r#"Deno[Deno.internal].core.ops.op_put_record(mapEvent(Deno[Deno.internal].core.ops.op_pop_record()));"#; -const ASYNC_CALL_SNIPPET: &'static str = r#"mapEvent(Deno[Deno.internal].core.ops.op_pop_record()).then(x => Deno[Deno.internal].core.ops.op_put_record(x));"#; - -impl Worker { - async fn map_record( - &mut self, - script: &str, - record: Record, - ) -> Result, String> { - let deno = &mut self.runtime; - - deno.js_runtime.op_state().borrow_mut().put(record); - - let res = deno.execute_script("", script); - - deno.run_event_loop(false).await.unwrap(); - - res.unwrap(); - - let out = deno.js_runtime.op_state().borrow_mut().try_take(); - trace!(?out, "deno mapping finished"); - - Ok(out) - } -} - -#[async_trait::async_trait(?Send)] -impl gasket::framework::Worker for Worker { - async fn bootstrap(stage: &Stage) -> Result { - let runtime = setup_deno(&stage.main_module).await; - - Ok(Self { runtime }) - } - - async fn schedule( - &mut self, - stage: &mut Stage, - ) -> Result, WorkerError> { - let msg = stage.input.recv().await.or_panic()?; - - Ok(WorkSchedule::Unit(msg.payload)) - } - - async fn execute(&mut self, unit: &ChainEvent, stage: &mut Stage) -> Result<(), WorkerError> { - match unit { - ChainEvent::Apply(p, r) => { - let mapped = self - .map_record(stage.call_snippet, r.clone()) - .await - .unwrap(); - - stage.fanout(p.clone(), mapped).await.or_panic()? - } - ChainEvent::Undo(..) => todo!(), - ChainEvent::Reset(p) => { - stage - .output - .send(ChainEvent::reset(p.clone())) - .await - .or_panic()?; - } - }; - - Ok(()) - } -} - -#[derive(Stage)] -#[stage(name = "filter", unit = "ChainEvent", worker = "Worker")] -pub struct Stage { - main_module: PathBuf, - call_snippet: &'static str, - - pub input: MapperInputPort, - pub output: MapperOutputPort, - - #[metric] - ops_count: gasket::metrics::Counter, -} - -impl Stage { - async fn fanout( - &mut self, - point: Point, - mapped: Option, - ) -> Result<(), gasket::error::Error> { - if let Some(mapped) = mapped { - self.ops_count.inc(1); - - match mapped { - serde_json::Value::Array(items) => { - for item in items { - self.output - .send( - ChainEvent::Apply(point.clone(), Record::GenericJson(item)).into(), - ) - .await?; - } - } - _ => { - self.output - .send(ChainEvent::Apply(point, Record::GenericJson(mapped)).into()) - .await?; - } - } - } - - Ok(()) - } -} - -#[derive(Deserialize)] -pub struct Config { - main_module: String, - use_async: bool, -} - -impl Config { - pub fn bootstrapper(self, _ctx: &Context) -> Result { - // let main_module = - // deno_core::resolve_path(&self.main_module, - // &ctx.current_dir).map_err(Error::config)?; - - let stage = Stage { - //main_module, - main_module: PathBuf::from(self.main_module), - call_snippet: if self.use_async { - ASYNC_CALL_SNIPPET - } else { - SYNC_CALL_SNIPPET - }, - input: Default::default(), - output: Default::default(), - ops_count: Default::default(), - }; - - Ok(stage) - } -} diff --git a/src/filters/deno/runtime.js b/src/filters/deno/runtime.js deleted file mode 100644 index f3723d74..00000000 --- a/src/filters/deno/runtime.js +++ /dev/null @@ -1,3 +0,0 @@ -import("oura:mapper").then(({ mapEvent }) => { - globalThis["mapEvent"] = mapEvent; -}); diff --git a/src/filters/legacy_v1/fingerprint.rs b/src/filters/fingerprint.rs similarity index 76% rename from src/filters/legacy_v1/fingerprint.rs rename to src/filters/fingerprint.rs index d226c483..2e805f15 100644 --- a/src/filters/legacy_v1/fingerprint.rs +++ b/src/filters/fingerprint.rs @@ -9,12 +9,12 @@ use log::{debug, warn}; use serde::Deserialize; use crate::{ - framework::{new_inter_stage_channel, FilterProvider, PartialBootstrapResult, StageReceiver}, model::{ CIP15AssetRecord, CIP25AssetRecord, Event, EventData, MetadataRecord, MintRecord, NativeWitnessRecord, OutputAssetRecord, PlutusDatumRecord, PlutusRedeemerRecord, PlutusWitnessRecord, VKeyWitnessRecord, }, + pipelining::{new_inter_stage_channel, FilterProvider, PartialBootstrapResult, StageReceiver}, Error, }; @@ -214,6 +214,66 @@ fn build_fingerprint(event: &Event, seed: u32) -> Result { .with_prefix("move") .append_optional(&event.context.tx_hash)? .append_optional_to_string(&event.context.certificate_idx)?, + EventData::RegCert { .. } => b + .with_slot(&event.context.slot) + .with_prefix("regc") + .append_optional(&event.context.tx_hash)? + .append_optional_to_string(&event.context.certificate_idx)?, + EventData::UnRegCert { .. } => b + .with_slot(&event.context.slot) + .with_prefix("unrc") + .append_optional(&event.context.tx_hash)? + .append_optional_to_string(&event.context.certificate_idx)?, + EventData::VoteDeleg { .. } => b + .with_slot(&event.context.slot) + .with_prefix("vode") + .append_optional(&event.context.tx_hash)? + .append_optional_to_string(&event.context.certificate_idx)?, + EventData::StakeVoteDeleg { .. } => b + .with_slot(&event.context.slot) + .with_prefix("stvo") + .append_optional(&event.context.tx_hash)? + .append_optional_to_string(&event.context.certificate_idx)?, + EventData::StakeRegDeleg { .. } => b + .with_slot(&event.context.slot) + .with_prefix("stre") + .append_optional(&event.context.tx_hash)? + .append_optional_to_string(&event.context.certificate_idx)?, + EventData::VoteRegDeleg { .. } => b + .with_slot(&event.context.slot) + .with_prefix("vore") + .append_optional(&event.context.tx_hash)? + .append_optional_to_string(&event.context.certificate_idx)?, + EventData::StakeVoteRegDeleg { .. } => b + .with_slot(&event.context.slot) + .with_prefix("stvr") + .append_optional(&event.context.tx_hash)? + .append_optional_to_string(&event.context.certificate_idx)?, + EventData::AuthCommitteeHot { .. } => b + .with_slot(&event.context.slot) + .with_prefix("auth") + .append_optional(&event.context.tx_hash)? + .append_optional_to_string(&event.context.certificate_idx)?, + EventData::ResignCommitteeCold { .. } => b + .with_slot(&event.context.slot) + .with_prefix("resi") + .append_optional(&event.context.tx_hash)? + .append_optional_to_string(&event.context.certificate_idx)?, + EventData::RegDRepCert { .. } => b + .with_slot(&event.context.slot) + .with_prefix("regd") + .append_optional(&event.context.tx_hash)? + .append_optional_to_string(&event.context.certificate_idx)?, + EventData::UnRegDRepCert { .. } => b + .with_slot(&event.context.slot) + .with_prefix("unrd") + .append_optional(&event.context.tx_hash)? + .append_optional_to_string(&event.context.certificate_idx)?, + EventData::UpdateDRepCert { .. } => b + .with_slot(&event.context.slot) + .with_prefix("updd") + .append_optional(&event.context.tx_hash)? + .append_optional_to_string(&event.context.certificate_idx)?, EventData::RollBack { block_slot, block_hash, diff --git a/src/filters/json.rs b/src/filters/json.rs deleted file mode 100644 index e2958311..00000000 --- a/src/filters/json.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! A noop filter used as example and placeholder for other filters - -use gasket::framework::*; -use serde::Deserialize; - -use crate::framework::*; - -#[derive(Default, Stage)] -#[stage(name = "filter", unit = "ChainEvent", worker = "Worker")] -pub struct Stage { - pub input: FilterInputPort, - pub output: FilterOutputPort, - - #[metric] - ops_count: gasket::metrics::Counter, -} - -#[derive(Default)] -pub struct Worker; - -impl From<&Stage> for Worker { - fn from(_: &Stage) -> Self { - Worker::default() - } -} - -gasket::impl_mapper!(|_worker: Worker, stage: Stage, unit: ChainEvent| => { - let out = unit.clone(); - stage.ops_count.inc(1); - out -}); - -#[derive(Default, Deserialize)] -pub struct Config {} - -impl Config { - pub fn bootstrapper(self, _ctx: &Context) -> Result { - Ok(Stage::default()) - } -} diff --git a/src/filters/legacy_v1/cip15.rs b/src/filters/legacy_v1/cip15.rs deleted file mode 100644 index e5ada478..00000000 --- a/src/filters/legacy_v1/cip15.rs +++ /dev/null @@ -1,44 +0,0 @@ -use gasket::framework::WorkerError; -use serde_json::Value as JsonValue; - -use pallas::ledger::primitives::alonzo::Metadatum; - -use super::EventWriter; -use crate::framework::legacy_v1::*; - -fn extract_json_property<'a>(json: &'a JsonValue, key: &str) -> Option<&'a JsonValue> { - json.as_object().and_then(|x| x.get(key)) -} - -fn extract_json_string_property(json: &JsonValue, key: &str) -> Option { - extract_json_property(json, key) - .and_then(|x| x.as_str()) - .map(|x| x.to_owned()) -} - -fn extract_json_int_property(json: &JsonValue, key: &str) -> Option { - extract_json_property(json, key).and_then(|x| x.as_i64()) -} - -impl EventWriter<'_> { - fn to_cip15_asset_record(&self, content: &Metadatum) -> CIP15AssetRecord { - let raw_json = self.to_metadatum_json(content); - - CIP15AssetRecord { - voting_key: extract_json_string_property(&raw_json, "1").unwrap_or_default(), - stake_pub: extract_json_string_property(&raw_json, "2").unwrap_or_default(), - reward_address: extract_json_string_property(&raw_json, "3").unwrap_or_default(), - nonce: extract_json_int_property(&raw_json, "4").unwrap_or_default(), - raw_json, - } - } - - pub(crate) fn crawl_metadata_label_61284( - &mut self, - content: &Metadatum, - ) -> Result<(), WorkerError> { - self.append_from(self.to_cip15_asset_record(content))?; - - Ok(()) - } -} diff --git a/src/filters/legacy_v1/crawl.rs b/src/filters/legacy_v1/crawl.rs deleted file mode 100644 index ba70200f..00000000 --- a/src/filters/legacy_v1/crawl.rs +++ /dev/null @@ -1,206 +0,0 @@ -use gasket::framework::{AsWorkError, WorkerError}; -use pallas::ledger::primitives::babbage::MintedDatumOption; -use pallas::ledger::traverse::{MultiEraBlock, MultiEraInput, MultiEraOutput, MultiEraTx}; -use pallas::network::miniprotocols::Point; - -use crate::framework::legacy_v1::*; -use crate::framework::Error as OuraError; - -use super::EventWriter; - -impl From for Era { - fn from(other: pallas::ledger::traverse::Era) -> Self { - match other { - pallas::ledger::traverse::Era::Byron => Era::Byron, - pallas::ledger::traverse::Era::Shelley => Era::Shelley, - pallas::ledger::traverse::Era::Allegra => Era::Allegra, - pallas::ledger::traverse::Era::Mary => Era::Mary, - pallas::ledger::traverse::Era::Alonzo => Era::Alonzo, - pallas::ledger::traverse::Era::Babbage => Era::Babbage, - _ => Era::Unknown, - } - } -} - -impl EventWriter<'_> { - fn crawl_collateral(&mut self, collateral: &MultiEraInput) -> Result<(), WorkerError> { - self.append(self.to_collateral_event(collateral)) - - // TODO: should we have a collateral idx in context? - // more complex event goes here (eg: ???) - } - - fn crawl_metadata(&mut self, tx: &MultiEraTx) -> Result<(), WorkerError> { - let metadata = tx.metadata(); - let metadata = metadata.collect::>(); - - for (label, content) in metadata.iter() { - let record = self.to_metadata_record(label, content); - self.append_from(record)?; - - match label { - 721u64 => self.crawl_metadata_label_721(content)?, - 61284u64 => self.crawl_metadata_label_61284(content)?, - _ => (), - } - } - - Ok(()) - } - - fn crawl_transaction_output(&mut self, output: &MultiEraOutput) -> Result<(), WorkerError> { - let record = self.to_transaction_output_record(output); - self.append(record.into()).or_panic()?; - - let address = output.address().or_panic()?; - - let mut child = self.child_writer(EventContext { - output_address: Some(address.to_string()), - ..EventContext::default() - }); - - for asset in output.assets() { - child.append_from(OutputAssetRecord::from(&asset))?; - } - - if let Some(MintedDatumOption::Data(datum)) = &output.datum() { - child.append_from(PlutusDatumRecord::from(&datum.0))?; - } - - Ok(()) - } - - fn crawl_witnesses(&mut self, tx: &MultiEraTx) -> Result<(), WorkerError> { - for script in tx.native_scripts() { - self.append_from(self.to_native_witness_record(script))?; - } - - for script in tx.plutus_v1_scripts() { - self.append_from(self.to_plutus_v1_witness_record(script))?; - } - - for script in tx.plutus_v2_scripts() { - self.append_from(self.to_plutus_v2_witness_record(script))?; - } - - for redeemer in tx.redeemers() { - self.append_from(self.to_plutus_redeemer_record(redeemer))?; - } - - for datum in tx.plutus_data() { - self.append_from(PlutusDatumRecord::from(datum))?; - } - - Ok(()) - } - - fn crawl_transaction(&mut self, tx: &MultiEraTx) -> Result<(), WorkerError> { - let record = self.to_transaction_record(tx); - self.append_from(record.clone())?; - - // crawl inputs - for (idx, input) in tx.inputs().iter().enumerate() { - let mut child = self.child_writer(EventContext { - input_idx: Some(idx), - ..EventContext::default() - }); - - child.append_from(TxInputRecord::from(input))?; - } - - for (idx, output) in tx.outputs().iter().enumerate() { - let mut child = self.child_writer(EventContext { - output_idx: Some(idx), - ..EventContext::default() - }); - - child.crawl_transaction_output(output)?; - } - - //crawl certs - for (idx, cert) in tx.certs().iter().enumerate() { - if let Some(evt) = self.to_certificate_event(cert) { - let mut child = self.child_writer(EventContext { - certificate_idx: Some(idx), - ..EventContext::default() - }); - - child.append(evt)?; - } - } - - for collateral in tx.collateral().iter() { - // TODO: collateral context? - self.crawl_collateral(collateral)?; - } - - // crawl mints - for asset in tx.mints() { - self.append_from(self.to_mint_record(&asset))?; - } - - self.crawl_metadata(tx)?; - - // crawl aux native scripts - for script in tx.aux_native_scripts() { - self.append(self.to_aux_native_script_event(script))?; - } - - // crawl aux plutus v1 scripts - for script in tx.aux_plutus_v1_scripts() { - self.append(self.to_aux_plutus_script_event(script))?; - } - - self.crawl_witnesses(tx)?; - - if self.config.include_transaction_end_events { - self.append(EventData::TransactionEnd(record))?; - } - - Ok(()) - } - - fn crawl_block(&mut self, block: &MultiEraBlock, cbor: &[u8]) -> Result<(), WorkerError> { - let record = self.to_block_record(block, cbor); - self.append(EventData::Block(record.clone()))?; - - for (idx, tx) in block.txs().iter().enumerate() { - let mut child = self.child_writer(EventContext { - tx_idx: Some(idx), - tx_hash: Some(tx.hash().to_string()), - ..EventContext::default() - }); - - child.crawl_transaction(tx)?; - } - - if self.config.include_block_end_events { - self.append(EventData::BlockEnd(record))?; - } - - Ok(()) - } - - /// Mapper entry-point for raw cbor blocks - pub fn crawl_cbor(&mut self, cbor: &[u8]) -> Result<(), WorkerError> { - let block = pallas::ledger::traverse::MultiEraBlock::decode(cbor) - .map_err(OuraError::parse) - .or_panic()?; - - let hash = block.hash(); - - let mut child = self.child_writer(EventContext { - block_hash: Some(hex::encode(hash)), - block_number: Some(block.number()), - slot: Some(block.slot()), - timestamp: Some(block.wallclock(self.genesis)), - ..EventContext::default() - }); - - child.crawl_block(&block, cbor) - } - - pub fn crawl_rollback(&mut self, point: Point) -> Result<(), WorkerError> { - self.append(point.into()) - } -} diff --git a/src/filters/legacy_v1/map.rs b/src/filters/legacy_v1/map.rs deleted file mode 100644 index 93906acf..00000000 --- a/src/filters/legacy_v1/map.rs +++ /dev/null @@ -1,513 +0,0 @@ -use gasket::framework::AsWorkError; -use lazy_static::__Deref; -use serde_json::{json, Value as JsonValue}; -use std::collections::HashMap; - -use pallas::ledger::primitives::babbage::{MintedDatumOption, NetworkId}; -use pallas::ledger::primitives::{ - alonzo::{ - self as alonzo, Certificate, InstantaneousRewardSource, InstantaneousRewardTarget, - Metadatum, MetadatumLabel, Relay, - }, - babbage, ToCanonicalJson, -}; -use pallas::ledger::traverse::{ - ComputeHash, MultiEraAsset, MultiEraBlock, MultiEraCert, MultiEraInput, MultiEraOutput, - MultiEraTx, OriginalHash, -}; -use pallas::network::miniprotocols::Point; -use pallas::{codec::utils::KeepRaw, crypto::hash::Hash}; - -use crate::framework::legacy_v1::*; - -use super::EventWriter; - -pub trait ToHex { - fn to_hex(&self) -> String; -} - -impl ToHex for Vec { - fn to_hex(&self) -> String { - hex::encode(self) - } -} - -impl ToHex for &[u8] { - fn to_hex(&self) -> String { - hex::encode(self) - } -} - -impl ToHex for Hash { - fn to_hex(&self) -> String { - hex::encode(self) - } -} - -impl From<&alonzo::StakeCredential> for StakeCredential { - fn from(other: &alonzo::StakeCredential) -> Self { - match other { - alonzo::StakeCredential::AddrKeyhash(x) => StakeCredential::AddrKeyhash(x.to_hex()), - alonzo::StakeCredential::Scripthash(x) => StakeCredential::Scripthash(x.to_hex()), - } - } -} - -fn ip_string_from_bytes(bytes: &[u8]) -> String { - format!("{}.{}.{}.{}", bytes[0], bytes[1], bytes[2], bytes[3]) -} - -impl From for EventData { - fn from(value: Point) -> Self { - match value { - Point::Origin => EventData::RollBack { - block_slot: 0, - block_hash: "".to_string(), - }, - Point::Specific(slot, hash) => EventData::RollBack { - block_slot: slot, - block_hash: hex::encode(hash), - }, - } - } -} - -impl From<&MultiEraInput<'_>> for TxInputRecord { - fn from(value: &MultiEraInput) -> Self { - Self { - tx_id: value.hash().to_string(), - index: value.index(), - } - } -} - -impl From<&MultiEraAsset<'_>> for OutputAssetRecord { - fn from(value: &MultiEraAsset<'_>) -> Self { - Self { - policy: value.policy().map(ToString::to_string).unwrap_or_default(), - asset: value.name().map(|x| x.to_hex()).unwrap_or_default(), - asset_ascii: value.to_ascii_name(), - amount: value.coin() as u64, - } - } -} - -impl From<&KeepRaw<'_, alonzo::PlutusData>> for PlutusDatumRecord { - fn from(value: &KeepRaw<'_, alonzo::PlutusData>) -> Self { - Self { - datum_hash: value.original_hash().to_hex(), - plutus_data: value.to_json(), - } - } -} - -fn relay_to_string(relay: &Relay) -> String { - match relay { - Relay::SingleHostAddr(port, ipv4, ipv6) => { - let ip = match (ipv6, ipv4) { - (None, None) => "".to_string(), - (_, Some(x)) => ip_string_from_bytes(x.as_ref()), - (Some(x), _) => ip_string_from_bytes(x.as_ref()), - }; - - match port { - Some(port) => format!("{ip}:{port}"), - None => ip, - } - } - Relay::SingleHostName(port, host) => match port { - Some(port) => format!("{host}:{port}"), - None => host.clone(), - }, - Relay::MultiHostName(host) => host.clone(), - } -} - -fn metadatum_to_string_key(datum: &Metadatum) -> String { - match datum { - Metadatum::Int(x) => x.to_string(), - Metadatum::Bytes(x) => hex::encode(x.as_slice()), - Metadatum::Text(x) => x.to_owned(), - x => { - log::warn!("unexpected metadatum type for label: {:?}", x); - Default::default() - } - } -} - -impl EventWriter<'_> { - pub fn to_transaction_output_record(&self, output: &MultiEraOutput) -> TxOutputRecord { - let address = output.address().or_panic(); - - TxOutputRecord { - address: address.map(|x| x.to_string()).unwrap_or_default(), - amount: output.lovelace_amount(), - assets: output - .non_ada_assets() - .iter() - .map(|x| OutputAssetRecord::from(x)) - .collect::>() - .into(), - datum_hash: match &output.datum() { - Some(MintedDatumOption::Hash(x)) => Some(x.to_string()), - Some(MintedDatumOption::Data(x)) => Some(x.original_hash().to_hex()), - None => None, - }, - inline_datum: match &output.datum() { - Some(MintedDatumOption::Data(x)) => Some(PlutusDatumRecord::from(x.deref())), - _ => None, - }, - } - } - pub fn to_withdrawal_record(&self, withdrawal: (&[u8], u64)) -> WithdrawalRecord { - WithdrawalRecord { - reward_account: { - let hex = withdrawal.0.to_hex(); - hex.strip_prefix("e1").map(|x| x.to_string()).unwrap_or(hex) - }, - coin: withdrawal.1, - } - } - - pub fn to_transaction_record(&self, tx: &MultiEraTx) -> TransactionRecord { - let mut record = TransactionRecord { - hash: tx.hash().to_string(), - size: tx.size() as u32, - fee: tx.fee().unwrap_or_default(), - ttl: tx.ttl(), - validity_interval_start: tx.validity_start(), - network_id: tx.network_id().map(|x| match x { - NetworkId::One => 1, - NetworkId::Two => 2, - }), - ..Default::default() - }; - - let outputs: Vec<_> = tx - .outputs() - .iter() - .map(|x| self.to_transaction_output_record(x)) - .collect(); - - record.output_count = outputs.len(); - record.total_output = outputs.iter().map(|o| o.amount).sum(); - - let inputs: Vec<_> = tx.inputs().iter().map(|x| TxInputRecord::from(x)).collect(); - - record.input_count = inputs.len(); - - let mints: Vec<_> = tx.mints().iter().map(|x| self.to_mint_record(x)).collect(); - - record.mint_count = mints.len(); - - let collateral_inputs: Vec<_> = tx - .collateral() - .iter() - .map(|x| TxInputRecord::from(x)) - .collect(); - - record.collateral_input_count = collateral_inputs.len(); - - let collateral_return = tx.collateral_return(); - - record.has_collateral_output = collateral_return.is_some(); - - // TODO - // TransactionBodyComponent::ScriptDataHash(_) - // TransactionBodyComponent::RequiredSigners(_) - // TransactionBodyComponent::AuxiliaryDataHash(_) - - if self.config.include_transaction_details { - record.outputs = Some(outputs); - record.inputs = Some(inputs); - record.mint = Some(mints); - - record.collateral_inputs = Some(collateral_inputs); - - record.collateral_output = - collateral_return.map(|x| self.to_transaction_output_record(&x)); - - record.metadata = tx - .metadata() - .collect::>() - .iter() - .map(|(l, v)| self.to_metadata_record(l, v)) - .collect::>() - .into(); - - record.vkey_witnesses = tx - .vkey_witnesses() - .iter() - .map(|x| self.to_vkey_witness_record(x)) - .collect::>() - .into(); - - record.native_witnesses = tx - .native_scripts() - .iter() - .map(|x| self.to_native_witness_record(x)) - .collect::>() - .into(); - - let v1_scripts = tx - .plutus_v1_scripts() - .iter() - .map(|x| self.to_plutus_v1_witness_record(x)) - .collect::>(); - - let v2_scripts = tx - .plutus_v2_scripts() - .iter() - .map(|x| self.to_plutus_v2_witness_record(x)) - .collect::>(); - - record.plutus_witnesses = Some([v1_scripts, v2_scripts].concat()); - - record.plutus_redeemers = tx - .redeemers() - .iter() - .map(|x| self.to_plutus_redeemer_record(x)) - .collect::>() - .into(); - - record.plutus_data = tx - .plutus_data() - .iter() - .map(|x| PlutusDatumRecord::from(x)) - .collect::>() - .into(); - - record.withdrawals = tx - .withdrawals() - .collect::>() - .iter() - .map(|x| self.to_withdrawal_record(*x)) - .collect::>() - .into(); - } - - record - } - - pub fn to_block_record(&self, source: &MultiEraBlock, cbor: &[u8]) -> BlockRecord { - let header = source.header(); - let (epoch, sub_slot) = source.epoch(self.genesis); - - let mut record = BlockRecord { - era: source.era().into(), - body_size: source.body_size().unwrap_or_default(), - issuer_vkey: header.issuer_vkey().map(hex::encode).unwrap_or_default(), - vrf_vkey: header.vrf_vkey().map(hex::encode).unwrap_or_default(), - tx_count: source.tx_count(), - hash: source.hash().to_string(), - number: source.number(), - slot: source.slot(), - epoch: Some(epoch), - epoch_slot: Some(sub_slot), - previous_hash: header - .previous_hash() - .map(|x| x.to_string()) - .unwrap_or_default(), - cbor_hex: match self.config.include_block_cbor { - true => Some(hex::encode(cbor)), - false => None, - }, - transactions: None, - }; - - if self.config.include_block_details { - let txs = source - .txs() - .iter() - .map(|x| self.to_transaction_record(x)) - .collect(); - - record.transactions = Some(txs); - } - - record - } - - pub fn to_mint_record(&self, asset: &MultiEraAsset) -> MintRecord { - MintRecord { - policy: asset.policy().map(|x| x.to_string()).unwrap_or_default(), - asset: asset.name().map(hex::encode).unwrap_or_default(), - quantity: asset.coin(), - } - } - - pub fn to_metadatum_json_map_entry( - &self, - pair: (&Metadatum, &Metadatum), - ) -> (String, JsonValue) { - let key = metadatum_to_string_key(pair.0); - let value = self.to_metadatum_json(pair.1); - (key, value) - } - - pub fn to_metadatum_json(&self, source: &Metadatum) -> JsonValue { - match source { - Metadatum::Int(x) => json!(i128::from(*x)), - Metadatum::Bytes(x) => json!(hex::encode(x.as_slice())), - Metadatum::Text(x) => json!(x), - Metadatum::Array(x) => { - let items: Vec<_> = x.iter().map(|x| self.to_metadatum_json(x)).collect(); - - json!(items) - } - Metadatum::Map(x) => { - let map: HashMap<_, _> = x - .iter() - .map(|(key, value)| self.to_metadatum_json_map_entry((key, value))) - .collect(); - - json!(map) - } - } - } - - pub fn to_metadata_record(&self, label: &MetadatumLabel, value: &Metadatum) -> MetadataRecord { - MetadataRecord { - label: label.to_string(), - content: match value { - Metadatum::Int(x) => MetadatumRendition::IntScalar(i128::from(*x)), - Metadatum::Bytes(x) => MetadatumRendition::BytesHex(hex::encode(x.as_slice())), - Metadatum::Text(x) => MetadatumRendition::TextScalar(x.clone()), - Metadatum::Array(_) => MetadatumRendition::ArrayJson(self.to_metadatum_json(value)), - Metadatum::Map(_) => MetadatumRendition::MapJson(self.to_metadatum_json(value)), - }, - } - } - - pub fn to_aux_native_script_event(&self, script: &alonzo::NativeScript) -> EventData { - EventData::NativeScript { - policy_id: script.compute_hash().to_hex(), - script: script.to_json(), - } - } - - pub fn to_aux_plutus_script_event(&self, script: &alonzo::PlutusScript) -> EventData { - EventData::PlutusScript { - hash: script.compute_hash().to_hex(), - data: script.0.to_hex(), - } - } - - pub fn to_plutus_redeemer_record(&self, redeemer: &alonzo::Redeemer) -> PlutusRedeemerRecord { - PlutusRedeemerRecord { - purpose: match redeemer.tag { - alonzo::RedeemerTag::Spend => "spend".to_string(), - alonzo::RedeemerTag::Mint => "mint".to_string(), - alonzo::RedeemerTag::Cert => "cert".to_string(), - alonzo::RedeemerTag::Reward => "reward".to_string(), - }, - ex_units_mem: redeemer.ex_units.mem, - ex_units_steps: redeemer.ex_units.steps, - input_idx: redeemer.index, - plutus_data: redeemer.data.to_json(), - } - } - - pub fn to_plutus_v1_witness_record( - &self, - script: &alonzo::PlutusScript, - ) -> PlutusWitnessRecord { - PlutusWitnessRecord { - script_hash: script.compute_hash().to_hex(), - script_hex: script.as_ref().to_hex(), - } - } - - pub fn to_plutus_v2_witness_record( - &self, - script: &babbage::PlutusV2Script, - ) -> PlutusWitnessRecord { - PlutusWitnessRecord { - script_hash: script.compute_hash().to_hex(), - script_hex: script.as_ref().to_hex(), - } - } - - pub fn to_native_witness_record(&self, script: &alonzo::NativeScript) -> NativeWitnessRecord { - NativeWitnessRecord { - policy_id: script.compute_hash().to_hex(), - script_json: script.to_json(), - } - } - - pub fn to_vkey_witness_record(&self, witness: &alonzo::VKeyWitness) -> VKeyWitnessRecord { - VKeyWitnessRecord { - vkey_hex: witness.vkey.to_hex(), - signature_hex: witness.signature.to_hex(), - } - } - - pub fn to_certificate_event(&self, cert: &MultiEraCert) -> Option { - let evt = match cert.as_alonzo()? { - Certificate::StakeRegistration(credential) => EventData::StakeRegistration { - credential: credential.into(), - }, - Certificate::StakeDeregistration(credential) => EventData::StakeDeregistration { - credential: credential.into(), - }, - Certificate::StakeDelegation(credential, pool) => EventData::StakeDelegation { - credential: credential.into(), - pool_hash: pool.to_hex(), - }, - Certificate::PoolRegistration { - operator, - vrf_keyhash, - pledge, - cost, - margin, - reward_account, - pool_owners, - relays, - pool_metadata, - } => EventData::PoolRegistration { - operator: operator.to_hex(), - vrf_keyhash: vrf_keyhash.to_hex(), - pledge: *pledge, - cost: *cost, - margin: (margin.numerator as f64 / margin.denominator as f64), - reward_account: reward_account.to_hex(), - pool_owners: pool_owners.iter().map(|p| p.to_hex()).collect(), - relays: relays.iter().map(relay_to_string).collect(), - pool_metadata: pool_metadata.as_ref().map(|m| m.url.clone()), - pool_metadata_hash: pool_metadata.as_ref().map(|m| m.hash.clone().to_hex()), - }, - Certificate::PoolRetirement(pool, epoch) => EventData::PoolRetirement { - pool: pool.to_hex(), - epoch: *epoch, - }, - Certificate::MoveInstantaneousRewardsCert(move_) => { - EventData::MoveInstantaneousRewardsCert { - from_reserves: matches!(move_.source, InstantaneousRewardSource::Reserves), - from_treasury: matches!(move_.source, InstantaneousRewardSource::Treasury), - to_stake_credentials: match &move_.target { - InstantaneousRewardTarget::StakeCredentials(creds) => { - let x = creds.iter().map(|(k, v)| (k.into(), *v)).collect(); - Some(x) - } - _ => None, - }, - to_other_pot: match move_.target { - InstantaneousRewardTarget::OtherAccountingPot(x) => Some(x), - _ => None, - }, - } - } - // TODO: not likely, leaving for later - Certificate::GenesisKeyDelegation(..) => EventData::GenesisKeyDelegation {}, - }; - - Some(evt) - } - - pub fn to_collateral_event(&self, collateral: &MultiEraInput) -> EventData { - EventData::Collateral { - tx_id: collateral.hash().to_string(), - index: collateral.index(), - } - } -} diff --git a/src/filters/legacy_v1/mod.rs b/src/filters/legacy_v1/mod.rs deleted file mode 100644 index 59942eb3..00000000 --- a/src/filters/legacy_v1/mod.rs +++ /dev/null @@ -1,105 +0,0 @@ -//! A mapper that maintains schema-compatibility with Oura v1 - -mod cip15; -mod cip25; -mod crawl; -mod map; -mod prelude; - -use gasket::framework::*; -use pallas::ledger::traverse::wellknown::GenesisValues; -use serde::Deserialize; - -use crate::framework::*; -pub use prelude::*; - -#[derive(Stage)] -#[stage(name = "filter", unit = "ChainEvent", worker = "Worker")] -pub struct Stage { - config: Config, - genesis: GenesisValues, - - pub input: MapperInputPort, - pub output: MapperOutputPort, - - #[metric] - ops_count: gasket::metrics::Counter, -} - -#[derive(Default)] -pub struct Worker; - -impl From<&Stage> for Worker { - fn from(_: &Stage) -> Self { - Worker::default() - } -} - -gasket::impl_splitter!(|_worker: Worker, stage: Stage, unit: ChainEvent| => { - let mut buffer = Vec::new(); - - match unit { - ChainEvent::Apply(point, Record::CborBlock(cbor)) => { - let mut writer = EventWriter::new( - point.clone(), - stage.output.clone(), - &stage.config, - &stage.genesis, - &mut buffer, - ); - - writer.crawl_cbor(&cbor)?; - } - ChainEvent::Reset(point) => { - let mut writer = EventWriter::new( - point.clone(), - stage.output.clone(), - &stage.config, - &stage.genesis, - &mut buffer, - ); - - writer.crawl_rollback(point.clone())?; - } - x => buffer.push(x.clone()), - }; - - stage.ops_count.inc(1); - - buffer -}); - -#[derive(Deserialize, Clone, Debug, Default)] -pub struct Config { - #[serde(default)] - pub include_block_end_events: bool, - - #[serde(default)] - pub include_transaction_details: bool, - - #[serde(default)] - pub include_transaction_end_events: bool, - - #[serde(default)] - pub include_block_details: bool, - - #[serde(default)] - pub include_block_cbor: bool, - - #[serde(default)] - pub include_byron_ebb: bool, -} - -impl Config { - pub fn bootstrapper(self, ctx: &Context) -> Result { - let stage = Stage { - config: self, - genesis: ctx.chain.clone(), - ops_count: Default::default(), - input: Default::default(), - output: Default::default(), - }; - - Ok(stage) - } -} diff --git a/src/filters/legacy_v1/prelude.rs b/src/filters/legacy_v1/prelude.rs deleted file mode 100644 index d55dc1a8..00000000 --- a/src/filters/legacy_v1/prelude.rs +++ /dev/null @@ -1,70 +0,0 @@ -use crate::framework::legacy_v1::*; -use crate::framework::*; - -use gasket::framework::WorkerError; -use merge::Merge; -use pallas::ledger::traverse::wellknown::GenesisValues; -use pallas::network::miniprotocols::Point; - -use super::Config; - -pub struct EventWriter<'a> { - context: EventContext, - point: Point, - output: MapperOutputPort, - pub(crate) config: &'a Config, - pub(crate) genesis: &'a GenesisValues, - buffer: &'a mut Vec, -} - -impl<'a> EventWriter<'a> { - pub fn new( - point: Point, - output: MapperOutputPort, - config: &'a Config, - genesis: &'a GenesisValues, - buffer: &'a mut Vec, - ) -> Self { - EventWriter { - context: EventContext::default(), - point, - output, - config, - genesis, - buffer, - } - } - - pub fn append(&mut self, data: EventData) -> Result<(), WorkerError> { - let evt = Event { - context: self.context.clone(), - data, - fingerprint: None, - }; - - let msg = ChainEvent::Apply(self.point.clone(), Record::OuraV1Event(evt)); - self.buffer.push(msg); - - Ok(()) - } - - pub fn append_from(&mut self, source: T) -> Result<(), WorkerError> - where - T: Into, - { - self.append(source.into()) - } - - pub fn child_writer(&mut self, mut extra_context: EventContext) -> EventWriter { - extra_context.merge(self.context.clone()); - - EventWriter { - context: extra_context, - point: self.point.clone(), - output: self.output.clone(), - config: self.config, - genesis: self.genesis, - buffer: self.buffer, - } - } -} diff --git a/src/filters/mod.rs b/src/filters/mod.rs index 237b357e..4b4229fe 100644 --- a/src/filters/mod.rs +++ b/src/filters/mod.rs @@ -1,102 +1,5 @@ -use gasket::{ - messaging::{RecvPort, SendPort}, - runtime::Tether, -}; -use serde::Deserialize; - -use crate::framework::*; - -pub mod deno; -pub mod dsl; -pub mod json; -pub mod legacy_v1; pub mod noop; -pub mod parse_cbor; -pub mod split_block; -pub mod wasm; - -pub enum Bootstrapper { - Noop(noop::Stage), - SplitBlock(split_block::Stage), - Dsl(dsl::Stage), - Json(json::Stage), - LegacyV1(legacy_v1::Stage), - Wasm(wasm::Stage), - Deno(deno::Stage), - ParseCbor(parse_cbor::Stage), -} - -impl StageBootstrapper for Bootstrapper { - fn connect_input(&mut self, adapter: InputAdapter) { - match self { - Bootstrapper::Noop(p) => p.input.connect(adapter), - Bootstrapper::SplitBlock(p) => p.input.connect(adapter), - Bootstrapper::Dsl(p) => p.input.connect(adapter), - Bootstrapper::Json(p) => p.input.connect(adapter), - Bootstrapper::LegacyV1(p) => p.input.connect(adapter), - Bootstrapper::Wasm(p) => p.input.connect(adapter), - Bootstrapper::Deno(p) => p.input.connect(adapter), - Bootstrapper::ParseCbor(p) => p.input.connect(adapter), - } - } - - fn connect_output(&mut self, adapter: OutputAdapter) { - match self { - Bootstrapper::Noop(p) => p.output.connect(adapter), - Bootstrapper::SplitBlock(p) => p.output.connect(adapter), - Bootstrapper::Dsl(p) => p.output.connect(adapter), - Bootstrapper::Json(p) => p.output.connect(adapter), - Bootstrapper::LegacyV1(p) => p.output.connect(adapter), - Bootstrapper::Wasm(p) => p.output.connect(adapter), - Bootstrapper::Deno(p) => p.output.connect(adapter), - Bootstrapper::ParseCbor(p) => p.output.connect(adapter), - } - } - - fn spawn(self, policy: gasket::runtime::Policy) -> Tether { - match self { - Bootstrapper::Noop(x) => gasket::runtime::spawn_stage(x, policy), - Bootstrapper::SplitBlock(x) => gasket::runtime::spawn_stage(x, policy), - Bootstrapper::Dsl(x) => gasket::runtime::spawn_stage(x, policy), - Bootstrapper::Json(x) => gasket::runtime::spawn_stage(x, policy), - Bootstrapper::LegacyV1(x) => gasket::runtime::spawn_stage(x, policy), - Bootstrapper::Wasm(x) => gasket::runtime::spawn_stage(x, policy), - Bootstrapper::Deno(x) => gasket::runtime::spawn_stage(x, policy), - Bootstrapper::ParseCbor(x) => gasket::runtime::spawn_stage(x, policy), - } - } -} - -#[derive(Deserialize)] -#[serde(tag = "type")] -pub enum Config { - Noop(noop::Config), - SplitBlock(split_block::Config), - Dsl(dsl::Config), - Json(json::Config), - LegacyV1(legacy_v1::Config), - Wasm(wasm::Config), - Deno(deno::Config), - ParseCbor(parse_cbor::Config), -} - -impl Config { - pub fn bootstrapper(self, ctx: &Context) -> Result { - match self { - Config::Noop(c) => Ok(Bootstrapper::Noop(c.bootstrapper(ctx)?)), - Config::SplitBlock(c) => Ok(Bootstrapper::SplitBlock(c.bootstrapper(ctx)?)), - Config::Dsl(c) => Ok(Bootstrapper::Dsl(c.bootstrapper(ctx)?)), - Config::Json(c) => Ok(Bootstrapper::Json(c.bootstrapper(ctx)?)), - Config::LegacyV1(c) => Ok(Bootstrapper::LegacyV1(c.bootstrapper(ctx)?)), - Config::Wasm(c) => Ok(Bootstrapper::Wasm(c.bootstrapper(ctx)?)), - Config::Deno(c) => Ok(Bootstrapper::Deno(c.bootstrapper(ctx)?)), - Config::ParseCbor(c) => Ok(Bootstrapper::ParseCbor(c.bootstrapper(ctx)?)), - } - } -} +pub mod selection; -impl Default for Config { - fn default() -> Self { - Config::LegacyV1(Default::default()) - } -} +#[cfg(feature = "fingerprint")] +pub mod fingerprint; diff --git a/src/filters/noop.rs b/src/filters/noop.rs index e2958311..04c31688 100644 --- a/src/filters/noop.rs +++ b/src/filters/noop.rs @@ -1,40 +1,26 @@ //! A noop filter used as example and placeholder for other filters -use gasket::framework::*; -use serde::Deserialize; - -use crate::framework::*; - -#[derive(Default, Stage)] -#[stage(name = "filter", unit = "ChainEvent", worker = "Worker")] -pub struct Stage { - pub input: FilterInputPort, - pub output: FilterOutputPort, +use std::thread; - #[metric] - ops_count: gasket::metrics::Counter, -} +use serde::Deserialize; -#[derive(Default)] -pub struct Worker; +use crate::pipelining::{ + new_inter_stage_channel, FilterProvider, PartialBootstrapResult, StageReceiver, +}; -impl From<&Stage> for Worker { - fn from(_: &Stage) -> Self { - Worker::default() - } -} +#[derive(Debug, Deserialize)] +pub struct Config {} -gasket::impl_mapper!(|_worker: Worker, stage: Stage, unit: ChainEvent| => { - let out = unit.clone(); - stage.ops_count.inc(1); - out -}); +impl FilterProvider for Config { + fn bootstrap(&self, input: StageReceiver) -> PartialBootstrapResult { + let (output_tx, output_rx) = new_inter_stage_channel(None); -#[derive(Default, Deserialize)] -pub struct Config {} + let handle = thread::spawn(move || { + for msg in input.iter() { + output_tx.send(msg).expect("error sending filter message"); + } + }); -impl Config { - pub fn bootstrapper(self, _ctx: &Context) -> Result { - Ok(Stage::default()) + Ok((handle, output_rx)) } } diff --git a/src/filters/parse_cbor.rs b/src/filters/parse_cbor.rs deleted file mode 100644 index 17c6e0eb..00000000 --- a/src/filters/parse_cbor.rs +++ /dev/null @@ -1,128 +0,0 @@ -//! A filter that turns raw cbor Tx into the corresponding parsed representation - -use gasket::framework::*; -use serde::Deserialize; - -use pallas::ledger::traverse as trv; -use utxorpc_spec_ledger::v1 as u5c; - -use crate::framework::*; - -fn from_traverse_tx(tx: &trv::MultiEraTx) -> u5c::Tx { - u5c::Tx { - inputs: tx - .inputs() - .iter() - .map(|i| u5c::TxInput { - tx_hash: i.hash().to_vec().into(), - output_index: i.index() as u32, - as_output: None, - }) - .collect(), - outputs: tx - .outputs() - .iter() - .map(|o| u5c::TxOutput { - address: o.address().map(|a| a.to_vec()).unwrap_or_default().into(), - coin: o.lovelace_amount(), - // TODO: this is wrong, we're crating a new item for each asset even if they share - // the same policy id. We need to adjust Pallas' interface to make this mapping more - // ergonomic. - assets: o - .non_ada_assets() - .iter() - .map(|a| u5c::Multiasset { - policy_id: a.policy().map(|x| x.to_vec()).unwrap_or_default().into(), - assets: vec![u5c::Asset { - name: a.name().map(|x| x.to_vec()).unwrap_or_default().into(), - quantity: a.coin() as u64, - }], - }) - .collect(), - datum: None, - datum_hash: Default::default(), - script: None, - redeemer: None, - }) - .collect(), - certificates: vec![], - withdrawals: vec![], - mint: vec![], - reference_inputs: vec![], - witnesses: u5c::WitnessSet { - vkeywitness: vec![], - script: vec![], - plutus_datums: vec![], - } - .into(), - collateral: u5c::Collateral { - collateral: vec![], - collateral_return: None, - total_collateral: Default::default(), - } - .into(), - fee: tx.fee().unwrap_or_default(), - validity: u5c::TxValidity { - start: tx.validity_start().unwrap_or_default(), - ttl: tx.ttl().unwrap_or_default(), - } - .into(), - successful: tx.is_valid(), - auxiliary: u5c::AuxData { - metadata: vec![], - scripts: vec![], - } - .into(), - } -} - -fn map_cbor_to_u5c(cbor: &[u8]) -> Result { - let tx = trv::MultiEraTx::decode(trv::Era::Babbage, cbor) - .or_else(|_| trv::MultiEraTx::decode(trv::Era::Alonzo, cbor)) - .or_else(|_| trv::MultiEraTx::decode(trv::Era::Byron, cbor)) - .or_panic()?; - - Ok(from_traverse_tx(&tx)) -} - -#[derive(Default, Stage)] -#[stage(name = "filter", unit = "ChainEvent", worker = "Worker")] -pub struct Stage { - pub input: FilterInputPort, - pub output: FilterOutputPort, - - #[metric] - ops_count: gasket::metrics::Counter, -} - -#[derive(Default)] -pub struct Worker; - -impl From<&Stage> for Worker { - fn from(_: &Stage) -> Self { - Worker::default() - } -} - -gasket::impl_mapper!(|_worker: Worker, stage: Stage, unit: ChainEvent| => { - let output = unit.clone().try_map_record(|r| match r { - Record::CborTx(cbor) => { - let tx = map_cbor_to_u5c(&cbor)?; - Ok(Record::ParsedTx(tx)) - } - x => Ok(x), - })?; - - stage.ops_count.inc(1); - - output -}); - -#[derive(Default, Deserialize)] -pub struct Config {} - -impl Config { - pub fn bootstrapper(self, _ctx: &Context) -> Result { - Ok(Stage::default()) - } -} diff --git a/src/filters/dsl.rs b/src/filters/selection.rs similarity index 85% rename from src/filters/dsl.rs rename to src/filters/selection.rs index 429a5257..2000d164 100644 --- a/src/filters/dsl.rs +++ b/src/filters/selection.rs @@ -1,10 +1,17 @@ //! A filter that can select which events to block and which to let pass -use gasket::framework::*; +use std::thread; + use serde::Deserialize; use serde_json::Value as JsonValue; -use crate::{framework::legacy_v1::*, framework::*}; +use crate::{ + model::{ + CIP25AssetRecord, Event, EventData, MetadataRecord, MetadatumRendition, MintRecord, + OutputAssetRecord, TransactionRecord, TxOutputRecord, + }, + pipelining::{new_inter_stage_channel, FilterProvider, PartialBootstrapResult, StageReceiver}, +}; #[derive(Debug, Deserialize, Clone, PartialEq)] #[serde(tag = "predicate", content = "argument", rename_all = "snake_case")] @@ -214,57 +221,25 @@ impl Predicate { } } -#[derive(Stage)] -#[stage(name = "filter", unit = "ChainEvent", worker = "Worker")] -pub struct Stage { - predicate: Predicate, - - pub input: FilterInputPort, - pub output: FilterOutputPort, - - #[metric] - ops_count: gasket::metrics::Counter, +#[derive(Debug, Deserialize)] +pub struct Config { + pub check: Predicate, } -#[derive(Default)] -pub struct Worker; +impl FilterProvider for Config { + fn bootstrap(&self, input: StageReceiver) -> PartialBootstrapResult { + let (output_tx, output_rx) = new_inter_stage_channel(None); -impl From<&Stage> for Worker { - fn from(_: &Stage) -> Self { - Worker::default() - } -} + let check = self.check.clone(); -gasket::impl_splitter!(|_worker: Worker, stage: Stage, unit: ChainEvent| => { - let out = match unit { - ChainEvent::Apply(_, Record::OuraV1Event(x)) => { - if stage.predicate.event_matches(x) { - Some(unit.clone()) - } else { - None + let handle = thread::spawn(move || { + for event in input.iter() { + if check.event_matches(&event) { + output_tx.send(event).expect("error sending filter message"); + } } - } - _ => todo!(), - }; - - stage.ops_count.inc(1); - out -}); - -#[derive(Debug, Deserialize)] -pub struct Config { - pub predicate: Predicate, -} - -impl Config { - pub fn bootstrapper(self, _ctx: &Context) -> Result { - let stage = Stage { - predicate: self.predicate, - ops_count: Default::default(), - input: Default::default(), - output: Default::default(), - }; + }); - Ok(stage) + Ok((handle, output_rx)) } } diff --git a/src/filters/split_block.rs b/src/filters/split_block.rs deleted file mode 100644 index 9af22c64..00000000 --- a/src/filters/split_block.rs +++ /dev/null @@ -1,71 +0,0 @@ -//! A noop filter used as example and placeholder for other filters - -use gasket::framework::*; -use serde::Deserialize; -use std::borrow::Cow; - -use pallas::ledger::traverse as trv; - -use crate::framework::*; - -type CborBlock<'a> = Cow<'a, [u8]>; -type CborTx<'a> = Cow<'a, [u8]>; - -fn map_block_to_tx(cbor: CborBlock) -> Result, WorkerError> { - let block = trv::MultiEraBlock::decode(cbor.as_ref()).or_panic()?; - - let txs: Vec<_> = block - .txs() - .iter() - .map(|tx| tx.encode()) - .map(Cow::Owned) - .collect(); - - Ok(txs) -} - -#[derive(Default, Stage)] -#[stage(name = "filter", unit = "ChainEvent", worker = "Worker")] -pub struct Stage { - pub input: FilterInputPort, - pub output: FilterOutputPort, - - #[metric] - ops_count: gasket::metrics::Counter, -} - -#[derive(Default)] -pub struct Worker; - -impl From<&Stage> for Worker { - fn from(_: &Stage) -> Self { - Worker::default() - } -} - -gasket::impl_splitter!(|_worker: Worker, stage: Stage, unit: ChainEvent| => { - let output = unit.clone().try_map_record_to_many(|r| match r { - Record::CborBlock(cbor) => { - let out = map_block_to_tx(Cow::Borrowed(&cbor))? - .into_iter() - .map(|tx| Record::CborTx(tx.into())) - .collect(); - - Ok(out) - } - x => Ok(vec![x]), - })?; - - stage.ops_count.inc(1); - - output -}); - -#[derive(Default, Deserialize)] -pub struct Config {} - -impl Config { - pub fn bootstrapper(self, _ctx: &Context) -> Result { - Ok(Stage::default()) - } -} diff --git a/src/filters/wasm.rs b/src/filters/wasm.rs deleted file mode 100644 index e2958311..00000000 --- a/src/filters/wasm.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! A noop filter used as example and placeholder for other filters - -use gasket::framework::*; -use serde::Deserialize; - -use crate::framework::*; - -#[derive(Default, Stage)] -#[stage(name = "filter", unit = "ChainEvent", worker = "Worker")] -pub struct Stage { - pub input: FilterInputPort, - pub output: FilterOutputPort, - - #[metric] - ops_count: gasket::metrics::Counter, -} - -#[derive(Default)] -pub struct Worker; - -impl From<&Stage> for Worker { - fn from(_: &Stage) -> Self { - Worker::default() - } -} - -gasket::impl_mapper!(|_worker: Worker, stage: Stage, unit: ChainEvent| => { - let out = unit.clone(); - stage.ops_count.inc(1); - out -}); - -#[derive(Default, Deserialize)] -pub struct Config {} - -impl Config { - pub fn bootstrapper(self, _ctx: &Context) -> Result { - Ok(Stage::default()) - } -} diff --git a/src/framework/cursor.rs b/src/framework/cursor.rs deleted file mode 100644 index 6ad24b70..00000000 --- a/src/framework/cursor.rs +++ /dev/null @@ -1,45 +0,0 @@ -use std::{ - collections::VecDeque, - sync::{Arc, RwLock}, -}; - -use pallas::network::miniprotocols::Point; - -const HARDCODED_BREADCRUMBS: usize = 20; - -type State = VecDeque; - -// TODO: include exponential breadcrumbs logic here -#[derive(Clone)] -pub struct Cursor(Arc>); - -impl Cursor { - pub fn new(state: State) -> Self { - Self(Arc::new(RwLock::new(state))) - } - - pub fn is_empty(&self) -> bool { - let v = self.0.read().unwrap(); - v.is_empty() - } - - pub fn clone_state(&self) -> State { - let v = self.0.read().unwrap(); - v.clone() - } - - pub fn latest_known_point(&self) -> Option { - let state = self.0.read().unwrap(); - state.front().cloned() - } - - pub fn add_breadcrumb(&self, value: Point) { - let mut state = self.0.write().unwrap(); - - state.push_front(value); - - if state.len() > HARDCODED_BREADCRUMBS { - state.pop_back(); - } - } -} diff --git a/src/framework/errors.rs b/src/framework/errors.rs deleted file mode 100644 index 94e72d07..00000000 --- a/src/framework/errors.rs +++ /dev/null @@ -1,27 +0,0 @@ -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum Error { - #[error("config error")] - Config(String), - - #[error("{0}")] - Custom(String), - - #[error("parse error {0}")] - Parse(String), -} - -impl Error { - pub fn config(err: impl ToString) -> Self { - Self::Config(err.to_string()) - } - - pub fn custom(err: impl ToString) -> Self { - Self::Custom(err.to_string()) - } - - pub fn parse(error: impl ToString) -> Self { - Self::Parse(error.to_string()) - } -} diff --git a/src/framework/legacy_v1.rs b/src/framework/legacy_v1.rs deleted file mode 100644 index 7ba82a44..00000000 --- a/src/framework/legacy_v1.rs +++ /dev/null @@ -1,375 +0,0 @@ -use std::fmt::Display; - -use merge::Merge; - -use serde::{Deserialize, Serialize}; -use serde_json::Value as JsonValue; - -use strum_macros::Display; - -// We're duplicating the Era struct from Pallas for two reasons: a) we need it -// to be serializable and we don't want to impose serde dependency on Pallas and -// b) we prefer not to add dependencies to Pallas outside of the sources that -// actually use it on an attempt to make the pipeline agnostic of particular -// implementation details. -#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Display)] -pub enum Era { - Undefined, - Unknown, - Byron, - Shelley, - Allegra, - Mary, - Alonzo, - Babbage, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum MetadatumRendition { - MapJson(JsonValue), - ArrayJson(JsonValue), - IntScalar(i128), - TextScalar(String), - BytesHex(String), -} - -impl Display for MetadatumRendition { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - MetadatumRendition::MapJson(x) => x.fmt(f), - MetadatumRendition::ArrayJson(x) => x.fmt(f), - MetadatumRendition::IntScalar(x) => x.fmt(f), - MetadatumRendition::TextScalar(x) => x.fmt(f), - MetadatumRendition::BytesHex(x) => x.fmt(f), - } - } -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct MetadataRecord { - pub label: String, - - #[serde(flatten)] - pub content: MetadatumRendition, -} - -impl From for EventData { - fn from(x: MetadataRecord) -> Self { - EventData::Metadata(x) - } -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct CIP25AssetRecord { - pub version: String, - pub policy: String, - pub asset: String, - pub name: Option, - pub image: Option, - pub media_type: Option, - pub description: Option, - pub raw_json: JsonValue, -} - -impl From for EventData { - fn from(x: CIP25AssetRecord) -> Self { - EventData::CIP25Asset(x) - } -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct CIP15AssetRecord { - pub voting_key: String, - pub stake_pub: String, - pub reward_address: String, - pub nonce: i64, - pub raw_json: JsonValue, -} - -impl From for EventData { - fn from(x: CIP15AssetRecord) -> Self { - EventData::CIP15Asset(x) - } -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct TxInputRecord { - pub tx_id: String, - pub index: u64, -} - -impl From for EventData { - fn from(x: TxInputRecord) -> Self { - EventData::TxInput(x) - } -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct OutputAssetRecord { - pub policy: String, - pub asset: String, - pub asset_ascii: Option, - pub amount: u64, -} - -impl From for EventData { - fn from(x: OutputAssetRecord) -> Self { - EventData::OutputAsset(x) - } -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct TxOutputRecord { - pub address: String, - pub amount: u64, - pub assets: Option>, - pub datum_hash: Option, - pub inline_datum: Option, -} - -impl From for EventData { - fn from(x: TxOutputRecord) -> Self { - EventData::TxOutput(x) - } -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct MintRecord { - pub policy: String, - pub asset: String, - pub quantity: i64, -} - -impl From for EventData { - fn from(x: MintRecord) -> Self { - EventData::Mint(x) - } -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct WithdrawalRecord { - pub reward_account: String, - pub coin: u64, -} - -#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)] -pub struct TransactionRecord { - pub hash: String, - pub fee: u64, - pub ttl: Option, - pub validity_interval_start: Option, - pub network_id: Option, - pub input_count: usize, - pub collateral_input_count: usize, - pub has_collateral_output: bool, - pub output_count: usize, - pub mint_count: usize, - pub total_output: u64, - - // include_details - pub metadata: Option>, - pub inputs: Option>, - pub outputs: Option>, - pub collateral_inputs: Option>, - pub collateral_output: Option, - pub mint: Option>, - pub vkey_witnesses: Option>, - pub native_witnesses: Option>, - pub plutus_witnesses: Option>, - pub plutus_redeemers: Option>, - pub plutus_data: Option>, - pub withdrawals: Option>, - pub size: u32, -} - -impl From for EventData { - fn from(x: TransactionRecord) -> Self { - EventData::Transaction(x) - } -} - -#[derive(Serialize, Deserialize, Debug, Clone, Merge, Default)] -pub struct EventContext { - pub block_hash: Option, - pub block_number: Option, - pub slot: Option, - pub timestamp: Option, - pub tx_idx: Option, - pub tx_hash: Option, - pub input_idx: Option, - pub output_idx: Option, - pub output_address: Option, - pub certificate_idx: Option, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub enum StakeCredential { - AddrKeyhash(String), - Scripthash(String), -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct VKeyWitnessRecord { - pub vkey_hex: String, - pub signature_hex: String, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct NativeWitnessRecord { - pub policy_id: String, - pub script_json: JsonValue, -} - -impl From for EventData { - fn from(x: NativeWitnessRecord) -> Self { - EventData::NativeWitness(x) - } -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct PlutusWitnessRecord { - pub script_hash: String, - pub script_hex: String, -} - -impl From for EventData { - fn from(x: PlutusWitnessRecord) -> Self { - EventData::PlutusWitness(x) - } -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct PlutusRedeemerRecord { - pub purpose: String, - pub ex_units_mem: u32, - pub ex_units_steps: u64, - pub input_idx: u32, - pub plutus_data: JsonValue, -} - -impl From for EventData { - fn from(x: PlutusRedeemerRecord) -> Self { - EventData::PlutusRedeemer(x) - } -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct PlutusDatumRecord { - pub datum_hash: String, - pub plutus_data: JsonValue, -} - -impl From for EventData { - fn from(x: PlutusDatumRecord) -> Self { - EventData::PlutusDatum(x) - } -} -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct BlockRecord { - pub era: Era, - pub epoch: Option, - pub epoch_slot: Option, - pub body_size: usize, - pub issuer_vkey: String, - pub vrf_vkey: String, - pub tx_count: usize, - pub slot: u64, - pub hash: String, - pub number: u64, - pub previous_hash: String, - pub cbor_hex: Option, - pub transactions: Option>, -} - -impl From for EventData { - fn from(x: BlockRecord) -> Self { - EventData::Block(x) - } -} - -#[derive(Serialize, Deserialize, Display, Debug, Clone)] -#[serde(rename_all = "snake_case")] -pub enum EventData { - Block(BlockRecord), - BlockEnd(BlockRecord), - Transaction(TransactionRecord), - TransactionEnd(TransactionRecord), - TxInput(TxInputRecord), - TxOutput(TxOutputRecord), - OutputAsset(OutputAssetRecord), - Metadata(MetadataRecord), - - VKeyWitness(VKeyWitnessRecord), - NativeWitness(NativeWitnessRecord), - PlutusWitness(PlutusWitnessRecord), - PlutusRedeemer(PlutusRedeemerRecord), - PlutusDatum(PlutusDatumRecord), - - #[serde(rename = "cip25_asset")] - CIP25Asset(CIP25AssetRecord), - - #[serde(rename = "cip15_asset")] - CIP15Asset(CIP15AssetRecord), - - Mint(MintRecord), - Collateral { - tx_id: String, - index: u64, - }, - NativeScript { - policy_id: String, - script: JsonValue, - }, - PlutusScript { - hash: String, - data: String, - }, - StakeRegistration { - credential: StakeCredential, - }, - StakeDeregistration { - credential: StakeCredential, - }, - StakeDelegation { - credential: StakeCredential, - pool_hash: String, - }, - PoolRegistration { - operator: String, - vrf_keyhash: String, - pledge: u64, - cost: u64, - margin: f64, - reward_account: String, - pool_owners: Vec, - relays: Vec, - pool_metadata: Option, - pool_metadata_hash: Option, - }, - PoolRetirement { - pool: String, - epoch: u64, - }, - GenesisKeyDelegation {}, - MoveInstantaneousRewardsCert { - from_reserves: bool, - from_treasury: bool, - to_stake_credentials: Option>, - to_other_pot: Option, - }, - RollBack { - block_slot: u64, - block_hash: String, - }, -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct Event { - pub context: EventContext, - - #[serde(flatten)] - pub data: EventData, - - pub fingerprint: Option, -} diff --git a/src/framework/mod.rs b/src/framework/mod.rs deleted file mode 100644 index 723abc59..00000000 --- a/src/framework/mod.rs +++ /dev/null @@ -1,248 +0,0 @@ -//! Internal pipeline framework - -use pallas::network::miniprotocols::Point; -use serde::Deserialize; -use std::fmt::Debug; -use std::path::PathBuf; - -// we use UtxoRpc as our canonical representation of a parsed Tx -pub use utxorpc_spec_ledger::v1::Tx as ParsedTx; - -// we use GenesisValues from Pallas as our ChainConfig -pub use pallas::ledger::traverse::wellknown::GenesisValues; - -pub mod cursor; -pub mod errors; -pub mod legacy_v1; - -pub use cursor::*; -pub use errors::*; - -#[derive(Deserialize)] -#[serde(tag = "type")] -pub enum ChainConfig { - Mainnet, - Testnet, - PreProd, - Preview, - Custom(GenesisValues), -} - -impl Default for ChainConfig { - fn default() -> Self { - Self::Mainnet - } -} - -impl From for GenesisValues { - fn from(other: ChainConfig) -> Self { - match other { - ChainConfig::Mainnet => GenesisValues::mainnet(), - ChainConfig::Testnet => GenesisValues::testnet(), - ChainConfig::PreProd => GenesisValues::preprod(), - ChainConfig::Preview => GenesisValues::preview(), - ChainConfig::Custom(x) => x, - } - } -} - -pub struct Context { - pub chain: GenesisValues, - pub intersect: IntersectConfig, - pub cursor: Cursor, - pub finalize: Option, - pub current_dir: PathBuf, -} - -use serde_json::{json, Value as JsonValue}; - -#[derive(Debug, Clone)] -pub enum Record { - CborBlock(Vec), - CborTx(Vec), - GenericJson(JsonValue), - OuraV1Event(legacy_v1::Event), - ParsedTx(ParsedTx), -} - -impl From for JsonValue { - fn from(value: Record) -> Self { - match value { - Record::CborBlock(x) => json!({ "hex": hex::encode(x) }), - Record::CborTx(x) => json!({ "hex": hex::encode(x) }), - Record::ParsedTx(x) => json!(x), - Record::OuraV1Event(x) => json!(x), - Record::GenericJson(x) => x, - } - } -} - -#[derive(Debug, Clone)] -pub enum ChainEvent { - Apply(Point, Record), - Undo(Point, Record), - Reset(Point), -} - -impl ChainEvent { - pub fn apply(point: Point, record: impl Into) -> gasket::messaging::Message { - gasket::messaging::Message { - payload: Self::Apply(point, record.into()), - } - } - - pub fn undo(point: Point, record: impl Into) -> gasket::messaging::Message { - gasket::messaging::Message { - payload: Self::Undo(point, record.into()), - } - } - - pub fn reset(point: Point) -> gasket::messaging::Message { - gasket::messaging::Message { - payload: Self::Reset(point), - } - } - - pub fn point(&self) -> &Point { - match self { - Self::Apply(x, _) => x, - Self::Undo(x, _) => x, - Self::Reset(x) => x, - } - } - - pub fn record(&self) -> Option<&Record> { - match self { - Self::Apply(_, x) => Some(x), - Self::Undo(_, x) => Some(x), - _ => None, - } - } - - pub fn map_record(self, f: fn(Record) -> Record) -> Self { - match self { - Self::Apply(p, x) => Self::Apply(p, f(x)), - Self::Undo(p, x) => Self::Undo(p, f(x)), - Self::Reset(x) => Self::Reset(x), - } - } - - pub fn try_map_record(self, f: fn(Record) -> Result) -> Result { - let out = match self { - Self::Apply(p, x) => Self::Apply(p, f(x)?), - Self::Undo(p, x) => Self::Undo(p, f(x)?), - Self::Reset(x) => Self::Reset(x), - }; - - Ok(out) - } - - pub fn try_map_record_to_many( - self, - f: fn(Record) -> Result, E>, - ) -> Result, E> { - let out = match self { - Self::Apply(p, x) => f(x)? - .into_iter() - .map(|i| Self::Apply(p.clone(), i)) - .collect(), - Self::Undo(p, x) => f(x)? - .into_iter() - .map(|i| Self::Undo(p.clone(), i)) - .collect(), - Self::Reset(x) => vec![Self::Reset(x)], - }; - - Ok(out) - } -} - -pub type SourceOutputPort = gasket::messaging::tokio::OutputPort; -pub type FilterInputPort = gasket::messaging::tokio::InputPort; -pub type FilterOutputPort = gasket::messaging::tokio::OutputPort; -pub type MapperInputPort = gasket::messaging::tokio::InputPort; -pub type MapperOutputPort = gasket::messaging::tokio::OutputPort; -pub type SinkInputPort = gasket::messaging::tokio::InputPort; - -pub type OutputAdapter = gasket::messaging::tokio::ChannelSendAdapter; -pub type InputAdapter = gasket::messaging::tokio::ChannelRecvAdapter; - -pub trait StageBootstrapper { - fn connect_output(&mut self, adapter: OutputAdapter); - fn connect_input(&mut self, adapter: InputAdapter); - fn spawn(self, policy: gasket::runtime::Policy) -> gasket::runtime::Tether; -} - -#[derive(Debug, Deserialize, Clone)] -#[serde(tag = "type", content = "value")] -pub enum IntersectConfig { - Tip, - Origin, - Point(u64, String), - Breadcrumbs(Vec<(u64, String)>), -} - -impl IntersectConfig { - pub fn points(&self) -> Option> { - match self { - IntersectConfig::Breadcrumbs(all) => { - let mapped = all - .iter() - .map(|(slot, hash)| { - let hash = hex::decode(hash).expect("valid hex hash"); - Point::Specific(*slot, hash) - }) - .collect(); - - Some(mapped) - } - IntersectConfig::Point(slot, hash) => { - let hash = hex::decode(hash).expect("valid hex hash"); - Some(vec![Point::Specific(*slot, hash)]) - } - _ => None, - } - } -} - -/// Optional configuration to stop processing new blocks after processing: -/// 1. a block with the given hash -/// 2. the first block on or after a given absolute slot -/// 3. TODO: a total of X blocks -#[derive(Deserialize, Debug, Clone)] -pub struct FinalizeConfig { - until_hash: Option, - max_block_slot: Option, - // max_block_quantity: Option, -} - -pub fn should_finalize( - config: &Option, - last_point: &Point, - // block_count: u64, -) -> bool { - let config = match config { - Some(x) => x, - None => return false, - }; - - if let Some(expected) = &config.until_hash { - if let Point::Specific(_, current) = last_point { - return expected == &hex::encode(current); - } - } - - if let Some(max) = config.max_block_slot { - if last_point.slot_or_default() >= max { - return true; - } - } - - // if let Some(max) = config.max_block_quantity { - // if block_count >= max { - // return true; - // } - // } - - false -} diff --git a/src/lib.rs b/src/lib.rs index 42aeeccb..e744857a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,10 @@ +pub mod model; +pub mod utils; + pub mod filters; -pub mod framework; +pub mod mapper; +pub mod pipelining; pub mod sinks; pub mod sources; + +pub type Error = Box; diff --git a/src/mapper/babbage.rs b/src/mapper/babbage.rs new file mode 100644 index 00000000..d84590f6 --- /dev/null +++ b/src/mapper/babbage.rs @@ -0,0 +1,510 @@ +use pallas::codec::utils::KeepRaw; +use std::collections::HashMap; + +use pallas::ledger::primitives::babbage::{ + AuxiliaryData, CostMdls, Language, MintedBlock, MintedDatumOption, + MintedPostAlonzoTransactionOutput, MintedTransactionBody, MintedTransactionOutput, + MintedWitnessSet, NetworkId, ProtocolParamUpdate, Update, +}; + +use pallas::crypto::hash::Hash; +use pallas::ledger::traverse::OriginalHash; +use serde_json::json; + +use crate::model::{ + BlockRecord, CostModelRecord, CostModelsRecord, Era, LanguageVersionRecord, + ProtocolParamUpdateRecord, TransactionRecord, UpdateRecord, +}; +use crate::utils::time::TimeProvider; +use crate::{ + model::{EventContext, EventData}, + Error, +}; + +use super::{map::ToHex, EventWriter}; + +impl EventWriter { + pub fn to_babbage_tx_size( + &self, + body: &KeepRaw, + aux_data: Option<&KeepRaw>, + witness_set: Option<&KeepRaw>, + ) -> usize { + body.raw_cbor().len() + + aux_data.map(|ax| ax.raw_cbor().len()).unwrap_or(2) + + witness_set.map(|ws| ws.raw_cbor().len()).unwrap_or(1) + } + + pub fn to_babbage_transaction_record( + &self, + body: &KeepRaw, + tx_hash: &str, + aux_data: Option<&KeepRaw>, + witness_set: Option<&KeepRaw>, + ) -> Result { + let mut record = TransactionRecord { + hash: tx_hash.to_owned(), + size: self.to_babbage_tx_size(body, aux_data, witness_set) as u32, + fee: body.fee, + ttl: body.ttl, + validity_interval_start: body.validity_interval_start, + network_id: body.network_id.as_ref().map(|x| match x { + NetworkId::One => 1, + NetworkId::Two => 2, + }), + ..Default::default() + }; + + let outputs = self.collect_any_output_records(&body.outputs)?; + record.output_count = outputs.len(); + record.total_output = outputs.iter().map(|o| o.amount).sum(); + + let inputs = self.collect_input_records(&body.inputs); + record.input_count = inputs.len(); + + if let Some(mint) = &body.mint { + let mints = self.collect_mint_records(mint); + record.mint_count = mints.len(); + + if self.config.include_transaction_details { + record.mint = mints.into(); + } + } + + if let Some(certs) = &body.certificates { + let certs = self.collect_certificate_records(certs); + record.certificate_count = certs.len(); + + if self.config.include_transaction_details { + record.certs = certs.into(); + } + } + + // Add Collateral Stuff + let collateral_inputs = &body.collateral; + record.collateral_input_count = collateral_inputs.iter().count(); + record.has_collateral_output = body.collateral_return.is_some(); + + if let Some(update) = &body.update { + if self.config.include_transaction_details { + record.update = Some(self.to_babbage_update_record(update)); + } + } + + if let Some(req_signers) = &body.required_signers { + let req_signers = self.collect_required_signers_records(req_signers.into())?; + record.required_signers_count = req_signers.len(); + + if self.config.include_transaction_details { + record.required_signers = Some(req_signers); + } + } + + // TODO + // TransactionBodyComponent::ScriptDataHash(_) + // TransactionBodyComponent::AuxiliaryDataHash(_) + + if self.config.include_transaction_details { + record.outputs = outputs.into(); + record.inputs = inputs.into(); + + // transaction_details collateral stuff + record.collateral_inputs = + collateral_inputs.as_ref().map(|inputs| self.collect_input_records(inputs)); + + record.collateral_output = body.collateral_return.as_ref().map(|output| match output { + MintedTransactionOutput::Legacy(x) => self.to_legacy_output_record(x).unwrap(), + MintedTransactionOutput::PostAlonzo(x) => { + self.to_post_alonzo_output_record(x).unwrap() + } + }); + + record.metadata = match aux_data { + Some(aux_data) => self.collect_metadata_records(aux_data)?.into(), + None => None, + }; + + if let Some(witnesses) = witness_set { + record.vkey_witnesses = self + .collect_vkey_witness_records_babbage(&witnesses.vkeywitness)? + .into(); + + record.native_witnesses = self + .collect_native_witness_records_babbage(&witnesses.native_script)? + .into(); + + record.plutus_witnesses = self + .collect_plutus_v1_witness_records_babbage(&witnesses.plutus_v1_script)? + .into(); + + record.plutus_redeemers = self + .collect_plutus_redeemer_records(&witnesses.redeemer)? + .into(); + + record.plutus_data = self + .collect_witness_plutus_datum_records_babbage(&witnesses.plutus_data)? + .into(); + } + + if let Some(withdrawals) = &body.withdrawals { + record.withdrawals = self.collect_withdrawal_records(withdrawals).into(); + } + } + + Ok(record) + } + + pub fn to_babbage_block_record( + &self, + source: &MintedBlock, + hash: &Hash<32>, + cbor: &[u8], + ) -> Result { + let relative_epoch = self + .utils + .time + .as_ref() + .map(|time| time.absolute_slot_to_relative(source.header.header_body.slot)); + + let mut record = BlockRecord { + era: Era::Babbage, + body_size: source.header.header_body.block_body_size as usize, + issuer_vkey: source.header.header_body.issuer_vkey.to_hex(), + vrf_vkey: source.header.header_body.vrf_vkey.to_hex(), + tx_count: source.transaction_bodies.len(), + hash: hex::encode(hash), + number: source.header.header_body.block_number, + slot: source.header.header_body.slot, + epoch: relative_epoch.map(|(epoch, _)| epoch), + epoch_slot: relative_epoch.map(|(_, epoch_slot)| epoch_slot), + previous_hash: source + .header + .header_body + .prev_hash + .map(hex::encode) + .unwrap_or_default(), + cbor_hex: match self.config.include_block_cbor { + true => hex::encode(cbor).into(), + false => None, + }, + transactions: None, + }; + + if self.config.include_block_details { + record.transactions = Some(self.collect_babbage_tx_records(source)?); + } + + Ok(record) + } + + pub fn collect_babbage_tx_records( + &self, + block: &MintedBlock, + ) -> Result, Error> { + block + .transaction_bodies + .iter() + .enumerate() + .map(|(idx, tx)| { + let aux_data = block + .auxiliary_data_set + .iter() + .find(|(k, _)| *k == (idx as u32)) + .map(|(_, v)| v); + + let witness_set = block.transaction_witness_sets.get(idx); + + let tx_hash = tx.original_hash().to_hex(); + + self.to_babbage_transaction_record(tx, &tx_hash, aux_data, witness_set) + }) + .collect() + } + + fn crawl_post_alonzo_output( + &self, + output: &MintedPostAlonzoTransactionOutput, + ) -> Result<(), Error> { + let record = self.to_post_alonzo_output_record(output)?; + self.append(record.into())?; + + let address = pallas::ledger::addresses::Address::from_bytes(&output.address)?; + + let child = &self.child_writer(EventContext { + output_address: address.to_string().into(), + ..EventContext::default() + }); + + child.crawl_transaction_output_amount(&output.value)?; + + if let Some(MintedDatumOption::Data(datum)) = &output.datum_option { + let record = self.to_plutus_datum_record(datum)?; + child.append(record.into())?; + } + + Ok(()) + } + + fn crawl_babbage_transaction_output( + &self, + output: &MintedTransactionOutput, + ) -> Result<(), Error> { + match output { + MintedTransactionOutput::Legacy(x) => self.crawl_legacy_output(x), + MintedTransactionOutput::PostAlonzo(x) => self.crawl_post_alonzo_output(x), + } + } + + fn crawl_babbage_witness_set( + &self, + witness_set: &KeepRaw, + ) -> Result<(), Error> { + if let Some(native) = &witness_set.native_script { + for script in native.iter() { + self.append_from(self.to_native_witness_record(script)?)?; + } + } + + if let Some(plutus) = &witness_set.plutus_v1_script { + for script in plutus.iter() { + self.append_from(self.to_plutus_v1_witness_record(script)?)?; + } + } + + if let Some(redeemers) = &witness_set.redeemer { + for redeemer in redeemers.iter() { + self.append_from(self.to_plutus_redeemer_record(redeemer)?)?; + } + } + + if let Some(datums) = &witness_set.plutus_data { + for datum in datums { + self.append_from(self.to_plutus_datum_record(datum)?)?; + } + } + + Ok(()) + } + + fn crawl_babbage_transaction( + &self, + tx: &KeepRaw, + tx_hash: &str, + aux_data: Option<&KeepRaw>, + witness_set: Option<&KeepRaw>, + ) -> Result<(), Error> { + let record = self.to_babbage_transaction_record(tx, tx_hash, aux_data, witness_set)?; + + self.append_from(record.clone())?; + + for (idx, input) in tx.inputs.iter().enumerate() { + let child = self.child_writer(EventContext { + input_idx: Some(idx), + ..EventContext::default() + }); + + child.crawl_transaction_input(input)?; + } + + for (idx, output) in tx.outputs.iter().enumerate() { + let child = self.child_writer(EventContext { + output_idx: Some(idx), + ..EventContext::default() + }); + + child.crawl_babbage_transaction_output(output)?; + } + + if let Some(certs) = &tx.certificates { + for (idx, cert) in certs.iter().enumerate() { + let child = self.child_writer(EventContext { + certificate_idx: Some(idx), + ..EventContext::default() + }); + + child.crawl_certificate(cert)?; + } + } + + if let Some(collateral) = &tx.collateral { + for (_idx, collateral) in collateral.iter().enumerate() { + // TODO: collateral context? + + self.crawl_collateral(collateral)?; + } + } + + if let Some(mint) = &tx.mint { + self.crawl_mints(mint)?; + } + + if let Some(aux_data) = aux_data { + self.crawl_auxdata(aux_data)?; + } + + if let Some(witness_set) = witness_set { + self.crawl_babbage_witness_set(witness_set)?; + } + + if self.config.include_transaction_end_events { + self.append(EventData::TransactionEnd(record))?; + } + + Ok(()) + } + + fn crawl_babbage_block( + &self, + block: &MintedBlock, + hash: &Hash<32>, + cbor: &[u8], + ) -> Result<(), Error> { + let record = self.to_babbage_block_record(block, hash, cbor)?; + + self.append(EventData::Block(record.clone()))?; + + for (idx, tx) in block.transaction_bodies.iter().enumerate() { + let aux_data = block + .auxiliary_data_set + .iter() + .find(|(k, _)| *k == (idx as u32)) + .map(|(_, v)| v); + + let witness_set = block.transaction_witness_sets.get(idx); + + let tx_hash = tx.original_hash().to_hex(); + + let child = self.child_writer(EventContext { + tx_idx: Some(idx), + tx_hash: Some(tx_hash.to_owned()), + ..EventContext::default() + }); + + child.crawl_babbage_transaction(tx, &tx_hash, aux_data, witness_set)?; + } + + if self.config.include_block_end_events { + self.append(EventData::BlockEnd(record))?; + } + + Ok(()) + } + + pub fn to_babbage_cost_models_record( + &self, + cost_models: &Option, + ) -> Option { + match cost_models { + Some(cost_models) => { + let mut cost_models_record = HashMap::new(); + if let Some(cost_model_v1) = &cost_models.plutus_v1 { + let language_version_record = LanguageVersionRecord::PlutusV1; + let cost_model_record = CostModelRecord(cost_model_v1.clone()); + cost_models_record.insert(language_version_record, cost_model_record); + } + if let Some(cost_model_v2) = &cost_models.plutus_v2 { + let language_version_record = LanguageVersionRecord::PlutusV2; + let cost_model_record = CostModelRecord(cost_model_v2.clone()); + cost_models_record.insert(language_version_record, cost_model_record); + } + + Some(CostModelsRecord(cost_models_record)) + } + None => None, + } + } + + pub fn to_babbage_language_version_record( + &self, + language_version: &Language, + ) -> LanguageVersionRecord { + match language_version { + Language::PlutusV1 => LanguageVersionRecord::PlutusV1, + Language::PlutusV2 => LanguageVersionRecord::PlutusV2, + } + } + + pub fn to_babbage_protocol_update_record( + &self, + update: &ProtocolParamUpdate, + ) -> ProtocolParamUpdateRecord { + ProtocolParamUpdateRecord { + minfee_a: update.minfee_a, + minfee_b: update.minfee_b, + max_block_body_size: update.max_block_body_size, + max_transaction_size: update.max_transaction_size, + max_block_header_size: update.max_block_header_size, + key_deposit: update.key_deposit, + pool_deposit: update.pool_deposit, + maximum_epoch: update.maximum_epoch, + desired_number_of_stake_pools: update.desired_number_of_stake_pools, + pool_pledge_influence: self + .to_rational_number_record_option(&update.pool_pledge_influence), + expansion_rate: self.to_unit_interval_record(&update.expansion_rate), + treasury_growth_rate: self.to_unit_interval_record(&update.treasury_growth_rate), + decentralization_constant: None, + extra_entropy: None, + protocol_version: update.protocol_version, + min_pool_cost: update.min_pool_cost, + ada_per_utxo_byte: update.ada_per_utxo_byte, + cost_models_for_script_languages: self + .to_babbage_cost_models_record(&update.cost_models_for_script_languages), + execution_costs: match &update.execution_costs { + Some(execution_costs) => Some(json!(execution_costs)), + None => None, + }, + max_tx_ex_units: self.to_ex_units_record(&update.max_tx_ex_units), + max_block_ex_units: self.to_ex_units_record(&update.max_block_ex_units), + max_value_size: update.max_value_size, + collateral_percentage: update.collateral_percentage, + max_collateral_inputs: update.max_collateral_inputs, + } + } + + pub fn to_babbage_update_record(&self, update: &Update) -> UpdateRecord { + let mut updates = HashMap::new(); + for update in update.proposed_protocol_parameter_updates.clone().to_vec() { + updates.insert( + update.0.to_hex(), + self.to_babbage_protocol_update_record(&update.1), + ); + } + + UpdateRecord { + proposed_protocol_parameter_updates: updates, + epoch: update.epoch, + } + } + + /// Mapper entry-point for decoded Babbage blocks + /// + /// Entry-point to start crawling a blocks for events. Meant to be used when + /// we already have a decoded block (for example, N2C). The raw CBOR is also + /// passed through in case we need to attach it to outbound events. + pub fn crawl_babbage_with_cbor<'b>( + &self, + block: &'b MintedBlock<'b>, + cbor: &'b [u8], + ) -> Result<(), Error> { + let hash = block.header.original_hash(); + + let child = self.child_writer(EventContext { + block_hash: Some(hex::encode(hash)), + block_number: Some(block.header.header_body.block_number), + slot: Some(block.header.header_body.slot), + timestamp: self.compute_timestamp(block.header.header_body.slot), + ..EventContext::default() + }); + + child.crawl_babbage_block(block, &hash, cbor) + } + + /// Mapper entry-point for raw Babbage cbor blocks + /// + /// Entry-point to start crawling a blocks for events. Meant to be used when + /// we haven't decoded the CBOR yet (for example, N2N). + pub fn crawl_from_babbage_cbor(&self, cbor: &[u8]) -> Result<(), Error> { + let (_, block): (u16, MintedBlock) = pallas::codec::minicbor::decode(cbor)?; + self.crawl_babbage_with_cbor(&block, cbor) + } +} diff --git a/src/mapper/byron.rs b/src/mapper/byron.rs new file mode 100644 index 00000000..27d836e4 --- /dev/null +++ b/src/mapper/byron.rs @@ -0,0 +1,359 @@ +use std::ops::Deref; + +use super::map::ToHex; +use super::EventWriter; +use crate::model::{BlockRecord, Era, EventData, TransactionRecord, TxInputRecord, TxOutputRecord}; +use crate::{model::EventContext, Error}; + +use pallas::crypto::hash::Hash; +use pallas::ledger::primitives::byron; +use pallas::ledger::traverse::OriginalHash; + +impl EventWriter { + fn to_byron_input_record(&self, source: &byron::TxIn) -> Option { + match source { + byron::TxIn::Variant0(x) => { + let (hash, index) = x.deref(); + + Some(TxInputRecord { + tx_id: hash.to_hex(), + index: *index as u64, + }) + } + byron::TxIn::Other(a, b) => { + log::warn!( + "don't know how to handle byron input: ({}, {})", + a, + b.to_hex() + ); + + None + } + } + } + + fn collect_byron_input_records(&self, source: &byron::Tx) -> Vec { + source + .inputs + .iter() + .filter_map(|x| self.to_byron_input_record(x)) + .collect() + } + + fn to_byron_output_record(&self, source: &byron::TxOut) -> Result { + let address: pallas::ledger::addresses::Address = + pallas::ledger::addresses::ByronAddress::new( + &source.address.payload.0, + source.address.crc, + ) + .into(); + + Ok(TxOutputRecord { + address: address.to_string(), + amount: source.amount, + assets: None, + datum_hash: None, + inline_datum: None, + inlined_script: None, + }) + } + + fn collect_byron_output_records( + &self, + source: &byron::Tx, + ) -> Result, Error> { + source + .outputs + .iter() + .map(|x| self.to_byron_output_record(x)) + .collect() + } + + fn to_byron_transaction_record( + &self, + source: &byron::MintedTxPayload, + tx_hash: &str, + ) -> Result { + let input_records = self.collect_byron_input_records(&source.transaction); + let output_records = self.collect_byron_output_records(&source.transaction)?; + + let mut record = TransactionRecord { + hash: tx_hash.to_owned(), + // TODO: we have a problem here. AFAIK, there's no reference to the tx fee in the + // block contents. This leaves us with the two alternative: a) compute the value, b) + // omit the value. + // + // Computing the value is not trivial, the linear policy is easy to + // implement, but tracking the parameters for each epoch means hardcoding values or + // doing some extra queries. + // + // Ommiting the value elegantly would require turning the property data type into an + // option, which is a breaking change. + // + // Chossing the lesser evil, going to send a `0` in the field and add a comment to the + // docs notifying about this as a known issue to be fixed in v2. + + //fee: source.compute_fee_with_defaults()?, + fee: 0, + size: (source.transaction.raw_cbor().len() + source.witness.raw_cbor().len()) as u32, + input_count: input_records.len(), + output_count: output_records.len(), + total_output: output_records.iter().map(|o| o.amount).sum(), + ..Default::default() + }; + + if self.config.include_transaction_details { + record.inputs = input_records.into(); + record.outputs = output_records.into(); + } + + Ok(record) + } + + pub fn collect_byron_tx_records( + &self, + block: &byron::MintedBlock, + ) -> Result, Error> { + block + .body + .tx_payload + .iter() + .map(|tx| { + let tx_hash = tx.transaction.original_hash().to_hex(); + self.to_byron_transaction_record(tx, &tx_hash) + }) + .collect() + } + + fn crawl_byron_transaction( + &self, + source: &byron::MintedTxPayload, + tx_hash: &str, + ) -> Result<(), Error> { + let record = self.to_byron_transaction_record(source, tx_hash)?; + + self.append_from(record.clone())?; + + for (idx, input) in source.transaction.inputs.iter().enumerate() { + let child = self.child_writer(EventContext { + input_idx: Some(idx), + ..EventContext::default() + }); + + if let Some(record) = self.to_byron_input_record(input) { + child.append_from(record)?; + } + } + + for (idx, output) in source.transaction.outputs.iter().enumerate() { + let child = self.child_writer(EventContext { + output_idx: Some(idx), + ..EventContext::default() + }); + + if let Ok(record) = self.to_byron_output_record(output) { + child.append_from(record)?; + } + } + + if self.config.include_transaction_end_events { + self.append(EventData::TransactionEnd(record))?; + } + + Ok(()) + } + + pub fn to_byron_block_record( + &self, + source: &byron::MintedBlock, + hash: &Hash<32>, + cbor: &[u8], + ) -> Result { + let abs_slot = pallas::ledger::traverse::time::byron_epoch_slot_to_absolute( + source.header.consensus_data.0.epoch, + source.header.consensus_data.0.slot, + ); + + let mut record = BlockRecord { + era: Era::Byron, + body_size: cbor.len(), + issuer_vkey: source.header.consensus_data.1.to_hex(), + vrf_vkey: Default::default(), + tx_count: source.body.tx_payload.len(), + hash: hash.to_hex(), + number: source.header.consensus_data.2[0], + slot: abs_slot, + epoch: Some(source.header.consensus_data.0.epoch), + epoch_slot: Some(source.header.consensus_data.0.slot), + previous_hash: source.header.prev_block.to_hex(), + cbor_hex: match self.config.include_block_cbor { + true => hex::encode(cbor).into(), + false => None, + }, + transactions: None, + }; + + if self.config.include_block_details { + record.transactions = Some(self.collect_byron_tx_records(source)?); + } + + Ok(record) + } + + fn crawl_byron_main_block( + &self, + block: &byron::MintedBlock, + hash: &Hash<32>, + cbor: &[u8], + ) -> Result<(), Error> { + let record = self.to_byron_block_record(block, hash, cbor)?; + + self.append(EventData::Block(record.clone()))?; + + for (idx, tx) in block.body.tx_payload.iter().enumerate() { + let tx_hash = tx.transaction.original_hash().to_hex(); + + let child = self.child_writer(EventContext { + tx_idx: Some(idx), + tx_hash: Some(tx_hash.to_owned()), + ..EventContext::default() + }); + + child.crawl_byron_transaction(tx, &tx_hash)?; + } + + if self.config.include_block_end_events { + self.append(EventData::BlockEnd(record))?; + } + + Ok(()) + } + + pub fn to_byron_epoch_boundary_record( + &self, + source: &byron::MintedEbBlock, + hash: &Hash<32>, + cbor: &[u8], + ) -> Result { + let abs_slot = pallas::ledger::traverse::time::byron_epoch_slot_to_absolute( + source.header.consensus_data.epoch_id, + 0, + ); + + Ok(BlockRecord { + era: Era::Byron, + body_size: cbor.len(), + hash: hash.to_hex(), + issuer_vkey: Default::default(), + vrf_vkey: Default::default(), + tx_count: 0, + number: source.header.consensus_data.difficulty[0], + slot: abs_slot, + epoch: Some(source.header.consensus_data.epoch_id), + epoch_slot: Some(0), + previous_hash: source.header.prev_block.to_hex(), + cbor_hex: match self.config.include_block_cbor { + true => hex::encode(cbor).into(), + false => None, + }, + transactions: None, + }) + } + + fn crawl_byron_ebb_block( + &self, + block: &byron::MintedEbBlock, + hash: &Hash<32>, + cbor: &[u8], + ) -> Result<(), Error> { + let record = self.to_byron_epoch_boundary_record(block, hash, cbor)?; + + self.append_from(record.clone())?; + + if self.config.include_block_end_events { + self.append(EventData::BlockEnd(record))?; + } + + Ok(()) + } + + /// Mapper entry-point for decoded Byron blocks + /// + /// Entry-point to start crawling a blocks for events. Meant to be used when + /// we already have a decoded block (for example, N2C). The raw CBOR is also + /// passed through in case we need to attach it to outbound events. + pub fn crawl_byron_with_cbor( + &self, + block: &byron::MintedBlock, + cbor: &[u8], + ) -> Result<(), Error> { + let hash = block.header.original_hash(); + + let abs_slot = pallas::ledger::traverse::time::byron_epoch_slot_to_absolute( + block.header.consensus_data.0.epoch, + block.header.consensus_data.0.slot, + ); + + let child = self.child_writer(EventContext { + block_hash: Some(hex::encode(hash)), + block_number: Some(block.header.consensus_data.2[0]), + slot: Some(abs_slot), + timestamp: self.compute_timestamp(abs_slot), + ..EventContext::default() + }); + + child.crawl_byron_main_block(block, &hash, cbor)?; + + Ok(()) + } + + /// Mapper entry-point for raw Byron cbor blocks + /// + /// Entry-point to start crawling a blocks for events. Meant to be used when + /// we haven't decoded the CBOR yet (for example, N2N). + pub fn crawl_from_byron_cbor(&self, cbor: &[u8]) -> Result<(), Error> { + let (_, block): (u16, byron::MintedBlock) = pallas::codec::minicbor::decode(cbor)?; + self.crawl_byron_with_cbor(&block, cbor) + } + + /// Mapper entry-point for decoded Byron Epoch-Boundary blocks + /// + /// Entry-point to start crawling a blocks for events. Meant to be used when + /// we already have a decoded block (for example, N2C). The raw CBOR is also + /// passed through in case we need to attach it to outbound events. + pub fn crawl_ebb_with_cbor( + &self, + block: &byron::MintedEbBlock, + cbor: &[u8], + ) -> Result<(), Error> { + if self.config.include_byron_ebb { + let hash = block.header.original_hash(); + + let abs_slot = pallas::ledger::traverse::time::byron_epoch_slot_to_absolute( + block.header.consensus_data.epoch_id, + 0, + ); + + let child = self.child_writer(EventContext { + block_hash: Some(hex::encode(hash)), + block_number: Some(block.header.consensus_data.difficulty[0]), + slot: Some(abs_slot), + timestamp: self.compute_timestamp(abs_slot), + ..EventContext::default() + }); + + child.crawl_byron_ebb_block(block, &hash, cbor)?; + } + + Ok(()) + } + + /// Mapper entry-point for raw EBB cbor blocks + /// + /// Entry-point to start crawling a blocks for events. Meant to be used when + /// we haven't decoded the CBOR yet (for example, N2N). + pub fn crawl_from_ebb_cbor(&self, cbor: &[u8]) -> Result<(), Error> { + let (_, block): (u16, byron::MintedEbBlock) = pallas::codec::minicbor::decode(cbor)?; + self.crawl_ebb_with_cbor(&block, cbor) + } +} diff --git a/src/mapper/cip15.rs b/src/mapper/cip15.rs new file mode 100644 index 00000000..1d269206 --- /dev/null +++ b/src/mapper/cip15.rs @@ -0,0 +1,57 @@ +use super::EventWriter; +use crate::model::CIP15AssetRecord; +use crate::Error; +use serde_json::Value as JsonValue; + +use pallas::ledger::primitives::alonzo::Metadatum; + +fn extract_json_property<'a>( + json: &'a JsonValue, + key: &str, +) -> Result, Error> { + let result = json + .as_object() + .ok_or_else(|| Error::from("invalid metadatum object for CIP15"))? + .get(key); + + Ok(result) +} + +fn extract_json_string_property(json: &JsonValue, key: &str) -> Result, Error> { + let result = extract_json_property(json, key)? + .and_then(|x| x.as_str()) + .map(|x| x.to_string()); + + Ok(result) +} + +fn extract_json_int_property(json: &JsonValue, key: &str) -> Result, Error> { + let result = extract_json_property(json, key)?.and_then(|x| x.as_i64()); + + Ok(result) +} + +impl EventWriter { + fn to_cip15_asset_record(&self, content: &Metadatum) -> Result { + let raw_json = self.to_metadatum_json(content)?; + + Ok(CIP15AssetRecord { + voting_key: extract_json_string_property(&raw_json, "1")? + .ok_or_else(|| Error::from("invalid value type for CIP15"))?, + stake_pub: extract_json_string_property(&raw_json, "2")? + .ok_or_else(|| Error::from("invalid value type for CIP15"))?, + reward_address: extract_json_string_property(&raw_json, "3")?.unwrap_or_default(), + nonce: extract_json_int_property(&raw_json, "4")?.unwrap_or_default(), + raw_json, + }) + } + + pub(crate) fn crawl_metadata_label_61284(&self, content: &Metadatum) -> Result<(), Error> { + match self.to_cip15_asset_record(content) { + Ok(record) => self.append_from(record)?, + Err(err) => log::info!("error parsing CIP15: {:?}", err), + } + + Ok(()) + } +} diff --git a/src/filters/legacy_v1/cip25.rs b/src/mapper/cip25.rs similarity index 85% rename from src/filters/legacy_v1/cip25.rs rename to src/mapper/cip25.rs index 9f865d80..9def5b1a 100644 --- a/src/filters/legacy_v1/cip25.rs +++ b/src/mapper/cip25.rs @@ -1,8 +1,8 @@ -use gasket::framework::WorkerError; -use pallas::ledger::primitives::alonzo::Metadatum; use serde_json::Value as JsonValue; -use crate::framework::legacy_v1::CIP25AssetRecord; +use pallas::ledger::primitives::alonzo::Metadatum; + +use crate::{model::CIP25AssetRecord, Error}; use super::EventWriter; @@ -34,7 +34,7 @@ fn extract_json_property(json: &JsonValue, key: &str) -> Option { .map(|x| x.to_string()) } -impl EventWriter<'_> { +impl EventWriter { fn search_cip25_version(&self, content_721: &Metadatum) -> Option { match content_721 { Metadatum::Map(entries) => entries.iter().find_map(|(key, value)| match key { @@ -54,10 +54,10 @@ impl EventWriter<'_> { policy: &str, asset: &str, content: &Metadatum, - ) -> CIP25AssetRecord { - let raw_json = self.to_metadatum_json(content); + ) -> Result { + let raw_json = self.to_metadatum_json(content)?; - CIP25AssetRecord { + Ok(CIP25AssetRecord { policy: policy.to_string(), asset: asset.to_string(), version: version.to_string(), @@ -66,19 +66,20 @@ impl EventWriter<'_> { image: extract_json_property(&raw_json, "image"), description: extract_json_property(&raw_json, "description"), raw_json, - } + }) } fn crawl_721_policy( - &mut self, + &self, version: &str, policy: &str, content: &Metadatum, - ) -> Result<(), WorkerError> { + ) -> Result<(), Error> { if let Metadatum::Map(entries) = content { for (key, sub_content) in entries.iter() { if let Some(asset) = is_asset_key(key) { - let record = self.to_cip25_asset_record(version, policy, &asset, sub_content); + let record = + self.to_cip25_asset_record(version, policy, &asset, sub_content)?; self.append_from(record)?; } } @@ -89,10 +90,7 @@ impl EventWriter<'_> { Ok(()) } - pub(crate) fn crawl_metadata_label_721( - &mut self, - content: &Metadatum, - ) -> Result<(), WorkerError> { + pub(crate) fn crawl_metadata_label_721(&self, content: &Metadatum) -> Result<(), Error> { let version = self .search_cip25_version(content) .unwrap_or_else(|| "1.0".to_string()); diff --git a/src/mapper/collect.rs b/src/mapper/collect.rs new file mode 100644 index 00000000..ed448f19 --- /dev/null +++ b/src/mapper/collect.rs @@ -0,0 +1,297 @@ +use std::option::IntoIter; +use pallas::ledger::primitives::alonzo::{AddrKeyhash, AddrKeyhashes, Certificates, RequiredSigners, TransactionInputs}; +use pallas::{ + codec::utils::{KeepRaw, KeyValuePairs, MaybeIndefArray}, + ledger::{ + primitives::{ + alonzo::{ + AuxiliaryData, Coin, MintedBlock, Multiasset, NativeScript, PlutusData, + PlutusScript, Redeemer, RewardAccount, TransactionInput, VKeyWitness, Value, + }, + babbage::{ + LegacyTransactionOutput, MintedPostAlonzoTransactionOutput, + MintedTransactionOutput, PlutusV2Script, + }, + }, + traverse::OriginalHash, + }, +}; +use pallas::ledger::primitives::babbage::{KeepRawPlutusDatas, NativeScripts, PlutusV1Scripts, VKeyWitnesses}; + +use crate::model::{CertificateRecord, RequiredSignerRecord}; +use crate::{ + model::{ + MetadataRecord, MintRecord, NativeWitnessRecord, OutputAssetRecord, PlutusDatumRecord, + PlutusRedeemerRecord, PlutusWitnessRecord, TransactionRecord, TxInputRecord, + TxOutputRecord, VKeyWitnessRecord, WithdrawalRecord, + }, + Error, +}; + +use super::{map::ToHex, EventWriter}; + +impl EventWriter { + pub fn collect_input_records(&self, source: &TransactionInputs) -> Vec { + source + .iter() + .map(|i| self.to_transaction_input_record(i)) + .collect() + } + + pub fn collect_legacy_output_records( + &self, + source: &[LegacyTransactionOutput], + ) -> Result, Error> { + source + .iter() + .map(|i| self.to_legacy_output_record(i)) + .collect() + } + + pub fn collect_post_alonzo_output_records( + &self, + source: &[MintedPostAlonzoTransactionOutput], + ) -> Result, Error> { + source + .iter() + .map(|i| self.to_post_alonzo_output_record(i)) + .collect() + } + + pub fn collect_any_output_records( + &self, + source: &[MintedTransactionOutput], + ) -> Result, Error> { + source + .iter() + .map(|x| match x { + MintedTransactionOutput::Legacy(x) => self.to_legacy_output_record(x), + MintedTransactionOutput::PostAlonzo(x) => self.to_post_alonzo_output_record(x), + }) + .collect() + } + + pub fn collect_asset_records(&self, amount: &Value) -> Vec { + match amount { + Value::Coin(_) => vec![], + Value::Multiasset(_, policies) => policies + .iter() + .flat_map(|(policy, assets)| { + assets.iter().map(|(asset, amount)| { + self.to_transaction_output_asset_record(policy, asset, *amount) + }) + }) + .collect(), + } + } + + pub fn collect_mint_records(&self, mint: &Multiasset) -> Vec { + mint.iter() + .flat_map(|(policy, assets)| { + assets + .iter() + .map(|(asset, amount)| self.to_mint_record(policy, asset, *amount)) + }) + .collect() + } + + pub fn collect_certificate_records( + &self, + certificates: &Certificates, + ) -> Vec { + certificates + .iter() + .map(|cert| self.to_certificate_record(cert)) + .collect() + } + + pub fn collect_withdrawal_records( + &self, + withdrawls: &KeyValuePairs, + ) -> Vec { + withdrawls + .iter() + .map(|(reward_account, coin)| WithdrawalRecord { + reward_account: { + let hex = reward_account.to_hex(); + hex.strip_prefix("e1").map(|x| x.to_string()).unwrap_or(hex) + }, + coin: *coin, + }) + .collect() + } + + pub fn collect_metadata_records( + &self, + aux_data: &AuxiliaryData, + ) -> Result, Error> { + let metadata = match aux_data { + AuxiliaryData::PostAlonzo(data) => data.metadata.as_deref(), + AuxiliaryData::Shelley(data) => Some(data.as_ref()), + AuxiliaryData::ShelleyMa(data) => Some(data.transaction_metadata.as_ref()), + }; + + match metadata { + Some(x) => x + .iter() + .map(|(label, content)| self.to_metadata_record(label, content)) + .collect(), + None => Ok(vec![]), + } + } + + pub fn collect_vkey_witness_records( + &self, + witness_set: &Option>, + ) -> Result, Error> { + match witness_set { + Some(all) => all.iter().map(|i| self.to_vkey_witness_record(i)).collect(), + None => Ok(vec![]), + } + } + + pub fn collect_vkey_witness_records_babbage( + &self, + witness_set: &Option, + ) -> Result, Error> { + match witness_set { + Some(all) => all.iter().map(|i| self.to_vkey_witness_record(i)).collect(), + None => Ok(vec![]), + } + } + + pub fn collect_native_witness_records( + &self, + witness_set: &Option>, + ) -> Result, Error> { + match witness_set { + Some(all) => all + .iter() + .map(|i| self.to_native_witness_record(i)) + .collect(), + None => Ok(vec![]), + } + } + + pub fn collect_native_witness_records_babbage( + &self, + witness_set: &Option, + ) -> Result, Error> { + match witness_set { + Some(all) => all + .iter() + .map(|i| self.to_native_witness_record(i)) + .collect(), + None => Ok(vec![]), + } + } + + pub fn collect_plutus_v1_witness_records( + &self, + witness_set: &Option>, + ) -> Result, Error> { + match &witness_set { + Some(all) => all + .iter() + .map(|i| self.to_plutus_v1_witness_record(i)) + .collect(), + None => Ok(vec![]), + } + } + + pub fn collect_plutus_v1_witness_records_babbage( + &self, + witness_set: &Option, + ) -> Result, Error> { + match &witness_set { + Some(all) => all + .iter() + .map(|i| self.to_plutus_v1_witness_record(i)) + .collect(), + None => Ok(vec![]), + } + } + + pub fn collect_plutus_v2_witness_records( + &self, + witness_set: &Option>, + ) -> Result, Error> { + match &witness_set { + Some(all) => all + .iter() + .map(|i| self.to_plutus_v2_witness_record(i)) + .collect(), + None => Ok(vec![]), + } + } + + pub fn collect_plutus_redeemer_records( + &self, + witness_set: &Option>, + ) -> Result, Error> { + match &witness_set { + Some(all) => all + .iter() + .map(|i| self.to_plutus_redeemer_record(i)) + .collect(), + None => Ok(vec![]), + } + } + + pub fn collect_witness_plutus_datum_records( + &self, + witness_set: &Option>>, + ) -> Result, Error> { + match &witness_set { + Some(all) => all.iter().map(|i| self.to_plutus_datum_record(i)).collect(), + None => Ok(vec![]), + } + } + + pub fn collect_witness_plutus_datum_records_babbage( + &self, + witness_set: &Option, + ) -> Result, Error> { + match &witness_set { + Some(all) => all.into_iter().map(|i| self.to_plutus_datum_record(i)).collect(), + None => Ok(vec![]), + } + } + + pub fn collect_shelley_tx_records( + &self, + block: &MintedBlock, + ) -> Result, Error> { + block + .transaction_bodies + .iter() + .enumerate() + .map(|(idx, tx)| { + let aux_data = block + .auxiliary_data_set + .iter() + .find(|(k, _)| *k == (idx as u32)) + .map(|(_, v)| v); + + let witness_set = block.transaction_witness_sets.get(idx); + + let tx_hash = tx.original_hash().to_hex(); + + self.to_transaction_record(tx, &tx_hash, aux_data, witness_set) + }) + .collect() + } + + pub fn collect_required_signers_records( + &self, + req_signers: &Vec, + ) -> Result, Error> { + let mut signers = vec![]; + for req_sign in req_signers { + let hex = req_sign.to_hex(); + signers.push(RequiredSignerRecord(hex)); + } + + Ok(signers) + } +} diff --git a/src/mapper/map.rs b/src/mapper/map.rs new file mode 100644 index 00000000..c76d8355 --- /dev/null +++ b/src/mapper/map.rs @@ -0,0 +1,904 @@ +use std::collections::HashMap; + +use pallas::ledger::primitives::alonzo::{ + CostMdls, CostModel, ExUnits, Language, MintedWitnessSet, Nonce, NonceVariant, + PositiveInterval, ProtocolParamUpdate, RationalNumber, UnitInterval, Update, +}; +use pallas::ledger::primitives::babbage::{MintedDatumOption, Script, ScriptRef}; +use pallas::ledger::traverse::{ComputeHash, OriginalHash}; +use pallas::{codec::utils::KeepRaw, crypto::hash::Hash}; + +use pallas::ledger::primitives::{ + alonzo::{ + self as alonzo, AuxiliaryData, Certificate, InstantaneousRewardSource, + InstantaneousRewardTarget, Metadatum, MetadatumLabel, MintedBlock, NetworkId, Relay, + TransactionBody, TransactionInput, Value, + }, + babbage, ToCanonicalJson, +}; + +use pallas::network::miniprotocols::Point; +use serde_json::{json, Value as JsonValue}; + +use crate::model::{ + AnchorRecord, AuthCommitteeHotCertRecord, BlockRecord, CertificateRecord, CostModelRecord, + CostModelsRecord, DRep, Era, EventData, ExUnitsRecord, GenesisKeyDelegationRecord, + LanguageVersionRecord, MetadataRecord, MetadatumRendition, MintRecord, + MoveInstantaneousRewardsCertRecord, NativeWitnessRecord, NonceRecord, NonceVariantRecord, + OutputAssetRecord, PlutusDatumRecord, PlutusRedeemerRecord, PlutusWitnessRecord, + PoolRegistrationRecord, PoolRetirementRecord, PositiveIntervalRecord, + ProtocolParamUpdateRecord, RationalNumberRecord, RegCertRecord, RegDRepCertRecord, + ResignCommitteeColdCertRecord, ScriptRefRecord, StakeCredential, StakeDelegationRecord, + StakeDeregistrationRecord, StakeRegDelegCertRecord, StakeRegistrationRecord, + StakeVoteDelegCertRecord, StakeVoteRegDelegCertRecord, TransactionRecord, TxInputRecord, + TxOutputRecord, UnRegCertRecord, UnRegDRepCertRecord, UnitIntervalRecord, UpdateDRepCertRecord, + UpdateRecord, VKeyWitnessRecord, VoteDelegCertRecord, VoteRegDelegCertRecord, +}; + +use crate::model::ScriptRefRecord::{NativeScript, PlutusV1, PlutusV2, PlutusV3}; +use crate::utils::time::TimeProvider; +use crate::Error; + +use super::EventWriter; + +pub trait ToHex { + fn to_hex(&self) -> String; +} + +impl ToHex for Vec { + fn to_hex(&self) -> String { + hex::encode(self) + } +} + +impl ToHex for &[u8] { + fn to_hex(&self) -> String { + hex::encode(self) + } +} + +impl ToHex for Hash { + fn to_hex(&self) -> String { + hex::encode(self) + } +} + +impl From<&alonzo::StakeCredential> for StakeCredential { + fn from(other: &alonzo::StakeCredential) -> Self { + match other { + alonzo::StakeCredential::AddrKeyhash(x) => StakeCredential::AddrKeyhash(x.to_hex()), + alonzo::StakeCredential::Scripthash(x) => StakeCredential::Scripthash(x.to_hex()), + } + } +} + +impl From<&alonzo::DRep> for DRep { + fn from(other: &alonzo::DRep) -> Self { + match other { + alonzo::DRep::Key(x) => DRep::KeyHash(x.to_hex()), + alonzo::DRep::Script(x) => DRep::ScriptHash(x.to_hex()), + alonzo::DRep::Abstain => DRep::Abstain, + alonzo::DRep::NoConfidence => DRep::NoConfidence, + } + } +} + +impl From<&alonzo::Anchor> for AnchorRecord { + fn from(other: &alonzo::Anchor) -> Self { + AnchorRecord { + url: other.0.clone(), + data_hash: other.1.to_hex(), + } + } +} + +fn to_option_anchor_record(anchor: &Option) -> Option { + match anchor { + Some(anchor) => Some(anchor.into()), + None => None, + } +} + +fn ip_string_from_bytes(bytes: &[u8]) -> String { + format!("{}.{}.{}.{}", bytes[0], bytes[1], bytes[2], bytes[3]) +} + +fn relay_to_string(relay: &Relay) -> String { + match relay { + Relay::SingleHostAddr(port, ipv4, ipv6) => { + let ip = match (ipv6, ipv4) { + (None, None) => "".to_string(), + (_, Some(x)) => ip_string_from_bytes(x.as_ref()), + (Some(x), _) => ip_string_from_bytes(x.as_ref()), + }; + + match port { + Some(port) => format!("{ip}:{port}"), + None => ip, + } + } + Relay::SingleHostName(port, host) => match port { + Some(port) => format!("{host}:{port}"), + None => host.clone(), + }, + Relay::MultiHostName(host) => host.clone(), + } +} + +fn metadatum_to_string_key(datum: &Metadatum) -> String { + match datum { + Metadatum::Int(x) => x.to_string(), + Metadatum::Bytes(x) => hex::encode(x.as_slice()), + Metadatum::Text(x) => x.to_owned(), + x => { + log::warn!("unexpected metadatum type for label: {:?}", x); + Default::default() + } + } +} + +fn get_tx_output_coin_value(amount: &Value) -> u64 { + match amount { + Value::Coin(x) => *x, + Value::Multiasset(x, _) => *x, + } +} + +impl EventWriter { + pub fn to_metadatum_json_map_entry( + &self, + pair: (&Metadatum, &Metadatum), + ) -> Result<(String, JsonValue), Error> { + let key = metadatum_to_string_key(pair.0); + let value = self.to_metadatum_json(pair.1)?; + Ok((key, value)) + } + + pub fn to_metadatum_json(&self, source: &Metadatum) -> Result { + match source { + Metadatum::Int(x) => Ok(json!(i128::from(*x))), + Metadatum::Bytes(x) => Ok(json!(hex::encode(x.as_slice()))), + Metadatum::Text(x) => Ok(json!(x)), + Metadatum::Array(x) => { + let items: Result, _> = + x.iter().map(|x| self.to_metadatum_json(x)).collect(); + + Ok(json!(items?)) + } + Metadatum::Map(x) => { + let map: Result, _> = x + .iter() + .map(|(key, value)| self.to_metadatum_json_map_entry((key, value))) + .collect(); + + Ok(json!(map?)) + } + } + } + + pub fn to_metadata_record( + &self, + label: &MetadatumLabel, + value: &Metadatum, + ) -> Result { + let data = MetadataRecord { + label: label.to_string(), + content: match value { + Metadatum::Int(x) => MetadatumRendition::IntScalar(i128::from(*x)), + Metadatum::Bytes(x) => MetadatumRendition::BytesHex(hex::encode(x.as_slice())), + Metadatum::Text(x) => MetadatumRendition::TextScalar(x.clone()), + Metadatum::Array(_) => { + MetadatumRendition::ArrayJson(self.to_metadatum_json(value)?) + } + Metadatum::Map(_) => MetadatumRendition::MapJson(self.to_metadatum_json(value)?), + }, + }; + + Ok(data) + } + + pub fn to_transaction_input_record(&self, input: &TransactionInput) -> TxInputRecord { + TxInputRecord { + tx_id: input.transaction_id.to_hex(), + index: input.index, + } + } + + pub fn to_legacy_output_record( + &self, + output: &alonzo::TransactionOutput, + ) -> Result { + let address = pallas::ledger::addresses::Address::from_bytes(&output.address)?; + + Ok(TxOutputRecord { + address: address.to_string(), + amount: get_tx_output_coin_value(&output.amount), + assets: self.collect_asset_records(&output.amount).into(), + datum_hash: output.datum_hash.map(|hash| hash.to_string()), + inline_datum: None, + inlined_script: None, + }) + } + + pub fn to_post_alonzo_output_record( + &self, + output: &babbage::MintedPostAlonzoTransactionOutput, + ) -> Result { + let address = pallas::ledger::addresses::Address::from_bytes(&output.address)?; + + Ok(TxOutputRecord { + address: address.to_string(), + amount: get_tx_output_coin_value(&output.value), + assets: self.collect_asset_records(&output.value).into(), + datum_hash: match &output.datum_option { + Some(MintedDatumOption::Hash(x)) => Some(x.to_string()), + Some(MintedDatumOption::Data(x)) => Some(x.original_hash().to_hex()), + None => None, + }, + inline_datum: match &output.datum_option { + Some(MintedDatumOption::Data(x)) => Some(self.to_plutus_datum_record(x)?), + _ => None, + }, + inlined_script: match &output.script_ref { + Some(script) => Some(self.to_script_ref_record(script)?), + None => None, + }, + }) + } + + pub fn to_transaction_output_asset_record( + &self, + policy: &Hash<28>, + asset: &pallas::codec::utils::Bytes, + amount: u64, + ) -> OutputAssetRecord { + OutputAssetRecord { + policy: policy.to_hex(), + asset: asset.to_hex(), + asset_ascii: String::from_utf8(asset.to_vec()).ok(), + amount, + } + } + + pub fn to_mint_record( + &self, + policy: &Hash<28>, + asset: &pallas::codec::utils::Bytes, + quantity: i64, + ) -> MintRecord { + MintRecord { + policy: policy.to_hex(), + asset: asset.to_hex(), + quantity, + } + } + + pub fn to_aux_native_script_event(&self, script: &alonzo::NativeScript) -> EventData { + EventData::NativeScript { + policy_id: script.compute_hash().to_hex(), + script: script.to_json(), + } + } + + pub fn to_aux_plutus_script_event(&self, script: &alonzo::PlutusScript) -> EventData { + EventData::PlutusScript { + hash: script.compute_hash().to_hex(), + data: script.0.to_hex(), + } + } + + pub fn to_plutus_redeemer_record( + &self, + redeemer: &alonzo::Redeemer, + ) -> Result { + Ok(PlutusRedeemerRecord { + purpose: match redeemer.tag { + alonzo::RedeemerTag::Spend => "spend".to_string(), + alonzo::RedeemerTag::Mint => "mint".to_string(), + alonzo::RedeemerTag::Cert => "cert".to_string(), + alonzo::RedeemerTag::Reward => "reward".to_string(), + }, + ex_units_mem: redeemer.ex_units.mem, + ex_units_steps: redeemer.ex_units.steps, + input_idx: redeemer.index, + plutus_data: redeemer.data.to_json(), + }) + } + + pub fn to_plutus_datum_record( + &self, + datum: &KeepRaw<'_, alonzo::PlutusData>, + ) -> Result { + Ok(PlutusDatumRecord { + datum_hash: datum.original_hash().to_hex(), + plutus_data: datum.to_json(), + }) + } + + pub fn to_plutus_v1_witness_record( + &self, + script: &alonzo::PlutusScript, + ) -> Result { + Ok(PlutusWitnessRecord { + script_hash: script.compute_hash().to_hex(), + script_hex: script.as_ref().to_hex(), + }) + } + + pub fn to_plutus_v2_witness_record( + &self, + script: &babbage::PlutusV2Script, + ) -> Result { + Ok(PlutusWitnessRecord { + script_hash: script.compute_hash().to_hex(), + script_hex: script.as_ref().to_hex(), + }) + } + + pub fn to_native_witness_record( + &self, + script: &alonzo::NativeScript, + ) -> Result { + Ok(NativeWitnessRecord { + policy_id: script.compute_hash().to_hex(), + script_json: script.to_json(), + }) + } + + pub fn to_script_ref_record( + &self, + script_ref: &ScriptRef, + ) -> Result { + match &script_ref.0 { + Script::PlutusV1Script(script) => Ok(PlutusV1 { + script_hash: script.compute_hash().to_hex(), + script_hex: script.as_ref().to_hex(), + }), + Script::PlutusV2Script(script) => Ok(PlutusV2 { + script_hash: script.compute_hash().to_hex(), + script_hex: script.as_ref().to_hex(), + }), + Script::PlutusV3Script(script) => Ok(PlutusV3 { + script_hash: script.compute_hash().to_hex(), + script_hex: script.as_ref().to_hex(), + }), + Script::NativeScript(script) => Ok(NativeScript { + policy_id: script.compute_hash().to_hex(), + script_json: script.to_json(), + }), + } + } + + pub fn to_vkey_witness_record( + &self, + witness: &alonzo::VKeyWitness, + ) -> Result { + Ok(VKeyWitnessRecord { + vkey_hex: witness.vkey.to_hex(), + signature_hex: witness.signature.to_hex(), + }) + } + + pub fn to_certificate_record(&self, certificate: &Certificate) -> CertificateRecord { + match certificate { + Certificate::StakeRegistration(credential) => { + CertificateRecord::StakeRegistration(StakeRegistrationRecord { + credential: credential.into(), + }) + } + Certificate::StakeDeregistration(credential) => { + CertificateRecord::StakeDeregistration(StakeDeregistrationRecord { + credential: credential.into(), + }) + } + Certificate::StakeDelegation(credential, pool) => { + CertificateRecord::StakeDelegation(StakeDelegationRecord { + credential: credential.into(), + pool_hash: pool.to_hex(), + }) + } + Certificate::PoolRegistration { + operator, + vrf_keyhash, + pledge, + cost, + margin, + reward_account, + pool_owners, + relays, + pool_metadata, + } => CertificateRecord::PoolRegistration(PoolRegistrationRecord { + operator: operator.to_hex(), + vrf_keyhash: vrf_keyhash.to_hex(), + pledge: *pledge, + cost: *cost, + margin: self.to_rational_number_record(margin), + reward_account: reward_account.to_hex(), + pool_owners: pool_owners.iter().map(|p| p.to_hex()).collect(), + relays: relays.iter().map(relay_to_string).collect(), + pool_metadata: pool_metadata.as_ref().map(|m| m.url.clone()), + pool_metadata_hash: pool_metadata.as_ref().map(|m| m.hash.clone().to_hex()), + }), + Certificate::PoolRetirement(pool, epoch) => { + CertificateRecord::PoolRetirement(PoolRetirementRecord { + pool: pool.to_hex(), + epoch: *epoch, + }) + } + Certificate::MoveInstantaneousRewardsCert(move_) => { + CertificateRecord::MoveInstantaneousRewardsCert( + MoveInstantaneousRewardsCertRecord { + from_reserves: matches!(move_.source, InstantaneousRewardSource::Reserves), + from_treasury: matches!(move_.source, InstantaneousRewardSource::Treasury), + to_stake_credentials: match &move_.target { + InstantaneousRewardTarget::StakeCredentials(creds) => { + let x = creds.iter().map(|(k, v)| (k.into(), *v)).collect(); + Some(x) + } + _ => None, + }, + to_other_pot: match move_.target { + InstantaneousRewardTarget::OtherAccountingPot(x) => Some(x), + _ => None, + }, + }, + ) + } + Certificate::GenesisKeyDelegation( + genesis_hash, + genesis_delegate_hash, + vrf_key_hash, + ) => CertificateRecord::GenesisKeyDelegation(GenesisKeyDelegationRecord { + genesis_hash: genesis_hash.to_hex(), + genesis_delegate_hash: genesis_delegate_hash.to_hex(), + vrf_key_hash: vrf_key_hash.to_hex(), + }), + Certificate::Reg(credential, coin) => CertificateRecord::RegCert(RegCertRecord { + credential: credential.into(), + coin: *coin, + }), + Certificate::UnReg(credential, coin) => CertificateRecord::UnRegCert(UnRegCertRecord { + credential: credential.into(), + coin: *coin, + }), + Certificate::VoteDeleg(credential, drep) => { + CertificateRecord::VoteDeleg(VoteDelegCertRecord { + credential: credential.into(), + drep: drep.into(), + }) + } + Certificate::StakeVoteDeleg(credential, pool, drep) => { + CertificateRecord::StakeVoteDeleg(StakeVoteDelegCertRecord { + credential: credential.into(), + pool_keyhash: pool.to_hex(), + drep: drep.into(), + }) + } + Certificate::StakeRegDeleg(credential, pool, coin) => { + CertificateRecord::StakeRegDeleg(StakeRegDelegCertRecord { + credential: credential.into(), + pool_keyhash: pool.to_hex(), + coin: *coin, + }) + } + Certificate::VoteRegDeleg(credential, drep, coin) => { + CertificateRecord::VoteRegDeleg(VoteRegDelegCertRecord { + credential: credential.into(), + drep: drep.into(), + coin: *coin, + }) + } + Certificate::StakeVoteRegDeleg(credential, pool, drep, coin) => { + CertificateRecord::StakeVoteRegDeleg(StakeVoteRegDelegCertRecord { + credential: credential.into(), + pool_keyhash: pool.to_hex(), + drep: drep.into(), + coin: *coin, + }) + } + Certificate::AuthCommitteeHot(cold, hot) => { + CertificateRecord::AuthCommitteeHot(AuthCommitteeHotCertRecord { + committee_cold_credential: cold.into(), + committee_hot_credential: hot.into(), + }) + } + Certificate::ResignCommitteeCold(cold, anchor) => { + CertificateRecord::ResignCommitteeCold(ResignCommitteeColdCertRecord { + committee_cold_credential: cold.into(), + anchor: to_option_anchor_record(anchor), + }) + } + Certificate::RegDRepCert(drep, coin, anchor) => { + CertificateRecord::RegDRepCert(RegDRepCertRecord { + credential: drep.into(), + coin: *coin, + anchor: to_option_anchor_record(anchor), + }) + } + Certificate::UnRegDRepCert(drep, coin) => { + CertificateRecord::UnRegDRepCert(UnRegDRepCertRecord { + credential: drep.into(), + coin: *coin, + }) + } + Certificate::UpdateDRepCert(credential, anchor) => { + CertificateRecord::UpdateDRepCert(UpdateDRepCertRecord { + credential: credential.into(), + anchor: to_option_anchor_record(anchor), + }) + } + } + } + + pub fn to_rational_number_record(&self, rational: &RationalNumber) -> RationalNumberRecord { + RationalNumberRecord { + numerator: rational.numerator, + denominator: rational.denominator, + } + } + + pub fn to_rational_number_record_option( + &self, + rational: &Option, + ) -> Option { + match rational { + Some(rational) => Some(self.to_rational_number_record(rational)), + None => None, + } + } + + pub fn to_unit_interval_record( + &self, + interval: &Option, + ) -> Option { + match interval { + Some(interval) => Some(UnitIntervalRecord( + interval.numerator as u64, + interval.denominator, + )), + None => None, + } + } + + pub fn to_positive_interval_record( + &self, + interval: &PositiveInterval, + ) -> PositiveIntervalRecord { + PositiveIntervalRecord(interval.numerator as u64, interval.denominator) + } + + pub fn to_nonce_record(&self, nonce: &Option) -> Option { + match nonce { + Some(nonce) => Some(NonceRecord { + variant: self.to_nonce_variant_record(&nonce.variant), + hash: nonce.hash.map(|x| x.to_hex()), + }), + None => None, + } + } + + pub fn to_cost_models_record( + &self, + cost_models: &Option, + ) -> Option { + match cost_models { + Some(cost_models) => { + let mut cost_models_record = HashMap::new(); + for cost_model_pair in cost_models.clone().to_vec() { + let language_version_record = + self.to_language_version_record(&cost_model_pair.0); + let cost_model_record = self.to_cost_model_record(cost_model_pair.1); + cost_models_record.insert(language_version_record, cost_model_record); + } + Some(CostModelsRecord(cost_models_record)) + } + None => None, + } + } + + pub fn to_language_version_record(&self, language_version: &Language) -> LanguageVersionRecord { + match language_version { + Language::PlutusV1 => LanguageVersionRecord::PlutusV1, + } + } + + pub fn to_cost_model_record(&self, cost_model: CostModel) -> CostModelRecord { + CostModelRecord(cost_model) + } + + pub fn to_nonce_variant_record(&self, nonce_variant: &NonceVariant) -> NonceVariantRecord { + match nonce_variant { + NonceVariant::NeutralNonce => NonceVariantRecord::NeutralNonce, + NonceVariant::Nonce => NonceVariantRecord::Nonce, + } + } + + pub fn to_ex_units_record(&self, ex_units: &Option) -> Option { + match ex_units { + Some(ex_units) => Some(ExUnitsRecord { + mem: ex_units.mem, + steps: ex_units.steps, + }), + None => None, + } + } + + pub fn to_certificate_event(&self, certificate: &Certificate) -> EventData { + let certificate_record = self.to_certificate_record(certificate); + match certificate_record { + CertificateRecord::StakeRegistration(cert_record) => { + EventData::StakeRegistration(cert_record) + } + CertificateRecord::StakeDeregistration(cert_record) => { + EventData::StakeDeregistration(cert_record) + } + CertificateRecord::StakeDelegation(cert_record) => { + EventData::StakeDelegation(cert_record) + } + CertificateRecord::PoolRegistration(cert_record) => { + EventData::PoolRegistration(cert_record) + } + CertificateRecord::PoolRetirement(cert_record) => { + EventData::PoolRetirement(cert_record) + } + CertificateRecord::MoveInstantaneousRewardsCert(cert_record) => { + EventData::MoveInstantaneousRewardsCert(cert_record) + } + CertificateRecord::GenesisKeyDelegation(cert_record) => { + EventData::GenesisKeyDelegation(cert_record) + } + CertificateRecord::RegCert(cert_record) => EventData::RegCert(cert_record), + CertificateRecord::UnRegCert(cert_record) => EventData::UnRegCert(cert_record), + CertificateRecord::VoteDeleg(cert_record) => EventData::VoteDeleg(cert_record), + CertificateRecord::StakeVoteDeleg(cert_record) => { + EventData::StakeVoteDeleg(cert_record) + } + CertificateRecord::StakeRegDeleg(cert_record) => EventData::StakeRegDeleg(cert_record), + CertificateRecord::VoteRegDeleg(cert_record) => EventData::VoteRegDeleg(cert_record), + CertificateRecord::StakeVoteRegDeleg(cert_record) => { + EventData::StakeVoteRegDeleg(cert_record) + } + CertificateRecord::AuthCommitteeHot(cert_record) => { + EventData::AuthCommitteeHot(cert_record) + } + CertificateRecord::ResignCommitteeCold(cert_record) => { + EventData::ResignCommitteeCold(cert_record) + } + CertificateRecord::RegDRepCert(cert_record) => EventData::RegDRepCert(cert_record), + CertificateRecord::UnRegDRepCert(cert_record) => EventData::UnRegDRepCert(cert_record), + CertificateRecord::UpdateDRepCert(cert_record) => { + EventData::UpdateDRepCert(cert_record) + } + } + } + + pub fn to_collateral_event(&self, collateral: &TransactionInput) -> EventData { + EventData::Collateral { + tx_id: collateral.transaction_id.to_hex(), + index: collateral.index, + } + } + + pub fn to_tx_size( + &self, + body: &KeepRaw, + aux_data: Option<&KeepRaw>, + witness_set: Option<&KeepRaw>, + ) -> usize { + body.raw_cbor().len() + + aux_data.map(|ax| ax.raw_cbor().len()).unwrap_or(2) + + witness_set.map(|ws| ws.raw_cbor().len()).unwrap_or(1) + } + + pub fn to_transaction_record( + &self, + body: &KeepRaw, + tx_hash: &str, + aux_data: Option<&KeepRaw>, + witness_set: Option<&KeepRaw>, + ) -> Result { + let mut record = TransactionRecord { + hash: tx_hash.to_owned(), + size: self.to_tx_size(body, aux_data, witness_set) as u32, + fee: body.fee, + ttl: body.ttl, + validity_interval_start: body.validity_interval_start, + network_id: body.network_id.as_ref().map(|x| match x { + NetworkId::One => 1, + NetworkId::Two => 2, + }), + ..TransactionRecord::default() + }; + + let outputs = self.collect_legacy_output_records(&body.outputs)?; + record.output_count = outputs.len(); + record.total_output = outputs.iter().map(|o| o.amount).sum(); + + let inputs = self.collect_input_records(&body.inputs); + record.input_count = inputs.len(); + + if let Some(mint) = &body.mint { + let mints = self.collect_mint_records(mint); + record.mint_count = mints.len(); + + if self.config.include_transaction_details { + record.mint = mints.into(); + } + } + + if let Some(certs) = &body.certificates { + let certs = self.collect_certificate_records(certs); + record.certificate_count = certs.len(); + + if self.config.include_transaction_details { + record.certs = certs.into(); + } + } + + if let Some(update) = &body.update { + if self.config.include_transaction_details { + record.update = Some(self.to_update_record(update)); + } + } + + if let Some(req_signers) = &body.required_signers { + let req_signers = self.collect_required_signers_records(req_signers)?; + record.required_signers_count = req_signers.len(); + + if self.config.include_transaction_details { + record.required_signers = Some(req_signers); + } + } + + // TODO + // TransactionBodyComponent::ScriptDataHash(_) + // TransactionBodyComponent::AuxiliaryDataHash(_) + + if self.config.include_transaction_details { + record.outputs = outputs.into(); + record.inputs = inputs.into(); + + record.metadata = match aux_data { + Some(aux_data) => self.collect_metadata_records(aux_data)?.into(), + None => None, + }; + + if let Some(witnesses) = witness_set { + record.vkey_witnesses = self + .collect_vkey_witness_records(&witnesses.vkeywitness)? + .into(); + + record.native_witnesses = self + .collect_native_witness_records(&witnesses.native_script)? + .into(); + + record.plutus_witnesses = self + .collect_plutus_v1_witness_records(&witnesses.plutus_script)? + .into(); + + record.plutus_redeemers = self + .collect_plutus_redeemer_records(&witnesses.redeemer)? + .into(); + + record.plutus_data = self + .collect_witness_plutus_datum_records(&witnesses.plutus_data)? + .into(); + } + + if let Some(withdrawals) = &body.withdrawals { + record.withdrawals = self.collect_withdrawal_records(withdrawals).into(); + } + } + + Ok(record) + } + + pub fn to_block_record( + &self, + source: &MintedBlock, + hash: &Hash<32>, + cbor: &[u8], + era: Era, + ) -> Result { + let relative_epoch = self + .utils + .time + .as_ref() + .map(|time| time.absolute_slot_to_relative(source.header.header_body.slot)); + + let mut record = BlockRecord { + era, + body_size: source.header.header_body.block_body_size as usize, + issuer_vkey: source.header.header_body.issuer_vkey.to_hex(), + vrf_vkey: source.header.header_body.vrf_vkey.to_hex(), + tx_count: source.transaction_bodies.len(), + hash: hex::encode(hash), + number: source.header.header_body.block_number, + slot: source.header.header_body.slot, + epoch: relative_epoch.map(|(epoch, _)| epoch), + epoch_slot: relative_epoch.map(|(_, epoch_slot)| epoch_slot), + previous_hash: source + .header + .header_body + .prev_hash + .map(hex::encode) + .unwrap_or_default(), + cbor_hex: match self.config.include_block_cbor { + true => hex::encode(cbor).into(), + false => None, + }, + transactions: None, + }; + + if self.config.include_block_details { + record.transactions = Some(self.collect_shelley_tx_records(source)?); + } + + Ok(record) + } + + pub fn to_protocol_update_record( + &self, + update: &ProtocolParamUpdate, + ) -> ProtocolParamUpdateRecord { + ProtocolParamUpdateRecord { + minfee_a: update.minfee_a, + minfee_b: update.minfee_b, + max_block_body_size: update.max_block_body_size, + max_transaction_size: update.max_transaction_size, + max_block_header_size: update.max_block_header_size, + key_deposit: update.key_deposit, + pool_deposit: update.pool_deposit, + maximum_epoch: update.maximum_epoch, + desired_number_of_stake_pools: update.desired_number_of_stake_pools, + pool_pledge_influence: self + .to_rational_number_record_option(&update.pool_pledge_influence), + expansion_rate: self.to_unit_interval_record(&update.expansion_rate), + treasury_growth_rate: self.to_unit_interval_record(&update.treasury_growth_rate), + decentralization_constant: self + .to_unit_interval_record(&update.decentralization_constant), + extra_entropy: self.to_nonce_record(&update.extra_entropy), + protocol_version: update.protocol_version, + min_pool_cost: update.min_pool_cost, + ada_per_utxo_byte: update.ada_per_utxo_byte, + cost_models_for_script_languages: self + .to_cost_models_record(&update.cost_models_for_script_languages), + execution_costs: match &update.execution_costs { + Some(execution_costs) => Some(json!(execution_costs)), + None => None, + }, + max_tx_ex_units: self.to_ex_units_record(&update.max_tx_ex_units), + max_block_ex_units: self.to_ex_units_record(&update.max_block_ex_units), + max_value_size: update.max_value_size, + collateral_percentage: update.collateral_percentage, + max_collateral_inputs: update.max_collateral_inputs, + } + } + + pub fn to_update_record(&self, update: &Update) -> UpdateRecord { + let mut updates = HashMap::new(); + for update in update.proposed_protocol_parameter_updates.clone().to_vec() { + updates.insert(update.0.to_hex(), self.to_protocol_update_record(&update.1)); + } + + UpdateRecord { + proposed_protocol_parameter_updates: updates, + epoch: update.epoch, + } + } + + pub(crate) fn append_rollback_event(&self, point: &Point) -> Result<(), Error> { + let data = match point { + Point::Origin => EventData::RollBack { + block_slot: 0, + block_hash: "".to_string(), + }, + Point::Specific(slot, hash) => EventData::RollBack { + block_slot: *slot, + block_hash: hex::encode(hash), + }, + }; + + self.append(data) + } +} diff --git a/src/mapper/mod.rs b/src/mapper/mod.rs new file mode 100644 index 00000000..a176e40f --- /dev/null +++ b/src/mapper/mod.rs @@ -0,0 +1,10 @@ +mod babbage; +mod byron; +mod cip15; +mod cip25; +mod collect; +mod map; +mod prelude; +mod shelley; + +pub use prelude::*; diff --git a/src/mapper/prelude.rs b/src/mapper/prelude.rs new file mode 100644 index 00000000..95cd7bf4 --- /dev/null +++ b/src/mapper/prelude.rs @@ -0,0 +1,122 @@ +use std::sync::Arc; + +use crate::{ + model::{Era, Event, EventContext, EventData}, + pipelining::StageSender, + utils::{time::TimeProvider, Utils}, +}; + +use merge::Merge; +use serde::Deserialize; + +use crate::Error; + +#[deprecated] +pub use crate::utils::ChainWellKnownInfo; + +#[derive(Deserialize, Clone, Debug, Default)] +pub struct Config { + #[serde(default)] + pub include_block_end_events: bool, + + #[serde(default)] + pub include_transaction_details: bool, + + #[serde(default)] + pub include_transaction_end_events: bool, + + #[serde(default)] + pub include_block_details: bool, + + #[serde(default)] + pub include_block_cbor: bool, + + #[serde(default)] + pub include_byron_ebb: bool, +} + +#[derive(Clone)] +pub struct EventWriter { + context: EventContext, + output: StageSender, + pub(crate) config: Config, + pub(crate) utils: Arc, +} + +impl EventWriter { + pub fn new(output: StageSender, utils: Arc, config: Config) -> Self { + EventWriter { + context: EventContext::default(), + output, + utils, + config, + } + } + + #[allow(unused)] + pub fn standalone( + output: StageSender, + well_known: Option, + config: Config, + ) -> Self { + let utils = Arc::new(Utils::new(well_known.unwrap_or_default())); + + Self::new(output, utils, config) + } + + pub fn append(&self, data: EventData) -> Result<(), Error> { + let evt = Event { + context: self.context.clone(), + data, + fingerprint: None, + }; + + self.utils.track_source_progress(&evt); + + self.output + .send(evt) + .expect("error sending event through output stage, pipeline must have crashed."); + + Ok(()) + } + + pub fn append_from(&self, source: T) -> Result<(), Error> + where + T: Into, + { + self.append(source.into()) + } + + pub fn child_writer(&self, mut extra_context: EventContext) -> EventWriter { + extra_context.merge(self.context.clone()); + + EventWriter { + context: extra_context, + output: self.output.clone(), + utils: self.utils.clone(), + config: self.config.clone(), + } + } + + pub fn compute_timestamp(&self, slot: u64) -> Option { + match &self.utils.time { + Some(provider) => provider.slot_to_wallclock(slot).into(), + _ => None, + } + } +} + +impl From for Era { + fn from(other: pallas::ledger::traverse::Era) -> Self { + match other { + pallas::ledger::traverse::Era::Byron => Era::Byron, + pallas::ledger::traverse::Era::Shelley => Era::Shelley, + pallas::ledger::traverse::Era::Allegra => Era::Allegra, + pallas::ledger::traverse::Era::Mary => Era::Mary, + pallas::ledger::traverse::Era::Alonzo => Era::Alonzo, + pallas::ledger::traverse::Era::Babbage => Era::Babbage, + pallas::ledger::traverse::Era::Conway => Era::Babbage, + _ => Era::Unknown, + } + } +} diff --git a/src/mapper/shelley.rs b/src/mapper/shelley.rs new file mode 100644 index 00000000..57464eb1 --- /dev/null +++ b/src/mapper/shelley.rs @@ -0,0 +1,331 @@ +use pallas::codec::utils::KeepRaw; + +use pallas::ledger::primitives::alonzo::{ + AuxiliaryData, Certificate, Metadata, MintedBlock, MintedWitnessSet, Multiasset, + TransactionBody, TransactionInput, TransactionOutput, Value, +}; + +use pallas::crypto::hash::Hash; +use pallas::ledger::traverse::OriginalHash; + +use crate::{ + model::{Era, EventContext, EventData}, + Error, +}; + +use super::{map::ToHex, EventWriter}; + +impl EventWriter { + pub(crate) fn crawl_metadata(&self, metadata: &Metadata) -> Result<(), Error> { + for (label, content) in metadata.iter() { + let record = self.to_metadata_record(label, content)?; + self.append_from(record)?; + + match label { + 721u64 => self.crawl_metadata_label_721(content)?, + 61284u64 => self.crawl_metadata_label_61284(content)?, + _ => (), + } + } + + Ok(()) + } + + pub(crate) fn crawl_auxdata(&self, aux_data: &AuxiliaryData) -> Result<(), Error> { + match aux_data { + AuxiliaryData::PostAlonzo(data) => { + if let Some(metadata) = &data.metadata { + self.crawl_metadata(metadata)?; + } + + if let Some(native) = &data.native_scripts { + for script in native.iter() { + self.append(self.to_aux_native_script_event(script))?; + } + } + + if let Some(plutus) = &data.plutus_scripts { + for script in plutus.iter() { + self.append(self.to_aux_plutus_script_event(script))?; + } + } + } + AuxiliaryData::Shelley(data) => { + self.crawl_metadata(data)?; + } + AuxiliaryData::ShelleyMa(data) => { + self.crawl_metadata(&data.transaction_metadata)?; + + if let Some(native) = &data.auxiliary_scripts { + for script in native.iter() { + self.append(self.to_aux_native_script_event(script))?; + } + } + } + } + + Ok(()) + } + + pub(crate) fn crawl_transaction_input(&self, input: &TransactionInput) -> Result<(), Error> { + self.append_from(self.to_transaction_input_record(input)) + } + + pub(crate) fn crawl_transaction_output_amount(&self, amount: &Value) -> Result<(), Error> { + if let Value::Multiasset(_, policies) = amount { + for (policy, assets) in policies.iter() { + for (asset, amount) in assets.iter() { + self.append_from( + self.to_transaction_output_asset_record(policy, asset, *amount), + )?; + } + } + } + + Ok(()) + } + + pub(crate) fn crawl_legacy_output(&self, output: &TransactionOutput) -> Result<(), Error> { + let record = self.to_legacy_output_record(output)?; + self.append(record.into())?; + + let address = pallas::ledger::addresses::Address::from_bytes(&output.address)?; + + let child = &self.child_writer(EventContext { + output_address: address.to_string().into(), + ..EventContext::default() + }); + + child.crawl_transaction_output_amount(&output.amount)?; + + Ok(()) + } + + pub(crate) fn crawl_certificate(&self, certificate: &Certificate) -> Result<(), Error> { + self.append(self.to_certificate_event(certificate)) + + // more complex event goes here (eg: pool metadata?) + } + + pub(crate) fn crawl_collateral(&self, collateral: &TransactionInput) -> Result<(), Error> { + self.append(self.to_collateral_event(collateral)) + + // TODO: should we have a collateral idx in context? + // more complex event goes here (eg: ???) + } + + pub(crate) fn crawl_mints(&self, mints: &Multiasset) -> Result<(), Error> { + // should we have a policy context? + + for (policy, assets) in mints.iter() { + for (asset, quantity) in assets.iter() { + self.append_from(self.to_mint_record(policy, asset, *quantity))?; + } + } + + Ok(()) + } + + pub(crate) fn crawl_witness_set( + &self, + witness_set: &KeepRaw, + ) -> Result<(), Error> { + if let Some(native) = &witness_set.native_script { + for script in native.iter() { + self.append_from(self.to_native_witness_record(script)?)?; + } + } + + if let Some(plutus) = &witness_set.plutus_script { + for script in plutus.iter() { + self.append_from(self.to_plutus_v1_witness_record(script)?)?; + } + } + + if let Some(redeemers) = &witness_set.redeemer { + for redeemer in redeemers.iter() { + self.append_from(self.to_plutus_redeemer_record(redeemer)?)?; + } + } + + if let Some(datums) = &witness_set.plutus_data { + for datum in datums.iter() { + self.append_from(self.to_plutus_datum_record(datum)?)?; + } + } + + Ok(()) + } + + fn crawl_shelley_transaction( + &self, + tx: &KeepRaw, + tx_hash: &str, + aux_data: Option<&KeepRaw>, + witness_set: Option<&KeepRaw>, + ) -> Result<(), Error> { + let record = self.to_transaction_record(tx, tx_hash, aux_data, witness_set)?; + + self.append_from(record.clone())?; + + for (idx, input) in tx.inputs.iter().enumerate() { + let child = self.child_writer(EventContext { + input_idx: Some(idx), + ..EventContext::default() + }); + + child.crawl_transaction_input(input)?; + } + + for (idx, output) in tx.outputs.iter().enumerate() { + let child = self.child_writer(EventContext { + output_idx: Some(idx), + ..EventContext::default() + }); + + child.crawl_legacy_output(output)?; + } + + if let Some(certs) = &tx.certificates { + for (idx, cert) in certs.iter().enumerate() { + let child = self.child_writer(EventContext { + certificate_idx: Some(idx), + ..EventContext::default() + }); + + child.crawl_certificate(cert)?; + } + } + + if let Some(collateral) = &tx.collateral { + for (_idx, collateral) in collateral.iter().enumerate() { + // TODO: collateral context? + + self.crawl_collateral(collateral)?; + } + } + + if let Some(mint) = &tx.mint { + self.crawl_mints(mint)?; + } + + if let Some(aux_data) = aux_data { + self.crawl_auxdata(aux_data)?; + } + + if let Some(witness_set) = witness_set { + self.crawl_witness_set(witness_set)?; + } + + if self.config.include_transaction_end_events { + self.append(EventData::TransactionEnd(record))?; + } + + Ok(()) + } + + fn crawl_shelley_block( + &self, + block: &MintedBlock, + hash: &Hash<32>, + cbor: &[u8], + era: Era, + ) -> Result<(), Error> { + let record = self.to_block_record(block, hash, cbor, era)?; + + self.append(EventData::Block(record.clone()))?; + + for (idx, tx) in block.transaction_bodies.iter().enumerate() { + let aux_data = block + .auxiliary_data_set + .iter() + .find(|(k, _)| *k == (idx as u32)) + .map(|(_, v)| v); + + let witness_set = block.transaction_witness_sets.get(idx); + + let tx_hash = tx.original_hash().to_hex(); + + let child = self.child_writer(EventContext { + tx_idx: Some(idx), + tx_hash: Some(tx_hash.to_owned()), + ..EventContext::default() + }); + + child.crawl_shelley_transaction(tx, &tx_hash, aux_data, witness_set)?; + } + + if self.config.include_block_end_events { + self.append(EventData::BlockEnd(record))?; + } + + Ok(()) + } + + #[deprecated(note = "use crawl_from_shelley_cbor instead")] + pub fn crawl_with_cbor(&self, block: &MintedBlock, cbor: &[u8]) -> Result<(), Error> { + let hash = block.header.original_hash(); + + let child = self.child_writer(EventContext { + block_hash: Some(hex::encode(hash)), + block_number: Some(block.header.header_body.block_number), + slot: Some(block.header.header_body.slot), + timestamp: self.compute_timestamp(block.header.header_body.slot), + ..EventContext::default() + }); + + child.crawl_shelley_block(block, &hash, cbor, Era::Undefined) + } + + #[deprecated(note = "use crawl_from_shelley_cbor instead")] + pub fn crawl(&self, block: &MintedBlock) -> Result<(), Error> { + let hash = block.header.original_hash(); + + let child = self.child_writer(EventContext { + block_hash: Some(hex::encode(hash)), + block_number: Some(block.header.header_body.block_number), + slot: Some(block.header.header_body.slot), + timestamp: self.compute_timestamp(block.header.header_body.slot), + ..EventContext::default() + }); + + child.crawl_shelley_block(block, &hash, &[], Era::Undefined) + } + + /// Mapper entry-point for decoded Shelley blocks + /// + /// Entry-point to start crawling a blocks for events. Meant to be used when + /// we already have a decoded block (for example, N2C). The raw CBOR is also + /// passed through in case we need to attach it to outbound events. + pub fn crawl_shelley_with_cbor<'b>( + &self, + block: &'b MintedBlock<'b>, + cbor: &'b [u8], + era: Era, + ) -> Result<(), Error> { + let hash = block.header.original_hash(); + + let child = self.child_writer(EventContext { + block_hash: Some(hex::encode(hash)), + block_number: Some(block.header.header_body.block_number), + slot: Some(block.header.header_body.slot), + timestamp: self.compute_timestamp(block.header.header_body.slot), + ..EventContext::default() + }); + + child.crawl_shelley_block(block, &hash, cbor, era) + } + + /// Mapper entry-point for raw Shelley cbor blocks + /// + /// Entry-point to start crawling a blocks for events. Meant to be used when + /// we haven't decoded the CBOR yet (for example, N2N). + /// + /// We use Alonzo primitives since they are backward compatible with + /// Shelley. In this way, we can avoid having to fork the crawling procedure + /// for each different hard-fork. + pub fn crawl_from_shelley_cbor(&self, cbor: &[u8], era: Era) -> Result<(), Error> { + let (_, block): (u16, MintedBlock) = pallas::codec::minicbor::decode(cbor)?; + self.crawl_shelley_with_cbor(&block, cbor, era) + } +} diff --git a/src/model.rs b/src/model.rs new file mode 100644 index 00000000..9f96af7b --- /dev/null +++ b/src/model.rs @@ -0,0 +1,641 @@ +use std::collections::HashMap; +use std::fmt::Display; + +use merge::Merge; + +use serde::{Deserialize, Serialize}; +use serde_json::Value as JsonValue; + +use strum_macros::Display; + +// We're duplicating the Era struct from Pallas for two reasons: a) we need it +// to be serializable and we don't want to impose serde dependency on Pallas and +// b) we prefer not to add dependencies to Pallas outside of the sources that +// actually use it on an attempt to make the pipeline agnostic of particular +// implementation details. +#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Display)] +pub enum Era { + Undefined, + Unknown, + Byron, + Shelley, + Allegra, + Mary, + Alonzo, + Babbage, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum MetadatumRendition { + MapJson(JsonValue), + ArrayJson(JsonValue), + IntScalar(i128), + TextScalar(String), + BytesHex(String), +} + +impl Display for MetadatumRendition { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + MetadatumRendition::MapJson(x) => x.fmt(f), + MetadatumRendition::ArrayJson(x) => x.fmt(f), + MetadatumRendition::IntScalar(x) => x.fmt(f), + MetadatumRendition::TextScalar(x) => x.fmt(f), + MetadatumRendition::BytesHex(x) => x.fmt(f), + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct MetadataRecord { + pub label: String, + + #[serde(flatten)] + pub content: MetadatumRendition, +} + +impl From for EventData { + fn from(x: MetadataRecord) -> Self { + EventData::Metadata(x) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct CIP25AssetRecord { + pub version: String, + pub policy: String, + pub asset: String, + pub name: Option, + pub image: Option, + pub media_type: Option, + pub description: Option, + pub raw_json: JsonValue, +} + +impl From for EventData { + fn from(x: CIP25AssetRecord) -> Self { + EventData::CIP25Asset(x) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct CIP15AssetRecord { + pub voting_key: String, + pub stake_pub: String, + pub reward_address: String, + pub nonce: i64, + pub raw_json: JsonValue, +} + +impl From for EventData { + fn from(x: CIP15AssetRecord) -> Self { + EventData::CIP15Asset(x) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct TxInputRecord { + pub tx_id: String, + pub index: u64, +} + +impl From for EventData { + fn from(x: TxInputRecord) -> Self { + EventData::TxInput(x) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct OutputAssetRecord { + pub policy: String, + pub asset: String, + pub asset_ascii: Option, + pub amount: u64, +} + +impl From for EventData { + fn from(x: OutputAssetRecord) -> Self { + EventData::OutputAsset(x) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct TxOutputRecord { + pub address: String, + pub amount: u64, + pub assets: Option>, + pub datum_hash: Option, + pub inline_datum: Option, + pub inlined_script: Option, +} + +impl From for EventData { + fn from(x: TxOutputRecord) -> Self { + EventData::TxOutput(x) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct MintRecord { + pub policy: String, + pub asset: String, + pub quantity: i64, +} + +impl From for EventData { + fn from(x: MintRecord) -> Self { + EventData::Mint(x) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct WithdrawalRecord { + pub reward_account: String, + pub coin: u64, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)] +pub struct TransactionRecord { + pub hash: String, + pub fee: u64, + pub ttl: Option, + pub validity_interval_start: Option, + pub network_id: Option, + pub input_count: usize, + pub collateral_input_count: usize, + pub has_collateral_output: bool, + pub output_count: usize, + pub mint_count: usize, + pub certificate_count: usize, + pub total_output: u64, + pub required_signers_count: usize, + + // include_details + pub required_signers: Option>, + pub update: Option, + pub metadata: Option>, + pub inputs: Option>, + pub outputs: Option>, + pub collateral_inputs: Option>, + pub collateral_output: Option, + pub certs: Option>, + pub mint: Option>, + pub vkey_witnesses: Option>, + pub native_witnesses: Option>, + pub plutus_witnesses: Option>, + pub plutus_redeemers: Option>, + pub plutus_data: Option>, + pub withdrawals: Option>, + pub size: u32, +} + +impl From for EventData { + fn from(x: TransactionRecord) -> Self { + EventData::Transaction(x) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, Merge, Default)] +pub struct EventContext { + pub block_hash: Option, + pub block_number: Option, + pub slot: Option, + pub timestamp: Option, + pub tx_idx: Option, + pub tx_hash: Option, + pub input_idx: Option, + pub output_idx: Option, + pub output_address: Option, + pub certificate_idx: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum StakeCredential { + AddrKeyhash(String), + Scripthash(String), +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub enum ScriptRefRecord { + PlutusV1 { + script_hash: String, + script_hex: String, + }, + PlutusV2 { + script_hash: String, + script_hex: String, + }, + PlutusV3 { + script_hash: String, + script_hex: String, + }, + NativeScript { + policy_id: String, + script_json: JsonValue, + }, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd)] +pub enum CertificateRecord { + StakeRegistration(StakeRegistrationRecord), + StakeDeregistration(StakeDeregistrationRecord), + StakeDelegation(StakeDelegationRecord), + PoolRegistration(PoolRegistrationRecord), + PoolRetirement(PoolRetirementRecord), + GenesisKeyDelegation(GenesisKeyDelegationRecord), + MoveInstantaneousRewardsCert(MoveInstantaneousRewardsCertRecord), + RegCert(RegCertRecord), + UnRegCert(UnRegCertRecord), + VoteDeleg(VoteDelegCertRecord), + StakeVoteDeleg(StakeVoteDelegCertRecord), + StakeRegDeleg(StakeRegDelegCertRecord), + VoteRegDeleg(VoteRegDelegCertRecord), + StakeVoteRegDeleg(StakeVoteRegDelegCertRecord), + AuthCommitteeHot(AuthCommitteeHotCertRecord), + ResignCommitteeCold(ResignCommitteeColdCertRecord), + RegDRepCert(RegDRepCertRecord), + UnRegDRepCert(UnRegDRepCertRecord), + UpdateDRepCert(UpdateDRepCertRecord), +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd)] +pub struct StakeRegistrationRecord { + pub credential: StakeCredential, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd)] +pub struct StakeDeregistrationRecord { + pub credential: StakeCredential, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd)] +pub struct StakeDelegationRecord { + pub credential: StakeCredential, + pub pool_hash: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd)] +pub struct PoolRegistrationRecord { + pub operator: String, + pub vrf_keyhash: String, + pub pledge: u64, + pub cost: u64, + pub margin: RationalNumberRecord, + pub reward_account: String, + pub pool_owners: Vec, + pub relays: Vec, + pub pool_metadata: Option, + pub pool_metadata_hash: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd)] +pub struct PoolRetirementRecord { + pub pool: String, + pub epoch: u64, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd)] +pub struct GenesisKeyDelegationRecord { + pub genesis_hash: String, + pub genesis_delegate_hash: String, + pub vrf_key_hash: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum DRep { + KeyHash(String), + ScriptHash(String), + Abstain, + NoConfidence, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct MoveInstantaneousRewardsCertRecord { + pub from_reserves: bool, + pub from_treasury: bool, + pub to_stake_credentials: Option>, + pub to_other_pot: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct RegCertRecord { + pub credential: StakeCredential, + pub coin: u64, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct UnRegCertRecord { + pub credential: StakeCredential, + pub coin: u64, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct VoteDelegCertRecord { + pub credential: StakeCredential, + pub drep: DRep, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct StakeVoteDelegCertRecord { + pub credential: StakeCredential, + pub pool_keyhash: String, + pub drep: DRep, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct StakeRegDelegCertRecord { + pub credential: StakeCredential, + pub pool_keyhash: String, + pub coin: u64, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct VoteRegDelegCertRecord { + pub credential: StakeCredential, + pub drep: DRep, + pub coin: u64, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct StakeVoteRegDelegCertRecord { + pub credential: StakeCredential, + pub pool_keyhash: String, + pub drep: DRep, + pub coin: u64, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct AuthCommitteeHotCertRecord { + pub committee_cold_credential: StakeCredential, + pub committee_hot_credential: StakeCredential, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct ResignCommitteeColdCertRecord { + pub committee_cold_credential: StakeCredential, + pub anchor: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct RegDRepCertRecord { + pub credential: StakeCredential, + pub coin: u64, + pub anchor: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct UnRegDRepCertRecord { + pub credential: StakeCredential, + pub coin: u64, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct UpdateDRepCertRecord { + pub credential: StakeCredential, + pub anchor: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct AnchorRecord { + pub url: String, + pub data_hash: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct RationalNumberRecord { + pub numerator: u64, + pub denominator: u64, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct UnitIntervalRecord(pub u64, pub u64); + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct PositiveIntervalRecord(pub u64, pub u64); + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct ExUnitsRecord { + pub mem: u32, + pub steps: u64, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct ExUnitPricesRecord { + pub mem_price: PositiveIntervalRecord, + pub step_price: PositiveIntervalRecord, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct NonceRecord { + pub variant: NonceVariantRecord, + pub hash: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum NonceVariantRecord { + NeutralNonce, + Nonce, +} + +#[derive(Serialize, Deserialize, Hash, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum LanguageVersionRecord { + PlutusV1, + PlutusV2, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct CostModelRecord(pub Vec); + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct CostModelsRecord(pub HashMap); + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct VKeyWitnessRecord { + pub vkey_hex: String, + pub signature_hex: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct RequiredSignerRecord(pub String); + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct NativeWitnessRecord { + pub policy_id: String, + pub script_json: JsonValue, +} + +impl From for EventData { + fn from(x: NativeWitnessRecord) -> Self { + EventData::NativeWitness(x) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct PlutusWitnessRecord { + pub script_hash: String, + pub script_hex: String, +} + +impl From for EventData { + fn from(x: PlutusWitnessRecord) -> Self { + EventData::PlutusWitness(x) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct PlutusRedeemerRecord { + pub purpose: String, + pub ex_units_mem: u32, + pub ex_units_steps: u64, + pub input_idx: u32, + pub plutus_data: JsonValue, +} + +impl From for EventData { + fn from(x: PlutusRedeemerRecord) -> Self { + EventData::PlutusRedeemer(x) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct PlutusDatumRecord { + pub datum_hash: String, + pub plutus_data: JsonValue, +} + +impl From for EventData { + fn from(x: PlutusDatumRecord) -> Self { + EventData::PlutusDatum(x) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct BlockRecord { + pub era: Era, + pub epoch: Option, + pub epoch_slot: Option, + pub body_size: usize, + pub issuer_vkey: String, + pub vrf_vkey: String, + pub tx_count: usize, + pub slot: u64, + pub hash: String, + pub number: u64, + pub previous_hash: String, + pub cbor_hex: Option, + pub transactions: Option>, +} + +impl From for EventData { + fn from(x: BlockRecord) -> Self { + EventData::Block(x) + } +} + +#[derive(Serialize, Deserialize, Display, Debug, Clone)] +#[serde(rename_all = "snake_case")] +pub enum EventData { + Block(BlockRecord), + BlockEnd(BlockRecord), + Transaction(TransactionRecord), + TransactionEnd(TransactionRecord), + TxInput(TxInputRecord), + TxOutput(TxOutputRecord), + OutputAsset(OutputAssetRecord), + Metadata(MetadataRecord), + + VKeyWitness(VKeyWitnessRecord), + NativeWitness(NativeWitnessRecord), + PlutusWitness(PlutusWitnessRecord), + PlutusRedeemer(PlutusRedeemerRecord), + PlutusDatum(PlutusDatumRecord), + + #[serde(rename = "cip25_asset")] + CIP25Asset(CIP25AssetRecord), + + #[serde(rename = "cip15_asset")] + CIP15Asset(CIP15AssetRecord), + + Mint(MintRecord), + Collateral { + tx_id: String, + index: u64, + }, + NativeScript { + policy_id: String, + script: JsonValue, + }, + PlutusScript { + hash: String, + data: String, + }, + StakeRegistration(StakeRegistrationRecord), + StakeDeregistration(StakeDeregistrationRecord), + StakeDelegation(StakeDelegationRecord), + PoolRegistration(PoolRegistrationRecord), + PoolRetirement(PoolRetirementRecord), + GenesisKeyDelegation(GenesisKeyDelegationRecord), + MoveInstantaneousRewardsCert(MoveInstantaneousRewardsCertRecord), + RegCert(RegCertRecord), + UnRegCert(UnRegCertRecord), + VoteDeleg(VoteDelegCertRecord), + StakeVoteDeleg(StakeVoteDelegCertRecord), + StakeRegDeleg(StakeRegDelegCertRecord), + VoteRegDeleg(VoteRegDelegCertRecord), + StakeVoteRegDeleg(StakeVoteRegDelegCertRecord), + AuthCommitteeHot(AuthCommitteeHotCertRecord), + ResignCommitteeCold(ResignCommitteeColdCertRecord), + RegDRepCert(RegDRepCertRecord), + UnRegDRepCert(UnRegDRepCertRecord), + UpdateDRepCert(UpdateDRepCertRecord), + + RollBack { + block_slot: u64, + block_hash: String, + }, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct ProtocolParamUpdateRecord { + pub minfee_a: Option, + pub minfee_b: Option, + pub max_block_body_size: Option, + pub max_transaction_size: Option, + pub max_block_header_size: Option, + pub key_deposit: Option, + pub pool_deposit: Option, + pub maximum_epoch: Option, + pub desired_number_of_stake_pools: Option, + pub pool_pledge_influence: Option, + pub expansion_rate: Option, + pub treasury_growth_rate: Option, + pub decentralization_constant: Option, + pub extra_entropy: Option, + pub protocol_version: Option<(u64, u64)>, + pub min_pool_cost: Option, + pub ada_per_utxo_byte: Option, + pub cost_models_for_script_languages: Option, + pub execution_costs: Option, + pub max_tx_ex_units: Option, + pub max_block_ex_units: Option, + pub max_value_size: Option, + pub collateral_percentage: Option, + pub max_collateral_inputs: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct UpdateRecord { + pub proposed_protocol_parameter_updates: HashMap, + pub epoch: u64, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct Event { + pub context: EventContext, + + #[serde(flatten)] + pub data: EventData, + + pub fingerprint: Option, +} diff --git a/src/pipelining.rs b/src/pipelining.rs new file mode 100644 index 00000000..2ef43c6f --- /dev/null +++ b/src/pipelining.rs @@ -0,0 +1,55 @@ +use std::thread::JoinHandle; + +use crate::{model::Event, Error}; + +pub type StageReceiver = std::sync::mpsc::Receiver; + +pub type StageSender = std::sync::mpsc::SyncSender; + +/// The amount of events an inter-stage channel can buffer before blocking +/// +/// If a filter or sink has a consumption rate lower than the rate of event +/// generations from a source, the pending events will buffer in a queue +/// provided by the corresponding mpsc channel implementation. This constant +/// defines the max amount of events that the buffer queue can hold. Once +/// reached, the previous stages in the pipeline will start blockin on 'send'. +/// +/// This value has a direct effect on the amount of memory consumed by the +/// process. The higher the buffer, the higher potential memory consumption. +/// +/// This value has a direct effect on performance. To allow _pipelining_ +/// benefits, stages should be allowed certain degree of flexibility to deal +/// with resource constrains (such as network or cpu). The lower the buffer, the +/// lower degree of flexibility. +const DEFAULT_INTER_STAGE_BUFFER_SIZE: usize = 1000; + +pub type StageChannel = (StageSender, StageReceiver); + +/// Centralizes the implementation details of inter-stage channel creation +/// +/// Concrete channel implementation is subject to change. We're still exploring +/// sync vs unbounded and threaded vs event-loop. Until we have a long-term +/// strategy, it makes sense to have a single place in the codebase that can be +/// used to change from one implementation to the other without incurring on +/// heavy refactoring throughout several files. +/// +/// Sometimes centralization is not such a bad thing :) +pub fn new_inter_stage_channel(buffer_size: Option) -> StageChannel { + std::sync::mpsc::sync_channel(buffer_size.unwrap_or(DEFAULT_INTER_STAGE_BUFFER_SIZE)) +} + +pub type PartialBootstrapResult = Result<(JoinHandle<()>, StageReceiver), Error>; + +pub type BootstrapResult = Result, Error>; + +pub trait SourceProvider { + fn bootstrap(&self) -> PartialBootstrapResult; +} + +pub trait FilterProvider { + fn bootstrap(&self, input: StageReceiver) -> PartialBootstrapResult; +} + +pub trait SinkProvider { + fn bootstrap(&self, input: StageReceiver) -> BootstrapResult; +} diff --git a/src/sinks/_pending/assert/checks.rs b/src/sinks/assert/checks.rs similarity index 100% rename from src/sinks/_pending/assert/checks.rs rename to src/sinks/assert/checks.rs diff --git a/src/sinks/_pending/assert/mod.rs b/src/sinks/assert/mod.rs similarity index 100% rename from src/sinks/_pending/assert/mod.rs rename to src/sinks/assert/mod.rs diff --git a/src/sinks/_pending/assert/prelude.rs b/src/sinks/assert/prelude.rs similarity index 100% rename from src/sinks/_pending/assert/prelude.rs rename to src/sinks/assert/prelude.rs diff --git a/src/sinks/_pending/assert/run.rs b/src/sinks/assert/run.rs similarity index 98% rename from src/sinks/_pending/assert/run.rs rename to src/sinks/assert/run.rs index db5ef6a9..3b3019bc 100644 --- a/src/sinks/_pending/assert/run.rs +++ b/src/sinks/assert/run.rs @@ -1,8 +1,8 @@ use std::sync::Arc; use crate::{ - framework::StageReceiver, model::{Event, EventData}, + pipelining::StageReceiver, utils::Utils, Error, }; diff --git a/src/sinks/_pending/assert/setup.rs b/src/sinks/assert/setup.rs similarity index 91% rename from src/sinks/_pending/assert/setup.rs rename to src/sinks/assert/setup.rs index 648f08e0..898a4a52 100644 --- a/src/sinks/_pending/assert/setup.rs +++ b/src/sinks/assert/setup.rs @@ -1,7 +1,7 @@ use serde::Deserialize; use crate::{ - framework::{BootstrapResult, SinkProvider, StageReceiver}, + pipelining::{BootstrapResult, SinkProvider, StageReceiver}, utils::WithUtils, }; diff --git a/src/sinks/_pending/aws_lambda/mod.rs b/src/sinks/aws_lambda/mod.rs similarity index 100% rename from src/sinks/_pending/aws_lambda/mod.rs rename to src/sinks/aws_lambda/mod.rs diff --git a/src/sinks/_pending/aws_lambda/run.rs b/src/sinks/aws_lambda/run.rs similarity index 94% rename from src/sinks/_pending/aws_lambda/run.rs rename to src/sinks/aws_lambda/run.rs index 5ffd0aba..b243a609 100644 --- a/src/sinks/_pending/aws_lambda/run.rs +++ b/src/sinks/aws_lambda/run.rs @@ -2,7 +2,7 @@ use aws_sdk_lambda::{types::Blob, Client}; use serde_json::json; use std::sync::Arc; -use crate::{framework::StageReceiver, model::Event, utils::Utils, Error}; +use crate::{model::Event, pipelining::StageReceiver, utils::Utils, Error}; async fn invoke_lambda_function( client: Arc, diff --git a/src/sinks/_pending/aws_lambda/setup.rs b/src/sinks/aws_lambda/setup.rs similarity index 95% rename from src/sinks/_pending/aws_lambda/setup.rs rename to src/sinks/aws_lambda/setup.rs index 65a71468..36de1b40 100644 --- a/src/sinks/_pending/aws_lambda/setup.rs +++ b/src/sinks/aws_lambda/setup.rs @@ -2,7 +2,7 @@ use aws_sdk_lambda::{Client, Region, RetryConfig}; use serde::Deserialize; use crate::{ - framework::{BootstrapResult, SinkProvider, StageReceiver}, + pipelining::{BootstrapResult, SinkProvider, StageReceiver}, utils::WithUtils, }; diff --git a/src/sinks/_pending/aws_s3/mod.rs b/src/sinks/aws_s3/mod.rs similarity index 100% rename from src/sinks/_pending/aws_s3/mod.rs rename to src/sinks/aws_s3/mod.rs diff --git a/src/sinks/_pending/aws_s3/run.rs b/src/sinks/aws_s3/run.rs similarity index 93% rename from src/sinks/_pending/aws_s3/run.rs rename to src/sinks/aws_s3/run.rs index 6d9a1502..bc7b99f8 100644 --- a/src/sinks/_pending/aws_s3/run.rs +++ b/src/sinks/aws_s3/run.rs @@ -1,9 +1,10 @@ use aws_sdk_s3::{types::ByteStream, Client}; +use serde_json::json; use std::sync::Arc; use crate::{ - framework::StageReceiver, model::{BlockRecord, EventData}, + pipelining::StageReceiver, utils::Utils, Error, }; @@ -44,6 +45,7 @@ fn define_obj_key(prefix: &str, policy: &Naming, record: &BlockRecord) -> String Naming::Hash => format!("{}{}", prefix, record.hash), Naming::SlotHash => format!("{}{}.{}", prefix, record.slot, record.hash), Naming::BlockHash => format!("{}{}.{}", prefix, record.number, record.hash), + Naming::BlockNumber => format!("{}", record.number), Naming::EpochHash => format!( "{}{}.{}", prefix, @@ -86,6 +88,10 @@ fn define_content(content_type: &ContentType, record: &BlockRecord) -> ByteStrea ByteStream::from(cbor) } ContentType::CborHex => ByteStream::from(hex.as_bytes().to_vec()), + ContentType::Json => { + let json = json!(record).to_string().as_bytes().to_vec(); + ByteStream::from(json) + } } } diff --git a/src/sinks/_pending/aws_s3/setup.rs b/src/sinks/aws_s3/setup.rs similarity index 93% rename from src/sinks/_pending/aws_s3/setup.rs rename to src/sinks/aws_s3/setup.rs index 65c78dab..cb7c0c71 100644 --- a/src/sinks/_pending/aws_s3/setup.rs +++ b/src/sinks/aws_s3/setup.rs @@ -2,7 +2,7 @@ use aws_sdk_s3::{Client, Region, RetryConfig}; use serde::Deserialize; use crate::{ - framework::{BootstrapResult, SinkProvider, StageReceiver}, + pipelining::{BootstrapResult, SinkProvider, StageReceiver}, utils::WithUtils, }; @@ -15,6 +15,7 @@ pub enum Naming { Hash, SlotHash, BlockHash, + BlockNumber, EpochHash, EpochSlotHash, EpochBlockHash, @@ -24,6 +25,7 @@ pub enum Naming { pub enum ContentType { Cbor, CborHex, + Json, } impl From<&ContentType> for String { @@ -31,6 +33,7 @@ impl From<&ContentType> for String { match other { ContentType::Cbor => "application/cbor".to_string(), ContentType::CborHex => "text/plain".to_string(), + ContentType::Json => "application/json".to_string(), } } } diff --git a/src/sinks/_pending/aws_sqs/mod.rs b/src/sinks/aws_sqs/mod.rs similarity index 100% rename from src/sinks/_pending/aws_sqs/mod.rs rename to src/sinks/aws_sqs/mod.rs diff --git a/src/sinks/_pending/aws_sqs/run.rs b/src/sinks/aws_sqs/run.rs similarity index 95% rename from src/sinks/_pending/aws_sqs/run.rs rename to src/sinks/aws_sqs/run.rs index 010f52e5..a8d8f25a 100644 --- a/src/sinks/_pending/aws_sqs/run.rs +++ b/src/sinks/aws_sqs/run.rs @@ -2,7 +2,7 @@ use aws_sdk_sqs::Client; use serde_json::json; use std::sync::Arc; -use crate::{framework::StageReceiver, model::Event, utils::Utils, Error}; +use crate::{model::Event, pipelining::StageReceiver, utils::Utils, Error}; async fn send_sqs_msg( client: Arc, diff --git a/src/sinks/_pending/aws_sqs/setup.rs b/src/sinks/aws_sqs/setup.rs similarity index 96% rename from src/sinks/_pending/aws_sqs/setup.rs rename to src/sinks/aws_sqs/setup.rs index 7c204a97..9982096e 100644 --- a/src/sinks/_pending/aws_sqs/setup.rs +++ b/src/sinks/aws_sqs/setup.rs @@ -2,7 +2,7 @@ use aws_sdk_sqs::{Client, Region, RetryConfig}; use serde::Deserialize; use crate::{ - framework::{BootstrapResult, SinkProvider, StageReceiver}, + pipelining::{BootstrapResult, SinkProvider, StageReceiver}, utils::WithUtils, }; diff --git a/src/sinks/_pending/common/mod.rs b/src/sinks/common/mod.rs similarity index 100% rename from src/sinks/_pending/common/mod.rs rename to src/sinks/common/mod.rs diff --git a/src/sinks/_pending/common/web.rs b/src/sinks/common/web.rs similarity index 98% rename from src/sinks/_pending/common/web.rs rename to src/sinks/common/web.rs index dc7c5001..3939af1d 100644 --- a/src/sinks/_pending/common/web.rs +++ b/src/sinks/common/web.rs @@ -7,8 +7,8 @@ use reqwest::{ use serde::{Deserialize, Serialize}; use crate::{ - framework::StageReceiver, model::Event, + pipelining::StageReceiver, utils::{retry, Utils}, Error, }; diff --git a/src/sinks/_pending/elastic/mod.rs b/src/sinks/elastic/mod.rs similarity index 100% rename from src/sinks/_pending/elastic/mod.rs rename to src/sinks/elastic/mod.rs diff --git a/src/sinks/_pending/elastic/run.rs b/src/sinks/elastic/run.rs similarity index 99% rename from src/sinks/_pending/elastic/run.rs rename to src/sinks/elastic/run.rs index e150cc02..fc60efb4 100644 --- a/src/sinks/_pending/elastic/run.rs +++ b/src/sinks/elastic/run.rs @@ -6,8 +6,8 @@ use std::sync::Arc; use tokio::runtime::Runtime; use crate::{ - framework::StageReceiver, model::Event, + pipelining::StageReceiver, utils::{retry, Utils}, }; diff --git a/src/sinks/_pending/elastic/setup.rs b/src/sinks/elastic/setup.rs similarity index 96% rename from src/sinks/_pending/elastic/setup.rs rename to src/sinks/elastic/setup.rs index 5b9c5436..bd3d246b 100644 --- a/src/sinks/_pending/elastic/setup.rs +++ b/src/sinks/elastic/setup.rs @@ -11,7 +11,7 @@ use elasticsearch::{ use serde::Deserialize; use crate::{ - framework::{BootstrapResult, SinkProvider, StageReceiver}, + pipelining::{BootstrapResult, SinkProvider, StageReceiver}, utils::{retry, WithUtils}, }; diff --git a/src/sinks/filerotate.rs b/src/sinks/filerotate.rs deleted file mode 100644 index 2c771014..00000000 --- a/src/sinks/filerotate.rs +++ /dev/null @@ -1,148 +0,0 @@ -use std::io::Write; -use std::path::PathBuf; - -use file_rotate::compression::Compression; -use file_rotate::suffix::AppendTimestamp; -use file_rotate::suffix::FileLimit; -use file_rotate::ContentLimit; -use file_rotate::FileRotate; -use gasket::framework::*; -use serde::Deserialize; -use serde_json::json; -use serde_json::Value as JsonValue; - -use crate::framework::*; - -pub struct Worker { - writer: FileRotate, -} - -#[async_trait::async_trait(?Send)] -impl gasket::framework::Worker for Worker { - async fn bootstrap(stage: &Stage) -> Result { - let output_path = match &stage.config.output_path { - Some(x) => PathBuf::try_from(x).map_err(Error::config).or_panic()?, - None => stage.current_dir.clone(), - }; - - let suffix_scheme = AppendTimestamp::default(FileLimit::MaxFiles( - stage - .config - .max_total_files - .unwrap_or(DEFAULT_MAX_TOTAL_FILES), - )); - - let content_limit = ContentLimit::BytesSurpassed( - stage - .config - .max_bytes_per_file - .unwrap_or(DEFAULT_MAX_BYTES_PER_FILE), - ); - - let compression = if let Some(true) = stage.config.compress_files { - Compression::OnRotate(2) - } else { - Compression::None - }; - - #[cfg(unix)] - let writer = FileRotate::new(output_path, suffix_scheme, content_limit, compression, None); - - #[cfg(not(unix))] - let writer = FileRotate::new(output_path, suffix_scheme, content_limit, compression); - - Ok(Self { writer }) - } - - async fn schedule( - &mut self, - stage: &mut Stage, - ) -> Result, WorkerError> { - let msg = stage.input.recv().await.or_panic()?; - Ok(WorkSchedule::Unit(msg.payload)) - } - - async fn execute(&mut self, unit: &ChainEvent, stage: &mut Stage) -> Result<(), WorkerError> { - let (point, json) = match unit { - ChainEvent::Apply(point, record) => { - let json = json!({ "event": "apply", "record": JsonValue::from(record.clone()) }); - (point, json) - } - ChainEvent::Undo(point, record) => { - let json = json!({ "event": "undo", "record": JsonValue::from(record.clone()) }); - (point, json) - } - ChainEvent::Reset(point) => { - let json_point = match &point { - pallas::network::miniprotocols::Point::Origin => JsonValue::from("origin"), - pallas::network::miniprotocols::Point::Specific(slot, hash) => { - json!({ "slot": slot, "hash": hex::encode(hash)}) - } - }; - - let json = json!({ "event": "reset", "point": json_point }); - (point, json) - } - }; - - self.writer - .write_all(json.to_string().as_bytes()) - .and_then(|_| self.writer.write_all(b"\n")) - .or_retry()?; - - stage.ops_count.inc(1); - - stage.latest_block.set(point.slot_or_default() as i64); - stage.cursor.add_breadcrumb(point.clone()); - - Ok(()) - } -} - -#[derive(Stage)] -#[stage(name = "filter", unit = "ChainEvent", worker = "Worker")] -pub struct Stage { - config: Config, - current_dir: PathBuf, - cursor: Cursor, - - pub input: MapperInputPort, - - #[metric] - ops_count: gasket::metrics::Counter, - - #[metric] - latest_block: gasket::metrics::Gauge, -} - -#[derive(Debug, Deserialize, Clone)] -pub enum Format { - JSONL, -} - -#[derive(Default, Debug, Deserialize)] -pub struct Config { - pub output_format: Option, - pub output_path: Option, - pub max_bytes_per_file: Option, - pub max_total_files: Option, - pub compress_files: Option, -} - -const DEFAULT_MAX_BYTES_PER_FILE: usize = 50 * 1024 * 1024; -const DEFAULT_MAX_TOTAL_FILES: usize = 200; - -impl Config { - pub fn bootstrapper(self, ctx: &Context) -> Result { - let stage = Stage { - config: self, - current_dir: ctx.current_dir.clone(), - cursor: ctx.cursor.clone(), - ops_count: Default::default(), - latest_block: Default::default(), - input: Default::default(), - }; - - Ok(stage) - } -} diff --git a/src/sinks/_pending/gcp_cloudfunction/mod.rs b/src/sinks/gcp_cloudfunction/mod.rs similarity index 100% rename from src/sinks/_pending/gcp_cloudfunction/mod.rs rename to src/sinks/gcp_cloudfunction/mod.rs diff --git a/src/sinks/_pending/gcp_cloudfunction/setup.rs b/src/sinks/gcp_cloudfunction/setup.rs similarity index 96% rename from src/sinks/_pending/gcp_cloudfunction/setup.rs rename to src/sinks/gcp_cloudfunction/setup.rs index 280ded8d..65e7c8c5 100644 --- a/src/sinks/_pending/gcp_cloudfunction/setup.rs +++ b/src/sinks/gcp_cloudfunction/setup.rs @@ -3,7 +3,7 @@ use std::time::Duration; use serde::Deserialize; use crate::{ - framework::{BootstrapResult, SinkProvider, StageReceiver}, + pipelining::{BootstrapResult, SinkProvider, StageReceiver}, sinks::common::web::{build_headers_map, request_loop, ErrorPolicy, APP_USER_AGENT}, utils::{retry, WithUtils}, }; diff --git a/src/sinks/_pending/gcp_pubsub/mod.rs b/src/sinks/gcp_pubsub/mod.rs similarity index 100% rename from src/sinks/_pending/gcp_pubsub/mod.rs rename to src/sinks/gcp_pubsub/mod.rs diff --git a/src/sinks/_pending/gcp_pubsub/run.rs b/src/sinks/gcp_pubsub/run.rs similarity index 98% rename from src/sinks/_pending/gcp_pubsub/run.rs rename to src/sinks/gcp_pubsub/run.rs index 27826b83..d89bf52d 100644 --- a/src/sinks/_pending/gcp_pubsub/run.rs +++ b/src/sinks/gcp_pubsub/run.rs @@ -9,8 +9,8 @@ use google_cloud_pubsub::{ use serde_json::json; use crate::{ - framework::StageReceiver, model::Event, + pipelining::StageReceiver, sinks::common::web::ErrorPolicy, utils::{retry, Utils}, }; diff --git a/src/sinks/_pending/gcp_pubsub/setup.rs b/src/sinks/gcp_pubsub/setup.rs similarity index 94% rename from src/sinks/_pending/gcp_pubsub/setup.rs rename to src/sinks/gcp_pubsub/setup.rs index 5dd597a6..b27e562a 100644 --- a/src/sinks/_pending/gcp_pubsub/setup.rs +++ b/src/sinks/gcp_pubsub/setup.rs @@ -1,7 +1,7 @@ use serde::Deserialize; use crate::{ - framework::{BootstrapResult, SinkProvider, StageReceiver}, + pipelining::{BootstrapResult, SinkProvider, StageReceiver}, sinks::common::web::ErrorPolicy, utils::{retry, WithUtils}, }; diff --git a/src/sinks/_pending/kafka/mod.rs b/src/sinks/kafka/mod.rs similarity index 100% rename from src/sinks/_pending/kafka/mod.rs rename to src/sinks/kafka/mod.rs diff --git a/src/sinks/_pending/kafka/run.rs b/src/sinks/kafka/run.rs similarity index 94% rename from src/sinks/_pending/kafka/run.rs rename to src/sinks/kafka/run.rs index e2feaf38..5818cddf 100644 --- a/src/sinks/_pending/kafka/run.rs +++ b/src/sinks/kafka/run.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use kafka::producer::{Producer, Record}; -use crate::{framework::StageReceiver, model::Event, utils::Utils, Error}; +use crate::{model::Event, pipelining::StageReceiver, utils::Utils, Error}; use super::PartitionStrategy; diff --git a/src/sinks/_pending/kafka/setup.rs b/src/sinks/kafka/setup.rs similarity index 95% rename from src/sinks/_pending/kafka/setup.rs rename to src/sinks/kafka/setup.rs index 52b6a904..44e3f18b 100644 --- a/src/sinks/_pending/kafka/setup.rs +++ b/src/sinks/kafka/setup.rs @@ -4,7 +4,7 @@ use kafka::{client::RequiredAcks, producer::Producer}; use serde::Deserialize; use crate::{ - framework::{BootstrapResult, SinkProvider, StageReceiver}, + pipelining::{BootstrapResult, SinkProvider, StageReceiver}, utils::WithUtils, }; diff --git a/src/sinks/_pending/logs/mod.rs b/src/sinks/logs/mod.rs similarity index 100% rename from src/sinks/_pending/logs/mod.rs rename to src/sinks/logs/mod.rs diff --git a/src/sinks/_pending/logs/run.rs b/src/sinks/logs/run.rs similarity index 92% rename from src/sinks/_pending/logs/run.rs rename to src/sinks/logs/run.rs index c3b97f53..68546b62 100644 --- a/src/sinks/_pending/logs/run.rs +++ b/src/sinks/logs/run.rs @@ -2,7 +2,7 @@ use std::{io::Write, sync::Arc}; use serde_json::json; -use crate::{framework::StageReceiver, utils::Utils, Error}; +use crate::{pipelining::StageReceiver, utils::Utils, Error}; pub fn jsonl_writer_loop( input: StageReceiver, diff --git a/src/sinks/_pending/logs/setup.rs b/src/sinks/logs/setup.rs similarity index 97% rename from src/sinks/_pending/logs/setup.rs rename to src/sinks/logs/setup.rs index 5076d157..a9ddb79a 100644 --- a/src/sinks/_pending/logs/setup.rs +++ b/src/sinks/logs/setup.rs @@ -7,7 +7,7 @@ use std::{io::Write, path::PathBuf, str::FromStr}; use serde::Deserialize; use crate::{ - framework::{BootstrapResult, SinkProvider, StageReceiver}, + pipelining::{BootstrapResult, SinkProvider, StageReceiver}, utils::WithUtils, Error, }; diff --git a/src/sinks/mod.rs b/src/sinks/mod.rs index 13a549f7..5eec177a 100644 --- a/src/sinks/mod.rs +++ b/src/sinks/mod.rs @@ -1,89 +1,40 @@ -use gasket::{messaging::RecvPort, runtime::Tether}; -use serde::Deserialize; +mod common; -use crate::framework::*; - -//pub mod assert; -//pub mod stdout; -pub mod filerotate; -pub mod noop; +pub mod assert; +pub mod stdout; pub mod terminal; -pub mod webhook; - -// #[cfg(feature = "kafkasink")] -// pub mod kafka; -// #[cfg(feature = "elasticsink")] -// pub mod elastic; +pub use common::*; -// #[cfg(feature = "aws")] -// pub mod aws_sqs; +#[cfg(feature = "logs")] +pub mod logs; -// #[cfg(feature = "aws")] -// pub mod aws_lambda; - -// #[cfg(feature = "aws")] -// pub mod aws_s3; - -// #[cfg(feature = "redissink")] -// pub mod redis; +#[cfg(feature = "webhook")] +pub mod webhook; -// #[cfg(feature = "gcp")] -// pub mod gcp_pubsub; +#[cfg(feature = "kafkasink")] +pub mod kafka; -// #[cfg(feature = "gcp")] -// pub mod gcp_cloudfunction; +#[cfg(feature = "elasticsink")] +pub mod elastic; -// #[cfg(feature = "rabbitmqsink")] -// pub mod rabbitmq; +#[cfg(feature = "aws")] +pub mod aws_sqs; -pub enum Bootstrapper { - Terminal(terminal::Stage), - FileRotate(filerotate::Stage), - WebHook(webhook::Stage), - Noop(noop::Stage), -} +#[cfg(feature = "aws")] +pub mod aws_lambda; -impl StageBootstrapper for Bootstrapper { - fn connect_output(&mut self, _: OutputAdapter) { - panic!("attempted to use sink stage as sender"); - } +#[cfg(feature = "aws")] +pub mod aws_s3; - fn connect_input(&mut self, adapter: InputAdapter) { - match self { - Bootstrapper::Terminal(p) => p.input.connect(adapter), - Bootstrapper::FileRotate(p) => p.input.connect(adapter), - Bootstrapper::WebHook(p) => p.input.connect(adapter), - Bootstrapper::Noop(p) => p.input.connect(adapter), - } - } +#[cfg(feature = "redissink")] +pub mod redis; - fn spawn(self, policy: gasket::runtime::Policy) -> Tether { - match self { - Bootstrapper::Terminal(x) => gasket::runtime::spawn_stage(x, policy), - Bootstrapper::FileRotate(x) => gasket::runtime::spawn_stage(x, policy), - Bootstrapper::WebHook(x) => gasket::runtime::spawn_stage(x, policy), - Bootstrapper::Noop(x) => gasket::runtime::spawn_stage(x, policy), - } - } -} +#[cfg(feature = "gcp")] +pub mod gcp_pubsub; -#[derive(Deserialize)] -#[serde(tag = "type")] -pub enum Config { - Terminal(terminal::Config), - FileRotate(filerotate::Config), - WebHook(webhook::Config), - Noop(noop::Config), -} +#[cfg(feature = "gcp")] +pub mod gcp_cloudfunction; -impl Config { - pub fn bootstrapper(self, ctx: &Context) -> Result { - match self { - Config::Terminal(c) => Ok(Bootstrapper::Terminal(c.bootstrapper(ctx)?)), - Config::FileRotate(c) => Ok(Bootstrapper::FileRotate(c.bootstrapper(ctx)?)), - Config::WebHook(c) => Ok(Bootstrapper::WebHook(c.bootstrapper(ctx)?)), - Config::Noop(c) => Ok(Bootstrapper::Noop(c.bootstrapper(ctx)?)), - } - } -} +#[cfg(feature = "rabbitmqsink")] +pub mod rabbitmq; diff --git a/src/sinks/noop.rs b/src/sinks/noop.rs deleted file mode 100644 index ec7cb55e..00000000 --- a/src/sinks/noop.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! A noop sink used as example and placeholder for other sinks - -use gasket::framework::*; -use pallas::network::miniprotocols::Point; -use serde::Deserialize; -use tracing::debug; - -use crate::framework::*; - -#[derive(Default)] -pub struct Worker; - -#[async_trait::async_trait(?Send)] -impl gasket::framework::Worker for Worker { - async fn bootstrap(_: &Stage) -> Result { - Ok(Self) - } - - async fn schedule(&mut self, stage: &mut Stage) -> Result, WorkerError> { - let msg = stage.input.recv().await.or_panic()?; - - let point = msg.payload.point().clone(); - Ok(WorkSchedule::Unit(point)) - } - - async fn execute(&mut self, unit: &Point, stage: &mut Stage) -> Result<(), WorkerError> { - debug!(?unit, "message received"); - stage.ops_count.inc(1); - - stage.latest_block.set(unit.slot_or_default() as i64); - stage.cursor.add_breadcrumb(unit.clone()); - - Ok(()) - } -} - -#[derive(Stage)] -#[stage(name = "filter", unit = "Point", worker = "Worker")] -pub struct Stage { - cursor: Cursor, - - pub input: FilterInputPort, - - #[metric] - ops_count: gasket::metrics::Counter, - - #[metric] - latest_block: gasket::metrics::Gauge, -} - -#[derive(Default, Deserialize)] -pub struct Config {} - -impl Config { - pub fn bootstrapper(self, ctx: &Context) -> Result { - let stage = Stage { - cursor: ctx.cursor.clone(), - ops_count: Default::default(), - latest_block: Default::default(), - input: Default::default(), - }; - - Ok(stage) - } -} diff --git a/src/sinks/_pending/rabbitmq/mod.rs b/src/sinks/rabbitmq/mod.rs similarity index 100% rename from src/sinks/_pending/rabbitmq/mod.rs rename to src/sinks/rabbitmq/mod.rs diff --git a/src/sinks/_pending/rabbitmq/run.rs b/src/sinks/rabbitmq/run.rs similarity index 98% rename from src/sinks/_pending/rabbitmq/run.rs rename to src/sinks/rabbitmq/run.rs index 3aa8967d..d1162815 100644 --- a/src/sinks/_pending/rabbitmq/run.rs +++ b/src/sinks/rabbitmq/run.rs @@ -4,8 +4,8 @@ use lapin::{options::BasicPublishOptions, BasicProperties, Channel, Connection}; use serde_json::json; use crate::{ - framework::StageReceiver, model::Event, + pipelining::StageReceiver, utils::{retry, Utils}, Error, }; diff --git a/src/sinks/_pending/rabbitmq/setup.rs b/src/sinks/rabbitmq/setup.rs similarity index 95% rename from src/sinks/_pending/rabbitmq/setup.rs rename to src/sinks/rabbitmq/setup.rs index 20047254..b27f1c86 100644 --- a/src/sinks/_pending/rabbitmq/setup.rs +++ b/src/sinks/rabbitmq/setup.rs @@ -2,7 +2,7 @@ use lapin::{Connection, ConnectionProperties}; use serde::Deserialize; use crate::{ - framework::{BootstrapResult, SinkProvider, StageReceiver}, + pipelining::{BootstrapResult, SinkProvider, StageReceiver}, utils::{retry, WithUtils}, }; diff --git a/src/sinks/_pending/redis/mod.rs b/src/sinks/redis/mod.rs similarity index 100% rename from src/sinks/_pending/redis/mod.rs rename to src/sinks/redis/mod.rs diff --git a/src/sinks/_pending/redis/run.rs b/src/sinks/redis/run.rs similarity index 94% rename from src/sinks/_pending/redis/run.rs rename to src/sinks/redis/run.rs index 1bf96dec..4f98b733 100644 --- a/src/sinks/_pending/redis/run.rs +++ b/src/sinks/redis/run.rs @@ -1,5 +1,5 @@ use super::StreamStrategy; -use crate::{framework::StageReceiver, model::Event, utils::Utils, Error}; +use crate::{model::Event, pipelining::StageReceiver, utils::Utils, Error}; use serde_json::json; use std::sync::Arc; diff --git a/src/sinks/_pending/redis/setup.rs b/src/sinks/redis/setup.rs similarity index 95% rename from src/sinks/_pending/redis/setup.rs rename to src/sinks/redis/setup.rs index 229f4259..671c85dd 100644 --- a/src/sinks/_pending/redis/setup.rs +++ b/src/sinks/redis/setup.rs @@ -2,7 +2,7 @@ use redis::Client; use serde::Deserialize; use crate::{ - framework::{BootstrapResult, SinkProvider, StageReceiver}, + pipelining::{BootstrapResult, SinkProvider, StageReceiver}, utils::WithUtils, }; diff --git a/src/sinks/_pending/stdout/mod.rs b/src/sinks/stdout/mod.rs similarity index 100% rename from src/sinks/_pending/stdout/mod.rs rename to src/sinks/stdout/mod.rs diff --git a/src/sinks/_pending/stdout/run.rs b/src/sinks/stdout/run.rs similarity index 91% rename from src/sinks/_pending/stdout/run.rs rename to src/sinks/stdout/run.rs index 1df60dc6..a3fa6b4c 100644 --- a/src/sinks/_pending/stdout/run.rs +++ b/src/sinks/stdout/run.rs @@ -2,7 +2,7 @@ use std::{io::Write, sync::Arc}; use serde_json::json; -use crate::{framework::StageReceiver, utils::Utils, Error}; +use crate::{pipelining::StageReceiver, utils::Utils, Error}; pub fn jsonl_writer_loop( input: StageReceiver, diff --git a/src/sinks/_pending/stdout/setup.rs b/src/sinks/stdout/setup.rs similarity index 93% rename from src/sinks/_pending/stdout/setup.rs rename to src/sinks/stdout/setup.rs index 68e1e8fc..b58d7ba9 100644 --- a/src/sinks/_pending/stdout/setup.rs +++ b/src/sinks/stdout/setup.rs @@ -3,7 +3,7 @@ use std::io::stdout; use serde::Deserialize; use crate::{ - framework::{BootstrapResult, SinkProvider, StageReceiver}, + pipelining::{BootstrapResult, SinkProvider, StageReceiver}, utils::WithUtils, }; diff --git a/src/sinks/terminal/format.rs b/src/sinks/terminal/format.rs index 30b5a437..5f2766cd 100644 --- a/src/sinks/terminal/format.rs +++ b/src/sinks/terminal/format.rs @@ -1,10 +1,17 @@ use std::fmt::{Display, Write}; use crossterm::style::{Attribute, Color, Stylize}; -use pallas::network::miniprotocols::Point; use unicode_truncate::UnicodeTruncateStr; -use crate::{framework::legacy_v1::*, framework::*}; +use crate::{ + model::{ + BlockRecord, CIP15AssetRecord, CIP25AssetRecord, Event, EventData, MetadataRecord, + MintRecord, NativeWitnessRecord, OutputAssetRecord, PlutusDatumRecord, + PlutusRedeemerRecord, PlutusWitnessRecord, TransactionRecord, TxInputRecord, + TxOutputRecord, VKeyWitnessRecord, + }, + utils::Utils, +}; pub struct LogLine { prefix: &'static str, @@ -17,7 +24,7 @@ pub struct LogLine { impl LogLine { fn new_raw( - source: &legacy_v1::Event, + source: &Event, prefix: &'static str, color: Color, max_width: Option, @@ -32,12 +39,10 @@ impl LogLine { block_num: source.context.block_number, } } +} - pub fn new_from_legacy_v1( - source: &legacy_v1::Event, - max_width: Option, - adahandle_policy: &str, - ) -> LogLine { +impl LogLine { + pub fn new(source: &Event, max_width: Option, utils: &Utils) -> LogLine { match &source.data { EventData::Block(BlockRecord { era, @@ -118,7 +123,7 @@ impl LogLine { asset, asset_ascii, .. - }) if policy == adahandle_policy => LogLine::new_raw( + }) if policy == &utils.well_known.adahandle_policy => LogLine::new_raw( source, "$HNDL", Color::DarkGreen, @@ -213,75 +218,143 @@ impl LogLine { max_width, format!("{{ vkey: {vkey_hex} }}"), ), - EventData::StakeRegistration { credential } => LogLine::new_raw( + EventData::StakeRegistration(cert) => LogLine::new_raw( source, "STAKE+", Color::Magenta, max_width, - format!("{{ credential: {credential:?} }}"), + format!("{{ credential: {0:?} }}", cert.credential), ), - EventData::StakeDeregistration { credential } => LogLine::new_raw( + EventData::StakeDeregistration(cert) => LogLine::new_raw( source, "STAKE-", Color::DarkMagenta, max_width, - format!("{{ credential: {credential:?} }}"), + format!("{{ credential: {0:?} }}", cert.credential), ), - EventData::StakeDelegation { - credential, - pool_hash, - } => LogLine::new_raw( + EventData::StakeDelegation(cert) => LogLine::new_raw( source, "DELE", Color::Magenta, max_width, - format!("{{ credential: {credential:?}, pool: {pool_hash} }}"), - ), - EventData::PoolRegistration { - operator, - vrf_keyhash: _, - pledge, - cost, - margin, - reward_account: _, - pool_owners: _, - relays: _, - pool_metadata, - pool_metadata_hash: _, - } => LogLine::new_raw( + format!("{{ credential: {0:?}, pool: {1} }}", cert.credential, cert.pool_hash), + ), + EventData::PoolRegistration(cert) => LogLine::new_raw( source, "POOL+", Color::Magenta, max_width, format!( - "{{ operator: {operator}, pledge: {pledge}, cost: {cost}, margin: {margin}, metadata: {pool_metadata:?} }}"), + "{{ operator: {0}, pledge: {1}, cost: {2}, margin: {{ numerator: {3}, denominator: {4} }}, metadata: {5:?} }}", + cert.operator, cert.pledge, cert.cost, cert.margin.numerator, cert.margin.denominator, cert.pool_metadata), ), - EventData::PoolRetirement { pool, epoch } => LogLine::new_raw( + EventData::PoolRetirement(cert) => LogLine::new_raw( source, "POOL-", Color::DarkMagenta, max_width, - format!("{{ pool: {pool}, epoch: {epoch} }}"), + format!("{{ pool: {0}, epoch: {1} }}", cert.pool, cert.epoch), ), - EventData::GenesisKeyDelegation { } => LogLine::new_raw( + EventData::GenesisKeyDelegation(cert) => LogLine::new_raw( source, "GENESIS", Color::Magenta, max_width, - "{{ ... }}".to_string(), + format!("{{ genesis_hash: {0}, genesis_delegate_hash: {1}, vrf_key_hash: {2} }}", + cert.genesis_hash, cert.genesis_delegate_hash, cert.vrf_key_hash), ), - EventData::MoveInstantaneousRewardsCert { - from_reserves, - from_treasury, - to_stake_credentials, - to_other_pot, - } => LogLine::new_raw( + EventData::MoveInstantaneousRewardsCert(cert) => LogLine::new_raw( source, "MOVE", Color::Magenta, max_width, format!( - "{{ reserves: {from_reserves}, treasury: {from_treasury}, to_credentials: {to_stake_credentials:?}, to_other_pot: {to_other_pot:?} }}"), + "{{ reserves: {0}, treasury: {1}, to_credentials: {2:?}, to_other_pot: {3:?} }}", + cert.from_reserves, cert.from_treasury, cert.to_stake_credentials, cert.to_other_pot), + ), + EventData::RegCert(cert) => LogLine::new_raw( + source, + "REG", + Color::Magenta, + max_width, + format!("{0:?}", serde_json::to_string(&cert).unwrap()), + ), + EventData::UnRegCert(cert) => LogLine::new_raw( + source, + "UNREG", + Color::DarkMagenta, + max_width, + format!("{0:?}", serde_json::to_string(&cert).unwrap()), + ), + EventData::VoteDeleg(cert) => LogLine::new_raw( + source, + "VOTE", + Color::Magenta, + max_width, + format!("{0:?}", serde_json::to_string(&cert).unwrap()), + ), + EventData::StakeVoteDeleg(cert) => LogLine::new_raw( + source, + "STAKEVOTE", + Color::Magenta, + max_width, + format!("{0:?}", serde_json::to_string(&cert).unwrap()), + ), + EventData::StakeRegDeleg(cert) => LogLine::new_raw( + source, + "STAKEREG", + Color::Magenta, + max_width, + format!("{0:?}", serde_json::to_string(&cert).unwrap()), + ), + EventData::VoteRegDeleg(cert) => LogLine::new_raw( + source, + "VOTEREG", + Color::Magenta, + max_width, + format!("{0:?}", serde_json::to_string(&cert).unwrap()), + ), + EventData::StakeVoteRegDeleg(cert) => LogLine::new_raw( + source, + "STAKEVOTEREG", + Color::Magenta, + max_width, + format!("{0:?}", serde_json::to_string(&cert).unwrap()), + ), + EventData::AuthCommitteeHot(cert) => LogLine::new_raw( + source, + "AUTHHOT", + Color::Magenta, + max_width, + format!("{0:?}", serde_json::to_string(&cert).unwrap()), + ), + EventData::ResignCommitteeCold(cert) => LogLine::new_raw( + source, + "RESIGNCOLD", + Color::DarkMagenta, + max_width, + format!("{0:?}", serde_json::to_string(&cert).unwrap()), + ), + EventData::RegDRepCert(cert) => LogLine::new_raw( + source, + "REGDREP", + Color::Magenta, + max_width, + format!("{0:?}", serde_json::to_string(&cert).unwrap()), + ), + EventData::UnRegDRepCert(cert) => LogLine::new_raw( + source, + "UNREGDREP", + Color::DarkMagenta, + max_width, + format!("{0:?}", serde_json::to_string(&cert).unwrap()), + ), + EventData::UpdateDRepCert(cert) => LogLine::new_raw( + source, + "UPDATEDREP", + Color::Magenta, + max_width, + format!("{0:?}", serde_json::to_string(&cert).unwrap()), ), EventData::RollBack { block_slot, @@ -332,40 +405,6 @@ impl LogLine { ), } } - - pub fn new_apply( - source: &Record, - max_width: Option, - adahandle_policy: &Option, - ) -> LogLine { - match source { - Record::OuraV1Event(evt) => LogLine::new_from_legacy_v1( - evt, - max_width, - adahandle_policy.as_deref().unwrap_or_default(), - ), - _ => todo!(), - } - } - - pub fn new_undo( - source: &Record, - max_width: Option, - adahandle_policy: &Option, - ) -> LogLine { - match source { - Record::OuraV1Event(evt) => LogLine::new_from_legacy_v1( - evt, - max_width, - adahandle_policy.as_deref().unwrap_or_default(), - ), - _ => todo!(), - } - } - - pub fn new_reset(_point: Point) -> LogLine { - todo!() - } } impl Display for LogLine { diff --git a/src/sinks/terminal/mod.rs b/src/sinks/terminal/mod.rs index d5cf81a0..54d387aa 100644 --- a/src/sinks/terminal/mod.rs +++ b/src/sinks/terminal/mod.rs @@ -1,120 +1,5 @@ -use crossterm::style::{Color, Print, Stylize}; -use crossterm::ExecutableCommand; -use gasket::framework::*; -use serde::Deserialize; -use std::io::{stdout, Stdout}; - -use crate::framework::*; - mod format; -mod throttle; - -use format::*; -use throttle::Throttle; - -pub struct Worker { - stdout: Stdout, - throttle: Throttle, -} - -impl Worker { - fn compute_terminal_width(&self, wrap: bool) -> Option { - if !wrap { - return None; - } - - if let Ok((x, _y)) = crossterm::terminal::size() { - return Some(x as usize); - } - - None - } -} - -#[async_trait::async_trait(?Send)] -impl gasket::framework::Worker for Worker { - async fn bootstrap(stage: &Stage) -> Result { - let mut stdout = stdout(); - - stdout - .execute(Print( - "Oura terminal output started, waiting for chain data\n".with(Color::DarkGrey), - )) - .or_panic()?; - - let worker = Self { - stdout, - throttle: stage.config.throttle_min_span_millis.into(), - }; - - Ok(worker) - } - - async fn schedule( - &mut self, - stage: &mut Stage, - ) -> Result, WorkerError> { - let msg = stage.input.recv().await.or_panic()?; - Ok(WorkSchedule::Unit(msg.payload)) - } - - async fn execute(&mut self, unit: &ChainEvent, stage: &mut Stage) -> Result<(), WorkerError> { - let width = self.compute_terminal_width(stage.config.wrap.unwrap_or_default()); - - let point = unit.point().clone(); - - let line = match unit { - ChainEvent::Apply(_, record) => { - LogLine::new_apply(&record, width, &stage.config.adahandle_policy) - } - ChainEvent::Undo(_, record) => { - LogLine::new_undo(&record, width, &stage.config.adahandle_policy) - } - ChainEvent::Reset(point) => LogLine::new_reset(point.clone()), - }; - - self.throttle.wait_turn(); - self.stdout.execute(Print(line)).or_panic()?; - - stage.latest_block.set(point.slot_or_default() as i64); - stage.cursor.add_breadcrumb(point); - - Ok(()) - } -} - -#[derive(Stage)] -#[stage(name = "filter", unit = "ChainEvent", worker = "Worker")] -pub struct Stage { - config: Config, - cursor: Cursor, - - pub input: MapperInputPort, - - #[metric] - latest_block: gasket::metrics::Gauge, - - #[metric] - ops_count: gasket::metrics::Counter, -} - -#[derive(Default, Debug, Deserialize)] -pub struct Config { - pub throttle_min_span_millis: Option, - pub wrap: Option, - pub adahandle_policy: Option, -} - -impl Config { - pub fn bootstrapper(self, ctx: &Context) -> Result { - let stage = Stage { - config: self, - ops_count: Default::default(), - latest_block: Default::default(), - input: Default::default(), - cursor: ctx.cursor.clone(), - }; +mod run; +mod setup; - Ok(stage) - } -} +pub use setup::*; diff --git a/src/sinks/terminal/run.rs b/src/sinks/terminal/run.rs new file mode 100644 index 00000000..a772f31c --- /dev/null +++ b/src/sinks/terminal/run.rs @@ -0,0 +1,54 @@ +use std::sync::Arc; +use std::time::Duration; + +use crate::pipelining::StageReceiver; +use crate::utils::throttle::Throttle; +use crate::utils::Utils; + +pub type Error = Box; + +use crossterm::style::{Color, Print, Stylize}; +use crossterm::ExecutableCommand; +use std::io::stdout; + +use super::format::*; + +pub fn reducer_loop( + throttle_min_span: Duration, + wrap: bool, + input: StageReceiver, + utils: Arc, +) -> Result<(), Error> { + let mut stdout = stdout(); + + let mut throttle = Throttle::new(throttle_min_span); + + stdout.execute(Print( + "Oura terminal output started, waiting for chain data\n".with(Color::DarkGrey), + ))?; + + for evt in input.iter() { + let width = match wrap { + true => None, + false => Some(crossterm::terminal::size()?.0 as usize), + }; + + throttle.wait_turn(); + let line = LogLine::new(&evt, width, &utils); + + let result = stdout.execute(Print(line)); + + match result { + Ok(_) => { + // notify progress to the pipeline + utils.track_sink_progress(&evt); + } + Err(err) => { + log::error!("error writing to terminal: {}", err); + return Err(Box::new(err)); + } + } + } + + Ok(()) +} diff --git a/src/sinks/terminal/setup.rs b/src/sinks/terminal/setup.rs new file mode 100644 index 00000000..b5725923 --- /dev/null +++ b/src/sinks/terminal/setup.rs @@ -0,0 +1,37 @@ +use std::time::Duration; + +use serde::Deserialize; + +use crate::{ + pipelining::{BootstrapResult, SinkProvider, StageReceiver}, + utils::WithUtils, +}; + +use super::run::reducer_loop; + +const THROTTLE_MIN_SPAN_MILLIS: u64 = 300; + +#[derive(Default, Debug, Deserialize)] +pub struct Config { + pub throttle_min_span_millis: Option, + pub wrap: Option, +} + +impl SinkProvider for WithUtils { + fn bootstrap(&self, input: StageReceiver) -> BootstrapResult { + let throttle_min_span = Duration::from_millis( + self.inner + .throttle_min_span_millis + .unwrap_or(THROTTLE_MIN_SPAN_MILLIS), + ); + + let wrap = self.inner.wrap.unwrap_or(false); + let utils = self.utils.clone(); + + let handle = std::thread::spawn(move || { + reducer_loop(throttle_min_span, wrap, input, utils).expect("terminal sink loop failed"); + }); + + Ok(handle) + } +} diff --git a/src/sinks/webhook.rs b/src/sinks/webhook.rs deleted file mode 100644 index 262a4932..00000000 --- a/src/sinks/webhook.rs +++ /dev/null @@ -1,153 +0,0 @@ -use gasket::framework::*; -use pallas::network::miniprotocols::Point; -use reqwest::header; -use serde::Deserialize; -use std::{collections::HashMap, time::Duration}; - -use crate::framework::*; - -pub static APP_USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); - -pub fn build_headers_map( - authorization: Option<&String>, - extra: Option<&HashMap>, -) -> Result { - let mut headers = header::HeaderMap::new(); - - headers.insert( - header::CONTENT_TYPE, - header::HeaderValue::try_from("application/json").map_err(Error::config)?, - ); - - if let Some(auth_value) = &authorization { - let auth_value = header::HeaderValue::try_from(*auth_value).map_err(Error::config)?; - headers.insert(header::AUTHORIZATION, auth_value); - } - - if let Some(custom) = &extra { - for (name, value) in custom.iter() { - let name = header::HeaderName::try_from(name).map_err(Error::config)?; - let value = header::HeaderValue::try_from(value).map_err(Error::config)?; - headers.insert(name, value); - } - } - - Ok(headers) -} - -pub struct Worker { - client: reqwest::Client, -} - -#[async_trait::async_trait(?Send)] -impl gasket::framework::Worker for Worker { - async fn bootstrap(stage: &Stage) -> Result { - let headers = build_headers_map( - stage.config.authorization.as_ref(), - stage.config.headers.as_ref(), - ) - .or_panic()?; - - let client = reqwest::ClientBuilder::new() - .user_agent(APP_USER_AGENT) - .default_headers(headers) - .danger_accept_invalid_certs(stage.config.allow_invalid_certs.unwrap_or(false)) - .timeout(Duration::from_millis(stage.config.timeout.unwrap_or(30000))) - .build() - .or_panic()?; - - Ok(Self { client }) - } - - async fn schedule( - &mut self, - stage: &mut Stage, - ) -> Result, WorkerError> { - let msg = stage.input.recv().await.or_panic()?; - Ok(WorkSchedule::Unit(msg.payload)) - } - - async fn execute(&mut self, unit: &ChainEvent, stage: &mut Stage) -> Result<(), WorkerError> { - let point = unit.point().clone(); - let record = unit.record().cloned(); - - if record.is_none() { - return Ok(()); - } - - let body = serde_json::Value::from(record.unwrap()); - - let point_header = match &point { - Point::Origin => String::from("origin"), - Point::Specific(a, b) => format!("{a},{}", hex::encode(b)), - }; - - let request = self - .client - .post(&stage.config.url) - .header("x-oura-chainsync-action", "apply") - .header("x-oura-chainsync-point", point_header) - .json(&body) - .build() - .or_panic()?; - - self.client - .execute(request) - .await - .and_then(|res| res.error_for_status()) - .or_retry()?; - - stage.ops_count.inc(1); - - stage.latest_block.set(point.slot_or_default() as i64); - stage.cursor.add_breadcrumb(point.clone()); - - Ok(()) - } -} - -#[derive(Stage)] -#[stage(name = "filter", unit = "ChainEvent", worker = "Worker")] -pub struct Stage { - config: Config, - cursor: Cursor, - - pub input: MapperInputPort, - - #[metric] - ops_count: gasket::metrics::Counter, - - #[metric] - latest_block: gasket::metrics::Gauge, -} - -#[derive(Default, Deserialize)] -pub struct Config { - pub url: String, - pub authorization: Option, - pub headers: Option>, - pub timeout: Option, - - /// Accept invalid TLS certificates - /// - /// DANGER Will Robinson! Set this flag to skip TLS verification. Main - /// use-case for this flag is to allow self-signed certificates. Beware that - /// other invalid properties will be omitted too, such as expiration date. - pub allow_invalid_certs: Option, - - pub retries: Option, -} - -impl Config { - pub fn bootstrapper(self, ctx: &Context) -> Result { - let stage = Stage { - config: self, - cursor: ctx.cursor.clone(), - ops_count: Default::default(), - latest_block: Default::default(), - input: Default::default(), - }; - - Ok(stage) - } -} diff --git a/src/sinks/webhook/mod.rs b/src/sinks/webhook/mod.rs new file mode 100644 index 00000000..172de9e5 --- /dev/null +++ b/src/sinks/webhook/mod.rs @@ -0,0 +1,3 @@ +mod setup; + +pub use setup::*; diff --git a/src/sinks/webhook/setup.rs b/src/sinks/webhook/setup.rs new file mode 100644 index 00000000..9d0a22e3 --- /dev/null +++ b/src/sinks/webhook/setup.rs @@ -0,0 +1,61 @@ +use std::{collections::HashMap, time::Duration}; + +use serde::Deserialize; + +use crate::{ + pipelining::{BootstrapResult, SinkProvider, StageReceiver}, + sinks::common::web::{build_headers_map, request_loop, ErrorPolicy, APP_USER_AGENT}, + utils::{retry, WithUtils}, +}; + +#[derive(Default, Debug, Deserialize)] +pub struct Config { + pub url: String, + pub authorization: Option, + pub headers: Option>, + pub timeout: Option, + + /// Accept invalid TLS certificates + /// + /// DANGER Will Robinson! Set this flag to skip TLS verification. Main + /// use-case for this flag is to allow self-signed certificates. Beware that + /// other invalid properties will be ommited too, such as expiration date. + pub allow_invalid_certs: Option, + + pub error_policy: Option, + pub retry_policy: Option, +} + +impl SinkProvider for WithUtils { + fn bootstrap(&self, input: StageReceiver) -> BootstrapResult { + let client = reqwest::blocking::ClientBuilder::new() + .user_agent(APP_USER_AGENT) + .default_headers(build_headers_map( + self.inner.authorization.as_ref(), + self.inner.headers.as_ref(), + )?) + .danger_accept_invalid_certs(self.inner.allow_invalid_certs.unwrap_or(false)) + .timeout(Duration::from_millis(self.inner.timeout.unwrap_or(30000))) + .build()?; + + let url = self.inner.url.clone(); + + let error_policy = self + .inner + .error_policy + .as_ref() + .cloned() + .unwrap_or(ErrorPolicy::Exit); + + let retry_policy = self.inner.retry_policy.unwrap_or_default(); + + let utils = self.utils.clone(); + + let handle = std::thread::spawn(move || { + request_loop(input, &client, &url, &error_policy, &retry_policy, utils) + .expect("request loop failed") + }); + + Ok(handle) + } +} diff --git a/src/sources/common.rs b/src/sources/common.rs new file mode 100644 index 00000000..2640b48a --- /dev/null +++ b/src/sources/common.rs @@ -0,0 +1,376 @@ +use core::fmt; +use std::{ops::Deref, str::FromStr, time::Duration}; + +use pallas::{ + ledger::traverse::{probe, Era}, + network::{ + miniprotocols::{chainsync, Point, MAINNET_MAGIC, TESTNET_MAGIC}, + multiplexer::{bearers::Bearer, StdChannel, StdPlexer}, + }, +}; + +use serde::{de::Visitor, Deserializer}; +use serde::{Deserialize, Serialize}; + +use crate::{ + mapper::EventWriter, + utils::{retry, SwallowResult, Utils}, + Error, +}; + +// TODO: these should come from Pallas +use crate::utils::{PREPROD_MAGIC, PREVIEW_MAGIC, SANCHO_MAGIC}; + +#[derive(Debug, Deserialize, Clone)] +pub enum BearerKind { + Tcp, + #[cfg(target_family = "unix")] + Unix, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct AddressArg(pub BearerKind, pub String); + +impl FromStr for BearerKind { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + match s { + "tcp" => Ok(BearerKind::Tcp), + #[cfg(target_family = "unix")] + "unix" => Ok(BearerKind::Unix), + _ => Err("can't parse bearer type value"), + } + } +} + +/// A serialization-friendly chain Point struct using a hex-encoded hash +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct PointArg(pub u64, pub String); + +impl TryInto for PointArg { + type Error = Error; + + fn try_into(self) -> Result { + let hash = hex::decode(&self.1)?; + Ok(Point::Specific(self.0, hash)) + } +} + +impl FromStr for PointArg { + type Err = Error; + + fn from_str(s: &str) -> Result { + if s.contains(',') { + let mut parts: Vec<_> = s.split(',').collect(); + let slot = parts.remove(0).parse()?; + let hash = parts.remove(0).to_owned(); + Ok(PointArg(slot, hash)) + } else { + Err("Can't parse chain point value, expecting `slot,hex-hash` format".into()) + } + } +} + +impl ToString for PointArg { + fn to_string(&self) -> String { + format!("{},{}", self.0, self.1) + } +} + +#[derive(Debug, Deserialize, Clone)] +pub struct MagicArg(pub u64); + +impl Deref for MagicArg { + type Target = u64; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl FromStr for MagicArg { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + let m = match s { + "testnet" => MagicArg(TESTNET_MAGIC), + "mainnet" => MagicArg(MAINNET_MAGIC), + "preview" => MagicArg(PREVIEW_MAGIC), + "preprod" => MagicArg(PREPROD_MAGIC), + "sancho" => MagicArg(SANCHO_MAGIC), + _ => MagicArg(u64::from_str(s).map_err(|_| "can't parse magic value")?), + }; + + Ok(m) + } +} + +impl Default for MagicArg { + fn default() -> Self { + Self(MAINNET_MAGIC) + } +} + +pub(crate) fn deserialize_magic_arg<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + struct MagicArgVisitor; + + impl<'de> Visitor<'de> for MagicArgVisitor { + type Value = Option; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("string or number") + } + + fn visit_str(self, value: &str) -> Result, E> + where + E: serde::de::Error, + { + let value = FromStr::from_str(value).map_err(serde::de::Error::custom)?; + Ok(Some(value)) + } + + fn visit_u64(self, value: u64) -> Result, E> + where + E: serde::de::Error, + { + Ok(Some(MagicArg(value))) + } + + fn visit_i64(self, v: i64) -> Result + where + E: serde::de::Error, + { + Ok(Some(MagicArg(v as u64))) + } + + fn visit_none(self) -> Result + where + E: serde::de::Error, + { + Ok(None) + } + } + + deserializer.deserialize_any(MagicArgVisitor) +} + +#[derive(Deserialize, Debug, Clone)] +pub struct RetryPolicy { + #[serde(default = "RetryPolicy::default_max_retries")] + pub chainsync_max_retries: u32, + + #[serde(default = "RetryPolicy::default_max_backoff")] + pub chainsync_max_backoff: u32, + + #[serde(default = "RetryPolicy::default_max_retries")] + pub connection_max_retries: u32, + + #[serde(default = "RetryPolicy::default_max_backoff")] + pub connection_max_backoff: u32, +} + +impl RetryPolicy { + fn default_max_retries() -> u32 { + 50 + } + + fn default_max_backoff() -> u32 { + 60 + } +} + +pub fn setup_multiplexer_attempt(bearer: &BearerKind, address: &str) -> Result { + match bearer { + BearerKind::Tcp => { + let bearer = Bearer::connect_tcp(address)?; + Ok(StdPlexer::new(bearer)) + } + #[cfg(target_family = "unix")] + BearerKind::Unix => { + let unix = Bearer::connect_unix(address)?; + Ok(StdPlexer::new(unix)) + } + } +} + +pub fn setup_multiplexer( + bearer: &BearerKind, + address: &str, + retry: &Option, +) -> Result { + match retry { + Some(policy) => retry::retry_operation( + || setup_multiplexer_attempt(bearer, address), + &retry::Policy { + max_retries: policy.connection_max_retries, + backoff_unit: Duration::from_secs(1), + backoff_factor: 2, + max_backoff: Duration::from_secs(policy.connection_max_backoff as u64), + }, + ), + None => setup_multiplexer_attempt(bearer, address), + } +} + +#[derive(Debug, Deserialize, Clone)] +#[serde(tag = "type", content = "value")] +pub enum IntersectArg { + Tip, + Origin, + Point(PointArg), + Fallbacks(Vec), +} + +#[derive(Deserialize, Debug, Clone)] +pub struct FinalizeConfig { + max_block_quantity: Option, + max_block_slot: Option, + until_hash: Option, +} + +pub fn should_finalize( + config: &Option, + last_point: &Point, + block_count: u64, +) -> bool { + let config = match config { + Some(x) => x, + None => return false, + }; + + if let Some(max) = config.max_block_quantity { + if block_count >= max { + return true; + } + } + + if let Some(max) = config.max_block_slot { + if last_point.slot_or_default() >= max { + return true; + } + } + + if let Some(expected) = &config.until_hash { + if let Point::Specific(_, current) = last_point { + return expected == &hex::encode(current); + } + } + + false +} + +pub(crate) fn intersect_starting_point( + client: &mut chainsync::Client, + intersect_arg: &Option, + since_arg: &Option, + utils: &Utils, +) -> Result, Error> +where + chainsync::Message: pallas::codec::Fragment, +{ + let cursor = utils.get_cursor_if_any(); + + match cursor { + Some(cursor) => { + log::info!("found persisted cursor, will use as starting point"); + let desired = cursor.try_into()?; + let (point, _) = client.find_intersect(vec![desired])?; + + Ok(point) + } + None => match intersect_arg { + Some(IntersectArg::Fallbacks(x)) => { + log::info!("found 'fallbacks' intersect argument, will use as starting point"); + let options: Result, _> = x.iter().map(|x| x.clone().try_into()).collect(); + + let (point, _) = client.find_intersect(options?)?; + + Ok(point) + } + Some(IntersectArg::Origin) => { + log::info!("found 'origin' intersect argument, will use as starting point"); + + let point = client.intersect_origin()?; + + Ok(Some(point)) + } + Some(IntersectArg::Point(x)) => { + log::info!("found 'point' intersect argument, will use as starting point"); + let options = vec![x.clone().try_into()?]; + + let (point, _) = client.find_intersect(options)?; + + Ok(point) + } + Some(IntersectArg::Tip) => { + log::info!("found 'tip' intersect argument, will use as starting point"); + + let point = client.intersect_tip()?; + + Ok(Some(point)) + } + None => match since_arg { + Some(x) => { + log::info!("explicit 'since' argument, will use as starting point"); + log::warn!("`since` value is deprecated, please use `intersect`"); + let options = vec![x.clone().try_into()?]; + + let (point, _) = client.find_intersect(options)?; + + Ok(point) + } + None => { + log::info!("no starting point specified, will use tip of chain"); + + let point = client.intersect_tip()?; + + Ok(Some(point)) + } + }, + }, + } +} + +pub fn unknown_block_to_events(writer: &EventWriter, body: &Vec) -> Result<(), Error> { + match probe::block_era(body) { + probe::Outcome::Matched(era) => match era { + Era::Byron => { + writer + .crawl_from_byron_cbor(body) + .ok_or_warn("error crawling byron block for events"); + } + Era::Allegra | Era::Alonzo | Era::Mary | Era::Shelley => { + writer + .crawl_from_shelley_cbor(body, era.into()) + .ok_or_warn("error crawling alonzo-compatible block for events"); + } + Era::Babbage => { + writer + .crawl_from_babbage_cbor(body) + .ok_or_warn("error crawling babbage block for events"); + } + Era::Conway => { + writer + .crawl_from_babbage_cbor(body) + .ok_or_warn("error crawling conway block for events"); + } + x => { + return Err(format!("This version of Oura can't handle era: {x}").into()); + } + }, + probe::Outcome::EpochBoundary => { + writer + .crawl_from_ebb_cbor(body) + .ok_or_warn("error crawling block for events"); + } + probe::Outcome::Inconclusive => { + log::error!("can't infer primitive block from cbor, inconclusive probing. CBOR hex for debugging: {}", hex::encode(body)); + } + } + + Ok(()) +} diff --git a/src/sources/mod.rs b/src/sources/mod.rs index 94121a72..0b52234c 100644 --- a/src/sources/mod.rs +++ b/src/sources/mod.rs @@ -1,71 +1,6 @@ -use gasket::{messaging::SendPort, runtime::Tether}; -use serde::Deserialize; - -use crate::framework::*; - -//#[cfg(target_family = "unix")] -//pub mod n2c; +mod common; pub mod n2c; pub mod n2n; -#[cfg(feature = "aws")] -pub mod s3; - -pub enum Bootstrapper { - N2N(n2n::Stage), - N2C(n2c::Stage), - - #[cfg(feature = "aws")] - S3(s3::Stage), -} - -impl StageBootstrapper for Bootstrapper { - fn connect_output(&mut self, adapter: OutputAdapter) { - match self { - Bootstrapper::N2N(p) => p.output.connect(adapter), - Bootstrapper::N2C(p) => p.output.connect(adapter), - - #[cfg(feature = "aws")] - Bootstrapper::S3(p) => p.output.connect(adapter), - } - } - - fn connect_input(&mut self, _: InputAdapter) { - panic!("attempted to use source stage as receiver"); - } - - fn spawn(self, policy: gasket::runtime::Policy) -> Tether { - match self { - Bootstrapper::N2N(x) => gasket::runtime::spawn_stage(x, policy), - Bootstrapper::N2C(x) => gasket::runtime::spawn_stage(x, policy), - - #[cfg(feature = "aws")] - Bootstrapper::S3(x) => gasket::runtime::spawn_stage(x, policy), - } - } -} - -#[derive(Deserialize)] -#[serde(tag = "type")] -pub enum Config { - N2N(n2n::Config), - - #[cfg(target_family = "unix")] - N2C(n2c::Config), - - #[cfg(feature = "aws")] - S3(s3::Config), -} - -impl Config { - pub fn bootstrapper(self, ctx: &Context) -> Result { - match self { - Config::N2N(c) => Ok(Bootstrapper::N2N(c.bootstrapper(ctx)?)), - Config::N2C(c) => Ok(Bootstrapper::N2C(c.bootstrapper(ctx)?)), - - #[cfg(feature = "aws")] - Config::S3(c) => Ok(Bootstrapper::S3(c.bootstrapper(ctx)?)), - } - } -} +pub use common::*; diff --git a/src/sources/n2c.rs b/src/sources/n2c.rs deleted file mode 100644 index 39e63027..00000000 --- a/src/sources/n2c.rs +++ /dev/null @@ -1,207 +0,0 @@ -use std::path::PathBuf; - -use gasket::framework::*; -use serde::Deserialize; -use tracing::{debug, info}; - -use pallas::ledger::traverse::MultiEraBlock; -use pallas::network::facades::NodeClient; -use pallas::network::miniprotocols::chainsync::{BlockContent, NextResponse}; -use pallas::network::miniprotocols::Point; - -use crate::framework::*; - -#[derive(Stage)] -#[stage( - name = "source", - unit = "NextResponse", - worker = "Worker" -)] -pub struct Stage { - config: Config, - - chain: GenesisValues, - - intersect: IntersectConfig, - - cursor: Cursor, - - pub output: SourceOutputPort, - - #[metric] - ops_count: gasket::metrics::Counter, - - #[metric] - chain_tip: gasket::metrics::Gauge, -} - -async fn intersect_from_config( - peer: &mut NodeClient, - intersect: &IntersectConfig, -) -> Result<(), WorkerError> { - let chainsync = peer.chainsync(); - - let intersect = match intersect { - IntersectConfig::Origin => { - info!("intersecting origin"); - chainsync.intersect_origin().await.or_restart()?.into() - } - IntersectConfig::Tip => { - info!("intersecting tip"); - chainsync.intersect_tip().await.or_restart()?.into() - } - IntersectConfig::Point(..) | IntersectConfig::Breadcrumbs(..) => { - info!("intersecting specific points"); - let points = intersect.points().unwrap_or_default(); - let (point, _) = chainsync.find_intersect(points).await.or_restart()?; - point - } - }; - - info!(?intersect, "intersected"); - - Ok(()) -} - -async fn intersect_from_cursor(peer: &mut NodeClient, cursor: &Cursor) -> Result<(), WorkerError> { - let points = cursor.clone_state(); - - let (intersect, _) = peer - .chainsync() - .find_intersect(points.into()) - .await - .or_restart()?; - - info!(?intersect, "intersected"); - - Ok(()) -} - -pub struct Worker { - peer_session: NodeClient, -} - -impl Worker { - async fn process_next( - &mut self, - stage: &mut Stage, - next: &NextResponse, - ) -> Result<(), WorkerError> { - match next { - NextResponse::RollForward(cbor, tip) => { - let block = MultiEraBlock::decode(&cbor).or_panic()?; - let slot = block.slot(); - let hash = block.hash(); - - debug!(slot, %hash, "chain sync roll forward"); - - let evt = ChainEvent::Apply( - pallas::network::miniprotocols::Point::Specific(slot, hash.to_vec()), - Record::CborBlock(cbor.to_vec()), - ); - - stage.output.send(evt.into()).await.or_panic()?; - - stage.chain_tip.set(tip.0.slot_or_default() as i64); - - Ok(()) - } - NextResponse::RollBackward(point, tip) => { - match &point { - Point::Origin => debug!("rollback to origin"), - Point::Specific(slot, _) => debug!(slot, "rollback"), - }; - - stage - .output - .send(ChainEvent::reset(point.clone()).into()) - .await - .or_panic()?; - - stage.chain_tip.set(tip.0.slot_or_default() as i64); - - Ok(()) - } - NextResponse::Await => { - info!("chain-sync reached the tip of the chain"); - Ok(()) - } - } - } -} - -#[async_trait::async_trait(?Send)] -impl gasket::framework::Worker for Worker { - async fn bootstrap(stage: &Stage) -> Result { - debug!("connecting"); - - let mut peer_session = NodeClient::connect(&stage.config.socket_path, stage.chain.magic) - .await - .or_retry()?; - - if stage.cursor.is_empty() { - intersect_from_config(&mut peer_session, &stage.intersect).await?; - } else { - intersect_from_cursor(&mut peer_session, &stage.cursor).await?; - } - - let worker = Self { peer_session }; - - Ok(worker) - } - - async fn schedule( - &mut self, - _stage: &mut Stage, - ) -> Result>, WorkerError> { - let client = self.peer_session.chainsync(); - - let next = match client.has_agency() { - true => { - info!("requesting next block"); - client.request_next().await.or_restart()? - } - false => { - info!("awaiting next block (blocking)"); - client.recv_while_must_reply().await.or_restart()? - } - }; - - Ok(WorkSchedule::Unit(next)) - } - - async fn execute( - &mut self, - unit: &NextResponse, - stage: &mut Stage, - ) -> Result<(), WorkerError> { - self.process_next(stage, unit).await - } - - async fn teardown(&mut self) -> Result<(), WorkerError> { - self.peer_session.abort(); - - Ok(()) - } -} - -#[derive(Deserialize)] -pub struct Config { - socket_path: PathBuf, -} - -impl Config { - pub fn bootstrapper(self, ctx: &Context) -> Result { - let stage = Stage { - config: self, - chain: ctx.chain.clone(), - intersect: ctx.intersect.clone(), - cursor: ctx.cursor.clone(), - output: Default::default(), - ops_count: Default::default(), - chain_tip: Default::default(), - }; - - Ok(stage) - } -} diff --git a/src/sources/n2c/mod.rs b/src/sources/n2c/mod.rs new file mode 100644 index 00000000..0a447c1d --- /dev/null +++ b/src/sources/n2c/mod.rs @@ -0,0 +1,4 @@ +mod run; +mod setup; + +pub use setup::*; diff --git a/src/sources/n2c/run.rs b/src/sources/n2c/run.rs new file mode 100644 index 00000000..73640341 --- /dev/null +++ b/src/sources/n2c/run.rs @@ -0,0 +1,281 @@ +use std::{collections::HashMap, fmt::Debug, ops::Deref, sync::Arc, time::Duration}; + +use pallas::{ + ledger::traverse::MultiEraBlock, + network::{ + miniprotocols::{chainsync, handshake, Point, MAINNET_MAGIC}, + multiplexer::StdChannel, + }, +}; + +use crate::{ + mapper::EventWriter, + pipelining::StageSender, + sources::{ + intersect_starting_point, setup_multiplexer, should_finalize, unknown_block_to_events, + FinalizeConfig, + }, + utils::{retry, Utils}, + Error, +}; + +struct ChainObserver { + chain_buffer: chainsync::RollbackBuffer, + min_depth: usize, + blocks: HashMap>, + event_writer: EventWriter, + finalize_config: Option, + block_count: u64, +} + +// workaround to put a stop on excessive debug requirement coming from Pallas +impl Debug for ChainObserver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("ChainObserver").finish() + } +} + +fn log_buffer_state(buffer: &chainsync::RollbackBuffer) { + log::info!( + "rollback buffer state, size: {}, oldest: {:?}, latest: {:?}", + buffer.size(), + buffer.oldest(), + buffer.latest(), + ); +} + +enum Continuation { + Proceed, + DropOut, +} + +impl ChainObserver { + fn on_roll_forward( + &mut self, + content: chainsync::BlockContent, + tip: &chainsync::Tip, + ) -> Result> { + // parse the block and extract the point of the chain + + let block = MultiEraBlock::decode(content.deref()).map_err( + |err| { + log::error!("error decoding block: {:?} block hex: {}", err, hex::encode(content.deref())); + err + } + )?; + let point = Point::Specific(block.slot(), block.hash().to_vec()); + + // store the block for later retrieval + self.blocks.insert(point.clone(), content.into()); + + // track the new point in our memory buffer + log::info!("rolling forward to point {:?}", point); + self.chain_buffer.roll_forward(point); + + // see if we have points that already reached certain depth + let ready = self.chain_buffer.pop_with_depth(self.min_depth); + log::debug!("found {} points with required min depth", ready.len()); + + // find confirmed block in memory and send down the pipeline + for point in ready { + let block = self + .blocks + .remove(&point) + .expect("required block not found in memory"); + + unknown_block_to_events(&self.event_writer, &block)?; + + self.block_count += 1; + + // evaluate if we should finalize the thread according to config + if should_finalize(&self.finalize_config, &point, self.block_count) { + return Ok(Continuation::DropOut); + } + } + + log_buffer_state(&self.chain_buffer); + + // notify chain tip to the pipeline metrics + self.event_writer.utils.track_chain_tip(tip.1); + + Ok(Continuation::Proceed) + } + + fn on_rollback(&mut self, point: &Point) -> Result<(), Error> { + log::info!("rolling block to point {:?}", point); + + match self.chain_buffer.roll_back(point) { + chainsync::RollbackEffect::Handled => { + log::debug!("handled rollback within buffer {:?}", point); + + // drain memory blocks after the rollback slot + self.blocks + .retain(|x, _| x.slot_or_default() <= point.slot_or_default()); + } + chainsync::RollbackEffect::OutOfScope => { + log::debug!("rollback out of buffer scope, sending event down the pipeline"); + + // clear all the blocks in memory, they are orphan + self.blocks.clear(); + + self.event_writer.append_rollback_event(point)?; + } + } + + log_buffer_state(&self.chain_buffer); + + Ok(()) + } + + fn on_next_message( + &mut self, + msg: chainsync::NextResponse, + client: &mut chainsync::N2CClient, + ) -> Result { + match msg { + chainsync::NextResponse::RollForward(c, t) => match self.on_roll_forward(c, &t) { + Ok(x) => Ok(x), + Err(err) => Err(AttemptError::Other(err)), + }, + chainsync::NextResponse::RollBackward(x, _) => match self.on_rollback(&x) { + Ok(_) => Ok(Continuation::Proceed), + Err(err) => Err(AttemptError::Other(err)), + }, + chainsync::NextResponse::Await => { + let next = client + .recv_while_must_reply() + .map_err(|x| AttemptError::Recoverable(x.into()))?; + + self.on_next_message(next, client) + } + } + } +} + +fn observe_forever( + mut client: chainsync::N2CClient, + event_writer: EventWriter, + min_depth: usize, + finalize_config: Option, +) -> Result<(), AttemptError> { + let mut observer = ChainObserver { + chain_buffer: Default::default(), + blocks: HashMap::new(), + min_depth, + event_writer, + block_count: 0, + finalize_config, + }; + + loop { + match client.request_next() { + Ok(next) => match observer.on_next_message(next, &mut client) { + Ok(Continuation::Proceed) => (), + Ok(Continuation::DropOut) => break Ok(()), + Err(err) => break Err(err), + }, + Err(err) => break Err(AttemptError::Recoverable(err.into())), + } + } +} + +#[derive(Debug)] +enum AttemptError { + Recoverable(Error), + Other(Error), +} + +fn do_handshake(channel: StdChannel, magic: u64) -> Result<(), AttemptError> { + let mut client = handshake::N2CClient::new(channel); + let versions = handshake::n2c::VersionTable::v1_and_above(magic); + + match client.handshake(versions) { + Ok(confirmation) => match confirmation { + handshake::Confirmation::Accepted(_, _) => Ok(()), + _ => Err(AttemptError::Other( + "couldn't agree on handshake version".into(), + )), + }, + Err(err) => Err(AttemptError::Recoverable(err.into())), + } +} + +fn do_chainsync_attempt( + config: &super::Config, + utils: Arc, + output_tx: &StageSender, +) -> Result<(), AttemptError> { + let magic = match config.magic.as_ref() { + Some(m) => *m.deref(), + None => MAINNET_MAGIC, + }; + + let mut plexer = setup_multiplexer(&config.address.0, &config.address.1, &config.retry_policy) + .map_err(|x| AttemptError::Recoverable(x))?; + + let hs_channel = plexer.use_channel(0); + let cs_channel = plexer.use_channel(5); + + plexer.muxer.spawn(); + plexer.demuxer.spawn(); + + do_handshake(hs_channel, magic)?; + + let mut client = chainsync::N2CClient::new(cs_channel); + + let intersection = intersect_starting_point( + &mut client, + &config.intersect, + #[allow(deprecated)] + &config.since, + &utils, + ) + .map_err(|err| AttemptError::Recoverable(err))?; + + if intersection.is_none() { + return Err(AttemptError::Other( + "Can't find chain intersection point".into(), + )); + } + + log::info!("starting chain sync from: {:?}", &intersection); + + let writer = EventWriter::new(output_tx.clone(), utils, config.mapper.clone()); + + observe_forever(client, writer, config.min_depth, config.finalize.clone())?; + + Ok(()) +} + +pub fn do_chainsync( + config: &super::Config, + utils: Arc, + output_tx: StageSender, +) -> Result<(), Error> { + retry::retry_operation( + || match do_chainsync_attempt(config, utils.clone(), &output_tx) { + Ok(()) => Ok(()), + Err(AttemptError::Other(msg)) => { + log::error!("N2C error: {}", msg); + log::warn!("unrecoverable error performing chainsync, will exit"); + Ok(()) + } + Err(AttemptError::Recoverable(err)) => Err(err), + }, + &retry::Policy { + max_retries: config + .retry_policy + .as_ref() + .map(|x| x.chainsync_max_retries) + .unwrap_or(50), + backoff_unit: Duration::from_secs(1), + backoff_factor: 2, + max_backoff: config + .retry_policy + .as_ref() + .map(|x| x.chainsync_max_backoff as u64) + .map(Duration::from_secs) + .unwrap_or_else(|| Duration::from_secs(60)), + }, + ) +} diff --git a/src/sources/n2c/setup.rs b/src/sources/n2c/setup.rs new file mode 100644 index 00000000..c1bc5c39 --- /dev/null +++ b/src/sources/n2c/setup.rs @@ -0,0 +1,61 @@ +use serde::Deserialize; + +use crate::{ + mapper::Config as MapperConfig, + pipelining::{new_inter_stage_channel, PartialBootstrapResult, SourceProvider}, + sources::{ + common::{AddressArg, MagicArg, PointArg}, + FinalizeConfig, IntersectArg, RetryPolicy, + }, + utils::{ChainWellKnownInfo, WithUtils}, +}; + +use super::run::do_chainsync; + +#[derive(Debug, Deserialize, Clone)] +pub struct Config { + pub address: AddressArg, + + #[serde(deserialize_with = "crate::sources::common::deserialize_magic_arg")] + pub magic: Option, + + #[deprecated(note = "use intersect value instead")] + pub since: Option, + + pub intersect: Option, + + #[deprecated(note = "chain info is now pipeline-wide, use utils")] + pub well_known: Option, + + #[serde(default)] + pub mapper: MapperConfig, + + /// Min block depth (# confirmations) required + /// + /// The min depth a block requires to be considered safe to send down the + /// pipeline. This value is used to configure a rollback buffer used + /// internally by the stage. A high value (eg: ~6) will reduce the + /// probability of seeing rollbacks events. The trade-off is that the stage + /// will need some time to fill up the buffer before sending the 1st event. + #[serde(default)] + pub min_depth: usize, + + pub retry_policy: Option, + + pub finalize: Option, +} + +impl SourceProvider for WithUtils { + fn bootstrap(&self) -> PartialBootstrapResult { + let (output_tx, output_rx) = new_inter_stage_channel(None); + + let config = self.inner.clone(); + let utils = self.utils.clone(); + let handle = std::thread::spawn(move || { + do_chainsync(&config, utils, output_tx) + .expect("chainsync process fails after max retries") + }); + + Ok((handle, output_rx)) + } +} diff --git a/src/sources/n2n.rs b/src/sources/n2n.rs deleted file mode 100644 index 2f5bed58..00000000 --- a/src/sources/n2n.rs +++ /dev/null @@ -1,229 +0,0 @@ -use gasket::framework::*; -use serde::Deserialize; -use tracing::{debug, info}; - -use pallas::ledger::traverse::MultiEraHeader; -use pallas::network::facades::PeerClient; -use pallas::network::miniprotocols::chainsync::{self, HeaderContent, NextResponse}; -use pallas::network::miniprotocols::Point; - -use crate::framework::*; - -#[derive(Stage)] -#[stage( - name = "source", - unit = "NextResponse", - worker = "Worker" -)] -pub struct Stage { - config: Config, - - chain: GenesisValues, - - intersect: IntersectConfig, - - cursor: Cursor, - - pub output: SourceOutputPort, - - #[metric] - ops_count: gasket::metrics::Counter, - - #[metric] - chain_tip: gasket::metrics::Gauge, -} - -fn to_traverse(header: &HeaderContent) -> Result, WorkerError> { - let out = match header.byron_prefix { - Some((subtag, _)) => MultiEraHeader::decode(header.variant, Some(subtag), &header.cbor), - None => MultiEraHeader::decode(header.variant, None, &header.cbor), - }; - - out.or_panic() -} - -async fn intersect_from_config( - peer: &mut PeerClient, - intersect: &IntersectConfig, -) -> Result<(), WorkerError> { - let chainsync = peer.chainsync(); - - let intersect = match intersect { - IntersectConfig::Origin => { - info!("intersecting origin"); - chainsync.intersect_origin().await.or_restart()?.into() - } - IntersectConfig::Tip => { - info!("intersecting tip"); - chainsync.intersect_tip().await.or_restart()?.into() - } - IntersectConfig::Point(..) | IntersectConfig::Breadcrumbs(..) => { - info!("intersecting specific points"); - let points = intersect.points().unwrap_or_default(); - let (point, _) = chainsync.find_intersect(points).await.or_restart()?; - point - } - }; - - info!(?intersect, "intersected"); - - Ok(()) -} - -async fn intersect_from_cursor(peer: &mut PeerClient, cursor: &Cursor) -> Result<(), WorkerError> { - let points = cursor.clone_state(); - - let (intersect, _) = peer - .chainsync() - .find_intersect(points.into()) - .await - .or_restart()?; - - info!(?intersect, "intersected"); - - Ok(()) -} - -pub struct Worker { - peer_session: PeerClient, -} - -impl Worker { - async fn process_next( - &mut self, - stage: &mut Stage, - next: &NextResponse, - ) -> Result<(), WorkerError> { - match next { - NextResponse::RollForward(header, tip) => { - let header = to_traverse(header).or_panic()?; - let slot = header.slot(); - let hash = header.hash(); - - debug!(slot, %hash, "chain sync roll forward"); - - let block = self - .peer_session - .blockfetch() - .fetch_single(Point::Specific(slot, hash.to_vec())) - .await - .or_retry()?; - - let evt = ChainEvent::Apply( - pallas::network::miniprotocols::Point::Specific(slot, hash.to_vec()), - Record::CborBlock(block), - ); - - stage.output.send(evt.into()).await.or_panic()?; - - stage.chain_tip.set(tip.0.slot_or_default() as i64); - - Ok(()) - } - chainsync::NextResponse::RollBackward(point, tip) => { - match &point { - Point::Origin => debug!("rollback to origin"), - Point::Specific(slot, _) => debug!(slot, "rollback"), - }; - - stage - .output - .send(ChainEvent::reset(point.clone()).into()) - .await - .or_panic()?; - - stage.chain_tip.set(tip.0.slot_or_default() as i64); - - Ok(()) - } - chainsync::NextResponse::Await => { - info!("chain-sync reached the tip of the chain"); - Ok(()) - } - } - } -} - -#[async_trait::async_trait(?Send)] -impl gasket::framework::Worker for Worker { - async fn bootstrap(stage: &Stage) -> Result { - debug!("connecting"); - - let peer_address = stage - .config - .peers - .first() - .cloned() - .ok_or_else(|| Error::config("at least one upstream peer is required")) - .or_panic()?; - - let mut peer_session = PeerClient::connect(&peer_address, stage.chain.magic) - .await - .or_retry()?; - - if stage.cursor.is_empty() { - intersect_from_config(&mut peer_session, &stage.intersect).await?; - } else { - intersect_from_cursor(&mut peer_session, &stage.cursor).await?; - } - - let worker = Self { peer_session }; - - Ok(worker) - } - - async fn schedule( - &mut self, - _stage: &mut Stage, - ) -> Result>, WorkerError> { - let client = self.peer_session.chainsync(); - - let next = match client.has_agency() { - true => { - info!("requesting next block"); - client.request_next().await.or_restart()? - } - false => { - info!("awaiting next block (blocking)"); - client.recv_while_must_reply().await.or_restart()? - } - }; - - Ok(WorkSchedule::Unit(next)) - } - - async fn execute( - &mut self, - unit: &NextResponse, - stage: &mut Stage, - ) -> Result<(), WorkerError> { - self.process_next(stage, unit).await - } - - async fn teardown(&mut self) -> Result<(), WorkerError> { - self.peer_session.abort(); - - Ok(()) - } -} - -#[derive(Deserialize)] -pub struct Config { - peers: Vec, -} - -impl Config { - pub fn bootstrapper(self, ctx: &Context) -> Result { - let stage = Stage { - config: self, - chain: ctx.chain.clone(), - intersect: ctx.intersect.clone(), - cursor: ctx.cursor.clone(), - output: Default::default(), - ops_count: Default::default(), - chain_tip: Default::default(), - }; - - Ok(stage) - } -} diff --git a/src/sources/n2n/mod.rs b/src/sources/n2n/mod.rs new file mode 100644 index 00000000..0a447c1d --- /dev/null +++ b/src/sources/n2n/mod.rs @@ -0,0 +1,4 @@ +mod run; +mod setup; + +pub use setup::*; diff --git a/src/sources/n2n/run.rs b/src/sources/n2n/run.rs new file mode 100644 index 00000000..dfff62ac --- /dev/null +++ b/src/sources/n2n/run.rs @@ -0,0 +1,300 @@ +use std::{fmt::Debug, ops::Deref, sync::Arc, time::Duration}; + +use pallas::network::{ + miniprotocols::{blockfetch, chainsync, handshake, Point, MAINNET_MAGIC}, + multiplexer::StdChannel, +}; + +use pallas::network::miniprotocols::handshake::n2n::VersionData; +use std::sync::mpsc::{Receiver, SyncSender}; + +use crate::{ + mapper::EventWriter, + pipelining::StageSender, + sources::{ + intersect_starting_point, setup_multiplexer, should_finalize, unknown_block_to_events, + FinalizeConfig, + }, + utils::{retry, Utils}, + Error, +}; + +struct ChainObserver { + min_depth: usize, + chain_buffer: chainsync::RollbackBuffer, + block_requests: SyncSender, + event_writer: EventWriter, + finalize_config: Option, + block_count: u64, +} + +// workaround to put a stop on excessive debug requirement coming from Pallas +impl Debug for ChainObserver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("ChainObserver").finish() + } +} + +fn log_buffer_state(buffer: &chainsync::RollbackBuffer) { + log::info!( + "rollback buffer state, size: {}, oldest: {:?}, latest: {:?}", + buffer.size(), + buffer.oldest(), + buffer.latest(), + ); +} + +enum Continuation { + Proceed, + DropOut, +} + +impl ChainObserver { + fn on_roll_forward( + &mut self, + content: chainsync::HeaderContent, + tip: &chainsync::Tip, + ) -> Result { + // parse the header and extract the point of the chain + + let header = pallas::ledger::traverse::MultiEraHeader::decode( + content.variant, + content.byron_prefix.map(|x| x.0), + &content.cbor, + )?; + + let point = Point::Specific(header.slot(), header.hash().to_vec()); + + // track the new point in our memory buffer + log::info!("rolling forward to point {:?}", point); + self.chain_buffer.roll_forward(point); + + // see if we have points that already reached certain depth + let ready = self.chain_buffer.pop_with_depth(self.min_depth); + log::debug!("found {} points with required min depth", ready.len()); + + // request download of blocks for confirmed points + for point in ready { + log::debug!("requesting block fetch for point {:?}", point); + self.block_requests.send(point.clone())?; + self.block_count += 1; + + // evaluate if we should finalize the thread according to config + if should_finalize(&self.finalize_config, &point, self.block_count) { + return Ok(Continuation::DropOut); + } + } + + log_buffer_state(&self.chain_buffer); + + // notify chain tip to the pipeline metrics + self.event_writer.utils.track_chain_tip(tip.1); + + Ok(Continuation::Proceed) + } + + fn on_rollback(&mut self, point: &Point) -> Result<(), Error> { + log::info!("rolling block to point {:?}", point); + + match self.chain_buffer.roll_back(point) { + chainsync::RollbackEffect::Handled => { + log::debug!("handled rollback within buffer {:?}", point); + } + chainsync::RollbackEffect::OutOfScope => { + log::debug!("rollback out of buffer scope, sending event down the pipeline"); + self.event_writer.append_rollback_event(point)?; + } + } + + log_buffer_state(&self.chain_buffer); + + Ok(()) + } + + fn on_next_message( + &mut self, + msg: chainsync::NextResponse, + client: &mut chainsync::N2NClient, + ) -> Result { + match msg { + chainsync::NextResponse::RollForward(c, t) => match self.on_roll_forward(c, &t) { + Ok(x) => Ok(x), + Err(err) => Err(AttemptError::Other(err)), + }, + chainsync::NextResponse::RollBackward(x, _) => match self.on_rollback(&x) { + Ok(_) => Ok(Continuation::Proceed), + Err(err) => Err(AttemptError::Other(err)), + }, + chainsync::NextResponse::Await => { + let next = client + .recv_while_must_reply() + .map_err(|x| AttemptError::Recoverable(x.into()))?; + + self.on_next_message(next, client) + } + } + } +} + +pub(crate) fn fetch_blocks_forever( + mut client: blockfetch::Client, + event_writer: EventWriter, + input: Receiver, +) -> Result<(), Error> { + for point in input { + let body = client.fetch_single(point.clone())?; + + unknown_block_to_events(&event_writer, &body)?; + + log::debug!("blockfetch succeeded: {:?}", point); + } + + Ok(()) +} + +fn observe_headers_forever( + mut client: chainsync::N2NClient, + event_writer: EventWriter, + block_requests: SyncSender, + min_depth: usize, + finalize_config: Option, +) -> Result<(), AttemptError> { + let observer = &mut ChainObserver { + chain_buffer: Default::default(), + min_depth, + event_writer, + block_requests, + block_count: 0, + finalize_config, + }; + + loop { + match client.request_next() { + Ok(next) => match observer.on_next_message(next, &mut client) { + Ok(Continuation::Proceed) => (), + Ok(Continuation::DropOut) => break Ok(()), + Err(err) => break Err(err), + }, + Err(err) => break Err(AttemptError::Recoverable(err.into())), + } + } +} + +#[derive(Debug)] +enum AttemptError { + Recoverable(Error), + Other(Error), +} + +fn do_handshake(channel: StdChannel, magic: u64) -> Result<(), AttemptError> { + let mut client = handshake::N2NClient::new(channel); + let versions = handshake::n2n::VersionTable::v4_and_above(magic); + + match client.handshake(versions) { + Ok(confirmation) => match confirmation { + handshake::Confirmation::Accepted(_, _) => Ok(()), + _ => Err(AttemptError::Other( + "couldn't agree on handshake version".into(), + )), + }, + Err(err) => Err(AttemptError::Recoverable(err.into())), + } +} + +fn do_chainsync_attempt( + config: &super::Config, + utils: Arc, + output_tx: &StageSender, +) -> Result<(), AttemptError> { + let magic = match config.magic.as_ref() { + Some(m) => *m.deref(), + None => MAINNET_MAGIC, + }; + + let mut plexer = setup_multiplexer(&config.address.0, &config.address.1, &config.retry_policy) + .map_err(|x| AttemptError::Recoverable(x))?; + + let hs_channel = plexer.use_channel(0); + let cs_channel = plexer.use_channel(2); + let bf_channel = plexer.use_channel(3); + + plexer.muxer.spawn(); + plexer.demuxer.spawn(); + + do_handshake(hs_channel, magic)?; + + let mut cs_client = chainsync::N2NClient::new(cs_channel); + + let intersection = intersect_starting_point( + &mut cs_client, + &config.intersect, + #[allow(deprecated)] + &config.since, + &utils, + ) + .map_err(|err| AttemptError::Recoverable(err))?; + + if intersection.is_none() { + return Err(AttemptError::Other( + "Can't find chain intersection point".into(), + )); + } + + log::info!("starting chain sync from: {:?}", &intersection); + + let bf_client = blockfetch::Client::new(bf_channel); + let writer = EventWriter::new(output_tx.clone(), utils, config.mapper.clone()); + + let (headers_tx, headers_rx) = std::sync::mpsc::sync_channel(100); + + let bf_writer = writer.clone(); + std::thread::spawn(move || { + fetch_blocks_forever(bf_client, bf_writer, headers_rx).expect("blockfetch loop failed"); + + log::info!("block fetch thread ended"); + }); + + // this will block + observe_headers_forever( + cs_client, + writer, + headers_tx, + config.min_depth, + config.finalize.clone(), + )?; + + Ok(()) +} + +pub fn do_chainsync( + config: &super::Config, + utils: Arc, + output_tx: StageSender, +) -> Result<(), Error> { + retry::retry_operation( + || match do_chainsync_attempt(config, utils.clone(), &output_tx) { + Ok(()) => Ok(()), + Err(AttemptError::Other(msg)) => { + log::error!("N2N error: {}", msg); + log::warn!("unrecoverable error performing chainsync, will exit"); + Ok(()) + } + Err(AttemptError::Recoverable(err)) => Err(err), + }, + &retry::Policy { + max_retries: config + .retry_policy + .as_ref() + .map(|x| x.chainsync_max_retries) + .unwrap_or(50), + backoff_unit: Duration::from_secs(1), + backoff_factor: 2, + max_backoff: config + .retry_policy + .as_ref() + .map(|x| x.chainsync_max_backoff as u64) + .map(Duration::from_secs) + .unwrap_or_else(|| Duration::from_secs(60)), + }, + ) +} diff --git a/src/sources/n2n/setup.rs b/src/sources/n2n/setup.rs new file mode 100644 index 00000000..a45f8612 --- /dev/null +++ b/src/sources/n2n/setup.rs @@ -0,0 +1,61 @@ +use serde::Deserialize; + +use crate::{ + mapper::Config as MapperConfig, + pipelining::{new_inter_stage_channel, PartialBootstrapResult, SourceProvider}, + sources::{ + common::{AddressArg, MagicArg, PointArg}, + FinalizeConfig, IntersectArg, RetryPolicy, + }, + utils::{ChainWellKnownInfo, WithUtils}, +}; + +use super::run::do_chainsync; + +#[derive(Debug, Deserialize, Clone)] +pub struct Config { + pub address: AddressArg, + + #[serde(deserialize_with = "crate::sources::common::deserialize_magic_arg")] + pub magic: Option, + + #[deprecated(note = "use intersect value instead")] + pub since: Option, + + pub intersect: Option, + + #[deprecated(note = "chain info is now pipeline-wide, use utils")] + pub well_known: Option, + + #[serde(default)] + pub mapper: MapperConfig, + + /// Min block depth (# confirmations) required + /// + /// The min depth a block requires to be considered safe to send down the + /// pipeline. This value is used to configure a rollback buffer used + /// internally by the stage. A high value (eg: ~6) will reduce the + /// probability of seeing rollbacks events. The trade-off is that the stage + /// will need some time to fill up the buffer before sending the 1st event. + #[serde(default)] + pub min_depth: usize, + + pub retry_policy: Option, + + pub finalize: Option, +} + +impl SourceProvider for WithUtils { + fn bootstrap(&self) -> PartialBootstrapResult { + let (output_tx, output_rx) = new_inter_stage_channel(None); + + let config = self.inner.clone(); + let utils = self.utils.clone(); + let handle = std::thread::spawn(move || { + do_chainsync(&config, utils, output_tx) + .expect("chainsync fails after applying max retry policy") + }); + + Ok((handle, output_rx)) + } +} diff --git a/src/sources/s3.rs b/src/sources/s3.rs deleted file mode 100644 index e83b69df..00000000 --- a/src/sources/s3.rs +++ /dev/null @@ -1,155 +0,0 @@ -use aws_sdk_s3::Client as S3Client; -use gasket::framework::*; -use gasket::messaging::SendPort; -use serde::Deserialize; - -use crate::framework::*; - -#[derive(Stage)] -#[stage(name = "source")] -pub struct Stage { - bucket: String, - items_per_batch: u32, - cursor: Cursor, - - retry_policy: gasket::retries::Policy, - - pub output: SourceOutputPort, - - #[metric] - ops_count: gasket::metrics::Counter, -} - -impl gasket::framework::Stage for Stage { - fn policy(&self) -> gasket::runtime::Policy { - gasket::runtime::Policy { - work_retry: self.retry_policy.clone(), - bootstrap_retry: self.retry_policy.clone(), - ..Default::default() - } - } -} - -pub struct Worker { - s3_client: S3Client, - last_key: String, -} - -pub struct KeyBatch { - keys: Vec, -} - -#[async_trait::async_trait(?Send)] -impl gasket::framework::Worker for Worker { - type Unit = KeyBatch; - type Stage = Stage; - - async fn bootstrap(stage: &Self::Stage) -> Result { - let sdk_config = aws_config::load_from_env().await; - let s3_client = aws_sdk_s3::Client::new(&sdk_config); - - let p = stage - .cursor - .latest_known_point() - .unwrap_or(pallas::network::miniprotocols::Point::Origin); - - let key = match p { - pallas::network::miniprotocols::Point::Origin => "origin".to_owned(), - pallas::network::miniprotocols::Point::Specific(slot, _) => format!("{slot}"), - }; - - Ok(Self { - s3_client, - last_key: key, - }) - } - - async fn schedule( - &mut self, - stage: &mut Self::Stage, - ) -> Result, WorkerError> { - let result = self - .s3_client - .list_objects_v2() - .bucket(&stage.bucket) - .max_keys(stage.items_per_batch as i32) - .start_after(self.last_key.clone()) - .send() - .await - .or_retry()?; - - let keys = result - .contents - .unwrap_or_default() - .into_iter() - .filter_map(|obj| obj.key) - .collect::>(); - - Ok(WorkSchedule::Unit(KeyBatch { keys })) - } - - async fn execute( - &mut self, - unit: &Self::Unit, - stage: &mut Self::Stage, - ) -> Result<(), WorkerError> { - for key in &unit.keys { - let object = self - .s3_client - .get_object() - .bucket(&stage.bucket) - .key(key) - .send() - .await - .or_retry()?; - - let metadata = object - .metadata - .ok_or("S3 object is missing metadata") - .or_panic()?; - let slot = metadata - .get("slot") - .ok_or("S3 object is missing block slot") - .or_panic()?; - let hash = metadata - .get("hash") - .ok_or("S3 object is missing block hash") - .or_panic()?; - - let point = pallas::network::miniprotocols::Point::Specific( - slot.parse().or_panic()?, - hex::decode(hash).or_panic()?, - ); - - let body = object.body.collect().await.or_retry()?; - - let event = ChainEvent::Apply(point, Record::CborBlock(body.into_bytes().to_vec())); - - stage.output_port.send(event.into()).await.or_panic()?; - } - - Ok(()) - } -} - -#[derive(Deserialize)] -pub struct Config { - bucket: String, - items_per_batch: u32, - retry_policy: gasket::retries::Policy, -} - -impl Config { - pub fn bootstrapper(self, ctx: &Context) -> Result { - let stage = Stage { - bucket: self.bucket, - items_per_batch: self.items_per_batch, - retry_policy: self.retry_policy, - cursor: ctx.cursor.clone(), - output_port: Default::default(), - ops_count: Default::default(), - }; - - Ok(stage) - } -} diff --git a/src/_utils/cursor.rs b/src/utils/cursor.rs similarity index 100% rename from src/_utils/cursor.rs rename to src/utils/cursor.rs diff --git a/src/_utils/facade.rs b/src/utils/facade.rs similarity index 100% rename from src/_utils/facade.rs rename to src/utils/facade.rs diff --git a/src/_utils/metrics.rs b/src/utils/metrics.rs similarity index 100% rename from src/_utils/metrics.rs rename to src/utils/metrics.rs diff --git a/src/utils/mod.rs b/src/utils/mod.rs new file mode 100644 index 00000000..3c0ff57c --- /dev/null +++ b/src/utils/mod.rs @@ -0,0 +1,261 @@ +//! Pipeline-wide utilities +//! +//! This module includes general-purpose utilities that could potentially be +//! used by more than a single stage. The entry point to this utilities is +//! designed as singleton [`Utils`] instance shared by all stages through an Arc +//! pointer. + +use std::sync::Arc; + +use pallas::network::miniprotocols::{Point, MAINNET_MAGIC, TESTNET_MAGIC}; + +// TODO: move these values to Pallas +pub const PREPROD_MAGIC: u64 = 1; +pub const PREVIEW_MAGIC: u64 = 2; + +pub const SANCHO_MAGIC: u64 = 4; + +use serde::{Deserialize, Serialize}; + +use crate::{model::Event, utils::time::NaiveProvider as NaiveTime}; + +use crate::Error; + +pub mod cursor; +pub mod metrics; +pub mod throttle; + +pub(crate) mod retry; +pub(crate) mod time; + +mod facade; + +pub(crate) trait SwallowResult { + fn ok_or_warn(self, context: &'static str); +} + +impl SwallowResult for Result<(), Error> { + fn ok_or_warn(self, context: &'static str) { + match self { + Ok(_) => (), + Err(e) => log::warn!("{}: {:?}", context, e), + } + } +} + +/// Well-known information about the blockhain network +/// +/// Some of the logic in Oura depends on particular characteristic of the +/// network that it's consuming from. For example: time calculation and bech32 +/// encoding. This struct groups all of these blockchain network specific +/// values. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ChainWellKnownInfo { + pub byron_epoch_length: u32, + pub byron_slot_length: u32, + pub byron_known_slot: u64, + pub byron_known_hash: String, + pub byron_known_time: u64, + pub shelley_epoch_length: u32, + pub shelley_slot_length: u32, + pub shelley_known_slot: u64, + pub shelley_known_hash: String, + pub shelley_known_time: u64, + pub address_hrp: String, + pub adahandle_policy: String, +} + +impl ChainWellKnownInfo { + /// Hardcoded values for mainnet + pub fn mainnet() -> Self { + ChainWellKnownInfo { + byron_epoch_length: 432000, + byron_slot_length: 20, + byron_known_slot: 0, + byron_known_time: 1506203091, + byron_known_hash: "f0f7892b5c333cffc4b3c4344de48af4cc63f55e44936196f365a9ef2244134f" + .to_string(), + shelley_epoch_length: 432000, + shelley_slot_length: 1, + shelley_known_slot: 4492800, + shelley_known_hash: "aa83acbf5904c0edfe4d79b3689d3d00fcfc553cf360fd2229b98d464c28e9de" + .to_string(), + shelley_known_time: 1596059091, + address_hrp: "addr".to_string(), + adahandle_policy: "f0ff48bbb7bbe9d59a40f1ce90e9e9d0ff5002ec48f232b49ca0fb9a" + .to_string(), + } + } + + /// Hardcoded values for testnet + pub fn testnet() -> Self { + ChainWellKnownInfo { + byron_epoch_length: 432000, + byron_slot_length: 20, + byron_known_slot: 0, + byron_known_time: 1564010416, + byron_known_hash: "8f8602837f7c6f8b8867dd1cbc1842cf51a27eaed2c70ef48325d00f8efb320f" + .to_string(), + shelley_epoch_length: 432000, + shelley_slot_length: 1, + shelley_known_slot: 1598400, + shelley_known_hash: "02b1c561715da9e540411123a6135ee319b02f60b9a11a603d3305556c04329f" + .to_string(), + shelley_known_time: 1595967616, + address_hrp: "addr_test".to_string(), + adahandle_policy: "8d18d786e92776c824607fd8e193ec535c79dc61ea2405ddf3b09fe3" + .to_string(), + } + } + + /// Hardcoded values for the "preview" testnet + pub fn preview() -> Self { + ChainWellKnownInfo { + byron_epoch_length: 432000, + byron_slot_length: 20, + byron_known_slot: 0, + byron_known_hash: "".to_string(), + byron_known_time: 1666656000, + shelley_epoch_length: 432000, + shelley_slot_length: 1, + shelley_known_slot: 0, + shelley_known_hash: "268ae601af8f9214804735910a3301881fbe0eec9936db7d1fb9fc39e93d1e37" + .to_string(), + shelley_known_time: 1666656000, + address_hrp: "addr_test".to_string(), + adahandle_policy: "".to_string(), + } + } + + /// Hardcoded values for the "pre-prod" testnet + pub fn preprod() -> Self { + ChainWellKnownInfo { + byron_epoch_length: 432000, + byron_slot_length: 20, + byron_known_slot: 0, + byron_known_hash: "9ad7ff320c9cf74e0f5ee78d22a85ce42bb0a487d0506bf60cfb5a91ea4497d2" + .to_string(), + byron_known_time: 1654041600, + shelley_epoch_length: 432000, + shelley_slot_length: 1, + shelley_known_slot: 86400, + shelley_known_hash: "c971bfb21d2732457f9febf79d9b02b20b9a3bef12c561a78b818bcb8b35a574" + .to_string(), + shelley_known_time: 1655769600, + address_hrp: "addr_test".to_string(), + adahandle_policy: "".to_string(), + } + } + pub fn sancho() -> Self { + ChainWellKnownInfo { + byron_epoch_length: 432000, + byron_slot_length: 20, + byron_known_slot: 0, + byron_known_hash: "9ad7ff320c9cf74e0f5ee78d22a85ce42bb0a487d0506bf60cfb5a91ea4497d2" + .to_string(), + byron_known_time: 1654041600, + shelley_epoch_length: 432000, + shelley_slot_length: 1, + shelley_known_slot: 86400, + shelley_known_hash: "c971bfb21d2732457f9febf79d9b02b20b9a3bef12c561a78b818bcb8b35a574" + .to_string(), + shelley_known_time: 1655769600, + address_hrp: "addr_test".to_string(), + adahandle_policy: "".to_string(), + } + } + + /// Try to identify the chain based on the specified magic value. + pub fn try_from_magic(magic: u64) -> Result { + match magic { + MAINNET_MAGIC => Ok(Self::mainnet()), + TESTNET_MAGIC => Ok(Self::testnet()), + PREVIEW_MAGIC => Ok(Self::preview()), + PREPROD_MAGIC => Ok(Self::preprod()), + SANCHO_MAGIC => Ok(Self::sancho()), + _ => Err(format!("can't identify chain from specified magic value: {magic}").into()), + } + } +} + +impl Default for ChainWellKnownInfo { + fn default() -> Self { + Self::mainnet() + } +} + +/// Entry point for all shared utilities +pub struct Utils { + pub(crate) well_known: ChainWellKnownInfo, + pub(crate) time: Option, + pub(crate) cursor: Option, + pub(crate) metrics: Option, +} + +// TODO: refactor this using the builder pattern +impl Utils { + pub fn new(well_known: ChainWellKnownInfo) -> Self { + Self { + time: NaiveTime::new(well_known.clone()).into(), + well_known, + cursor: None, + metrics: None, + } + } + + pub fn with_cursor(self, config: cursor::Config) -> Self { + let provider = cursor::Provider::initialize(config); + + Self { + cursor: provider.into(), + ..self + } + } + + pub fn with_metrics(self, config: metrics::Config) -> Self { + let provider = metrics::Provider::initialize(&config).expect("metric server started"); + + Self { + metrics: provider.into(), + ..self + } + } +} + +/// Wraps a struct with pipeline-wide utilities +/// +/// Most of the stage bootstrapping processes will require a custom config value +/// and a reference to the shared utilities singleton. This is a quality-of-life +/// artifact to wrap other structs (usually configs) and attach the utilities +/// singleton entrypoint. +#[derive(Clone)] +pub struct WithUtils { + pub utils: Arc, + pub inner: C, +} + +impl WithUtils { + pub fn new(inner: C, utils: Arc) -> Self { + WithUtils { inner, utils } + } + + pub fn attach_utils_to(&self, target: T) -> WithUtils { + WithUtils { + inner: target, + utils: self.utils.clone(), + } + } +} + +impl TryFrom for Point { + type Error = crate::Error; + + fn try_from(other: ChainWellKnownInfo) -> Result { + let out = Point::Specific( + other.shelley_known_slot, + hex::decode(other.shelley_known_hash)?, + ); + + Ok(out) + } +} diff --git a/src/_utils/retry.rs b/src/utils/retry.rs similarity index 100% rename from src/_utils/retry.rs rename to src/utils/retry.rs diff --git a/src/sinks/terminal/throttle.rs b/src/utils/throttle.rs similarity index 66% rename from src/sinks/terminal/throttle.rs rename to src/utils/throttle.rs index a37a18af..0da37e47 100644 --- a/src/sinks/terminal/throttle.rs +++ b/src/utils/throttle.rs @@ -3,8 +3,6 @@ use std::{ time::{Duration, Instant}, }; -const THROTTLE_MIN_SPAN_MILLIS: u64 = 300; - pub struct Throttle { last_action: Instant, min_delay: Duration, @@ -28,11 +26,3 @@ impl Throttle { self.last_action = Instant::now(); } } - -impl From> for Throttle { - fn from(value: Option) -> Self { - let millis = value.unwrap_or(THROTTLE_MIN_SPAN_MILLIS); - let duration = Duration::from_millis(millis); - Throttle::new(duration) - } -} diff --git a/src/_utils/time.rs b/src/utils/time.rs similarity index 100% rename from src/_utils/time.rs rename to src/utils/time.rs diff --git a/testdrive/cardano2gcp_pubsub/daemon.toml b/testdrive/cardano2gcp_pubsub/daemon.toml deleted file mode 100644 index 9ca789dd..00000000 --- a/testdrive/cardano2gcp_pubsub/daemon.toml +++ /dev/null @@ -1,18 +0,0 @@ -[source] -type = "N2N" -address = ["Tcp", "relays-new.cardano-mainnet.iohk.io:3001"] -magic = "mainnet" - -[source.finalize] -max_block_quantity = 10 - -[source.intersect] -type = "Point" -value = [ - 50030552, - "5a091e9831bc0fcd6e6bdb2877036bb260abb4e19195517e4153fb041f956df5", -] - -[sink] -type = "GcpPubSub" -topic = "oura-e2e"