Merges and fixing the pipeline

.github/workflows/release.yml

@@ -1,15 +1,9 @@
 name: Release
 on:
- push:
- tags:
- - "v*"
-
-env:
- REGISTRY: ghcr.io
- IMAGE_NAME: ${{ github.repository }}
+ release:
+ types: [published]
 
 jobs:
- # Compiles binaries using a bulder matrix
  binary_build:
  name: Build release binaries
  strategy:
@@ -111,51 +105,6 @@ jobs:
  name: ${{ matrix.name }}
  path: ${{ matrix.name }}
 
- # Creates a GitHub Container package using multi-arch docker builds
- container_build:
- runs-on: ubuntu-latest
-
- permissions:
- contents: read
- packages: write
-
- steps:
- - name: Checkout
- uses: actions/checkout@v3
-
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v2
-
- - name: Extract metadata (tags, labels) for Docker
- id: meta
- uses: docker/metadata-action@v4
- with:
- images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
- tags: |
- type=raw,value=latest,enable={{is_default_branch}}
- type=semver,pattern=v{{major}}
- type=semver,pattern=v{{major}}.{{minor}}
- type=semver,pattern=v{{version}}
-
- - name: Login to DockerHub
- uses: docker/login-action@v2
- with:
- registry: ${{ env.REGISTRY }}
- username: ${{ github.actor }}
- password: ${{ secrets.GITHUB_TOKEN }}
-
- - name: Build and push
- uses: docker/build-push-action@v4
- with:
- context: .
- # temporarly removing arm platform until we resolve the ca-certificates issue
- #platforms: linux/arm64,linux/amd64
- platforms: linux/amd64
- push: true
- tags: ${{ steps.meta.outputs.tags }}
- labels: ${{ steps.meta.outputs.labels }}
-
- # Create GitHub release with Rust build targets and release notes
  github_release:
  name: Create GitHub Release
  needs: [binary_build, container_build]
@@ -188,3 +137,13 @@ jobs:
  files: oura-*/oura-*
  body_path: RELEASE.md
  draft: true
+
+ - name: Deploying to Amazon S3
+ uses: dvelasquez/deploy-s3-action@main
+ with:
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+ AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }}
+ BUCKET_PATH: "/" 
+ DIST_LOCATION_CODE: oura-*/oura-*

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "oura"
 description = "The tail of Cardano"
-version = "2.0.0-alpha.2"
+version = "1.8.1"
 edition = "2021"
 repository = "https://github.com/txpipe/oura"
 homepage = "https://github.com/txpipe/oura"
@@ -12,22 +12,13 @@ authors = ["Santiago Carmuega <[email protected]>"]
 
 
 [dependencies]
-# pallas = "0.19.0-alpha.0"
-# pallas = { path = "../pallas/pallas" }
-pallas = { git = "https://github.com/txpipe/pallas" }
-
-gasket = { version = "^0.4", features = ["derive"] }
-# gasket = { path = "../../construkts/gasket-rs/gasket", features = ["derive"] }
-# gasket = { git = "https://github.com/construkts/gasket-rs.git", features = ["derive"] }
-
-utxorpc-spec-ledger = "1.0.0-alpha.0"
-# utxorpc-spec-ledger = { path = "../../utxorpc/spec/ledger/gen/rust" }
-# utxorpc-spec-ledger = { git = "https://github.com/utxorpc/spec.git" }
-
+pallas = { git = "https://github.com/Emurgo/pallas", tag = "conway_2" }
+# pallas = { git = "https://github.com/txpipe/pallas" }
+# pallas = { path = "../../pallas/pallas" }
 hex = "0.4.3"
 net2 = "0.2.37"
 bech32 = "0.9.1"
-clap = { version = "4.2.7", features = ["derive"] }
+clap = "3.2.22"
 log = "0.4.17"
 env_logger = "0.10.0"
 crossterm = "0.25"
@@ -38,20 +29,17 @@ config = { version = "0.13.2", default-features = false, features = [
  "json",
 ] }
 serde = { version = "1.0.152", features = ["derive"] }
-serde_json = { version = "1.0.89" }
+serde_json = { version = "1.0.89", features = ["arbitrary_precision"] }
 strum = "0.24"
 strum_macros = "0.24"
 prometheus_exporter = { version = "0.8.5", default-features = false }
 unicode-truncate = "0.2.0"
-thiserror = "1.0.39"
-indicatif = "0.17.3"
-lazy_static = "1.4.0"
-tracing = "0.1.37"
-tracing-subscriber = "0.3.17"
-file-rotate = "0.7.1"
-tokio = { version = "1", features = ["rt"] }
-async-trait = "0.1.68"
-reqwest = { version = "0.11", features = ["json"] }
+
+# feature logs
+file-rotate = { version = "0.7.1", optional = true }
+
+# feature: webhook
+reqwest = { version = "0.11", optional = true, features = ["blocking", "json"] }
 
 # feature: kafkasink
 kafka = { version = "0.8.0", optional = true }
@@ -68,11 +56,14 @@ aws-sdk-sqs = { version = "0.14.0", optional = true }
 aws-sdk-lambda = { version = "0.14.0", optional = true }
 aws-sdk-s3 = { version = "0.14.0", optional = true }
 
+# features: elasticsearch || aws
+tokio = { version = "1.24.2", optional = true, features = ["rt"] }
+
 # required for CI to complete successfully
 openssl = { version = "0.10", optional = true, features = ["vendored"] }
 
 # redis support
-redis = { version = "0.23.0", optional = true, features = ["tokio-comp"] }
+redis = { version = "0.21.6", optional = true, features = ["tokio-comp"] }
 
 # features: gcp
 google-cloud-pubsub = { version = "0.12.0", optional = true }
@@ -81,11 +72,15 @@ google-cloud-googleapis = { version = "0.7.0", optional = true }
 # features: rabbitmqsink
 lapin = { version = "2.1.1", optional = true }
 
-# features: deno
-deno_core = { version = "0.175.0", optional = true }
-serde_v8 = { version = "0.86.0", optional = true }
-deno_runtime = { version = "0.101.0", optional = true }
-
 [features]
-default = ["deno"]
-deno = ["deno_core", "deno_runtime", "serde_v8"]
+default = []
+web = ["reqwest"]
+logs = ["file-rotate"]
+webhook = ["web"]
+kafkasink = ["kafka", "openssl"]
+elasticsink = ["elasticsearch", "tokio"]
+fingerprint = ["murmur3"]
+aws = ["aws-config", "aws-sdk-sqs", "aws-sdk-lambda", "aws-sdk-s3", "tokio"]
+redissink = ["redis", "tokio"]
+gcp = ["google-cloud-pubsub", "google-cloud-googleapis", "tokio", "web"]
+rabbitmqsink = ["lapin", "tokio"]

README.md

@@ -8,9 +8,6 @@
  <hr/>
 </div>
 
-> **Warning**
-> `main` branch is now tracking **V2**. This new version is a complete overhaul of the processing pipeline, multiple breaking changes. If you're looking for **V1**, you can switch to the long-term support branch named `lts/v1`
-
 ## Introduction
 
 We have tools to "explore" the Cardano blockchain, which are useful when you know what you're looking for. We argue that there's a different, complementary use-case which is to "observe" the blockchain and react to particular event patterns.
@@ -35,7 +32,7 @@ In this terminal recording we get to see a few mins of live output from a testne
 
 All the heavy lifting required to communicate with the Cardano node is done by the [Pallas](https://github.com/txpipe/pallas) library, which provides an implementation of the Ouroboros multiplexer and a few of the required mini-protocol state-machines (ChainSync and LocalState in particular).
 
-The data pipeline is implemented by the [Gasket](https://github.com/construkts/gasket-rs) library which provides a framework for building staged, event-driven applications. Under this abstraction, each component of the pipeline (aka: _Stage_) runs in its own thread and communicates with other stages by sending messages (very similar to the _Actor pattern_).
+The data pipeline makes heavy use (maybe a bit too much) of multi-threading and mpsc channels provided by Rust's `std::sync` library.
 
 ## Use Cases
 
@@ -49,7 +46,7 @@ Similar to the well-known db-sync tool provided by IOHK, _Oura_ can be used as a
 
 Given its small memory / cpu footprint, _Oura_ can be deployed side-by-side with your Cardano node even in resource-constrained environments, such as Raspberry PIs.
 
-### As A Trigger Of Custom Actions
+### As a Trigger of Custom Actions
 
 _Oura_ running in `daemon` mode can be configured to use custom filters to pinpoint particular transaction patterns and trigger actions whenever it finds a match. For example: send an email when a particular policy / asset combination appears in a transaction; call an AWS Lambda function when a wallet delegates to a particular pool; send a http-call to a webhook each time a metadata key appears in the TX payload;
 
@@ -69,46 +66,53 @@ Oura is in its essence just a pipeline for processing events. Each stage of the
 
 ## Feature Status
 
-- Data Types
- - CBOR blocks
- - CBOR txs
- - Oura v1 model (for backward-compatibility)
- - Parsed Txs (structured objects with all tx data)
- - Generic JSON (any kind of JSON values)
 - Sources
- - chain-sync from local node
- - chain-sync + block-fetch from remote relay node
- - S3 bucket with block data
- - Kafka topic with block data 
+ - [x] chain-sync full-block (node-to-client)
+ - [x] chain-sync + block-fetch (node-to-node)
+ - [x] Parsing of Shelley-compatible blocks (Shelley, Allegra, Mary, Alonzo)
+ - [x] Parsing of Byron blocks
 - Sinks
- - Kafka topic
- - Elasticsearch index / data stream
- - Rotating log files with compression
- - Redis streams
- - AWS SQS queue
- - AWS Lambda call
- - AWS S3 objects
- - GCP PubSub
- - GCP Cloud Function
- - Azure Sinks
- - webhook (http post)
- - terminal (append-only, tail-like)
+ - [x] Kafka topic
+ - [x] Elasticsearch index / data stream
+ - [x] Rotating log files with compression
+ - [x] Redis streams
+ - [x] AWS SQS queue
+ - [x] AWS Lambda call
+ - [x] AWS S3 objects
+ - [ ] GCP Sinks
+ - [ ] Azure Sinks
+ - [x] webhook (http post)
+ - [x] terminal (append-only, tail-like)
+ - [x] RabbitMQ
+- Events / Parsers
+ - [x] block events (start, end)
+ - [x] transaction events (inputs, outputs, assets)
+ - [x] metadata events (labels, content)
+ - [x] mint events (policy, asset, quantity)
+ - [x] pool registrations events
+ - [x] delegation events
+ - [x] CIP-25 metadata parser (image, files)
+ - [ ] CIP-15 metadata parser
 - Filters
- - Parse block / tx CBOR 
- - Split block into txs
- - Select Txs by matching rules (address, metadata, policies, etc)
- - Enrich tx data with related inputs
- - Custom Typescript code (uses Deno)
- - Custom WASM plugin
- - Rollback buffer with compensating actions
+ - [x] cherry pick by event type (block, tx, mint, cert, etc)
+ - [x] cherry pick by asset subject (policy, name, etc)
+ - [x] cherry pick by metadata keys
+ - [ ] cherry pick by block property (size, tx count)
+ - [ ] cherry pick by tx property (fee, has native script, has plutus script, etc)
+ - [ ] cherry pick by utxo property (address, asset, amount range)
+ - [ ] enrich events with policy info from external metadata service
+ - [ ] enrich input tx info from Blockfrost API
+ - [ ] enrich addresses descriptions using ADAHandle
 - Other
- - stateful chain cursor to recover from restarts
- - buffer stage to hold blocks until they reach a certain depth
- - pipeline metrics to track the progress and performance
+ - [x] stateful chain cursor to recover from restarts
+ - [x] buffer stage to hold blocks until they reach a certain depth
+ - [x] pipeline metrics to track the progress and performance
 
 ## Known Limitations
 
-- Oura reads events from minted blocks / transactions. Support for querying the mempool is not yet implemented.
+- ~~Oura only knows how to process blocks from the Shelley era. We are working on adding support for Byron in a future release.~~ (available since v1.2)
+- Oura reads events from minted blocks / transactions. Support for querying the mempool is planned for a future release.
+- ~~Oura will notify about chain rollbacks as a new event. The business logic for "undoing" the already processed events is a responsability of the consumer. We're working on adding support for a "buffer" filter stage which can hold blocks until they reach a configurable depth (number of confirmations).~~ (rollback buffer available since v1.2)
 
 ## Contributing