test: add tests for GET /v1/chainhooks/<uuid> route (#415)

### Description

Add tests for chainhook service, fix a few things along the way.


---

### Checklist

- [X] All tests pass
- [X] Tests added in this PR (if applicable)
Test coverage before: 37.72%
Test coverage after: 47.07%

.github/workflows/ci.yaml

@@ -22,10 +22,13 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
 
+      - name: Install redis
+        run: sudo apt-get install -y redis-server
+
       - name: Cargo test
         run: |
           rustup update
-          RUST_BACKTRACE=1 cargo test --all -- --test-threads=1
+          RUST_BACKTRACE=1 cargo test --all --features redis_tests -- --test-threads=1
 
       - name: Semantic Release
         uses: cycjimmy/semantic-release-action@v3

components/chainhook-cli/Cargo.toml

@@ -58,6 +58,7 @@ default = ["cli"]
 cli = ["clap", "clap_generate", "toml", "ctrlc", "hiro-system-kit/log"]
 debug = ["hiro-system-kit/debug"]
 release = ["hiro-system-kit/release"]
+redis_tests = []
 
 # [patch.crates-io]
 # raft-proto = { git = "https://github.com/tikv/raft-rs", rev="95c532612ee6a83591fce9a8b51d6afe87b58835"}

components/chainhook-cli/src/scan/bitcoin.rs

@@ -1,7 +1,7 @@
 use crate::config::{Config, PredicatesApi};
 use crate::service::{
-    open_readwrite_predicates_db_conn_or_panic, set_predicate_scanning_status,
-    set_unconfirmed_expiration_status, ScanningData,
+    open_readwrite_predicates_db_conn_or_panic, set_confirmed_expiration_status,
+    set_predicate_scanning_status, set_unconfirmed_expiration_status, ScanningData,
 };
 use chainhook_sdk::bitcoincore_rpc::RpcApi;
 use chainhook_sdk::bitcoincore_rpc::{Auth, Client};
@@ -14,6 +14,7 @@ use chainhook_sdk::indexer;
 use chainhook_sdk::indexer::bitcoin::{
     build_http_client, download_and_parse_block_with_retry, retrieve_block_hash_with_retry,
 };
+use chainhook_sdk::indexer::fork_scratch_pad::CONFIRMED_SEGMENT_MINIMUM_LENGTH;
 use chainhook_sdk::observer::{gather_proofs, EventObserverConfig};
 use chainhook_sdk::types::{
     BitcoinBlockData, BitcoinChainEvent, BitcoinChainUpdatedWithBlocksData, BlockIdentifier, Chain,
@@ -41,6 +42,9 @@ pub async fn scan_bitcoin_chainstate_via_rpc_using_predicate(
     let mut floating_end_block = false;
 
     let mut block_heights_to_scan = if let Some(ref blocks) = predicate_spec.blocks {
+        // todo: if a user provides a number of blocks where start_block + blocks > chain tip,
+        // the predicate will fail to scan all blocks. we should calculate a valid end_block and
+        // switch to streaming mode at some point
         BlockHeights::Blocks(blocks.clone()).get_sorted_entries()
     } else {
         let start_block = match predicate_spec.start_block {
@@ -107,7 +111,7 @@ pub async fn scan_bitcoin_chainstate_via_rpc_using_predicate(
             None => (number_of_blocks_to_scan, 0, 0u64),
         }
     };
-
+    let mut last_scanned_block_confirmations = 0;
     let http_client = build_http_client();
 
     while let Some(current_block_height) = block_heights_to_scan.pop_front() {
@@ -123,6 +127,7 @@ pub async fn scan_bitcoin_chainstate_via_rpc_using_predicate(
         let block_breakdown =
             download_and_parse_block_with_retry(&http_client, &block_hash, &bitcoin_config, ctx)
                 .await?;
+        last_scanned_block_confirmations = block_breakdown.confirmations;
         let block = match indexer::bitcoin::standardize_bitcoin_block(
             block_breakdown,
             &event_observer_config.bitcoin_network,
@@ -226,6 +231,9 @@ pub async fn scan_bitcoin_chainstate_via_rpc_using_predicate(
                     predicates_db_conn,
                     ctx,
                 );
+                if last_scanned_block_confirmations >= CONFIRMED_SEGMENT_MINIMUM_LENGTH {
+                    set_confirmed_expiration_status(&predicate_spec.key(), predicates_db_conn, ctx);
+                }
                 return Ok(true);
             }
         }

components/chainhook-cli/src/service/mod.rs

@@ -533,7 +533,7 @@ impl Service {
     }
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 #[serde(rename_all = "snake_case")]
 /// A high-level view of how `PredicateStatus` is used/updated can be seen here: docs/images/predicate-status-flowchart/PredicateStatusFlowchart.png.
 pub enum PredicateStatus {
@@ -545,7 +545,7 @@ pub enum PredicateStatus {
     New,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)]
 pub struct ScanningData {
     pub number_of_blocks_to_scan: u64,
     pub number_of_blocks_evaluated: u64,
@@ -554,7 +554,7 @@ pub struct ScanningData {
     pub last_evaluated_block_height: u64,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct StreamingData {
     pub last_occurrence: u128,
     pub last_evaluation: u128,
@@ -563,7 +563,7 @@ pub struct StreamingData {
     pub last_evaluated_block_height: u64,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct ExpiredData {
     pub number_of_blocks_evaluated: u64,
     pub number_of_times_triggered: u64,
@@ -844,6 +844,7 @@ pub fn set_unconfirmed_expiration_status(
     ctx: &Context,
 ) {
     let current_status = retrieve_predicate_status(&predicate_key, predicates_db_conn);
+    let mut previously_was_unconfirmed = false;
     let (
         number_of_blocks_evaluated,
         number_of_times_triggered,
@@ -882,15 +883,22 @@ pub fn set_unconfirmed_expiration_status(
                 last_occurrence,
                 last_evaluated_block_height: _,
                 expired_at_block_height,
-            }) => (
-                number_of_blocks_evaluated + number_of_new_blocks_evaluated,
-                number_of_times_triggered,
-                last_occurrence,
-                expired_at_block_height,
-            ),
-            PredicateStatus::Interrupted(_) | PredicateStatus::ConfirmedExpiration(_) => {
+            }) => {
+                previously_was_unconfirmed = true;
+                (
+                    number_of_blocks_evaluated + number_of_new_blocks_evaluated,
+                    number_of_times_triggered,
+                    last_occurrence,
+                    expired_at_block_height,
+                )
+            }
+            PredicateStatus::Interrupted(_) => {
                 unreachable!("unreachable predicate status: {:?}", status)
             }
+            PredicateStatus::ConfirmedExpiration(_) => {
+                warn!(ctx.expect_logger(), "Attempting to set UnconfirmedExpiration status when ConfirmedExpiration status has already been set for predicate {}", predicate_key);
+                return;
+            }
         },
         None => (0, 0, 0, 0),
     };
@@ -906,13 +914,16 @@ pub fn set_unconfirmed_expiration_status(
         predicates_db_conn,
         &ctx,
     );
-    insert_predicate_expiration(
-        chain,
-        expired_at_block_height,
-        predicate_key,
-        predicates_db_conn,
-        &ctx,
-    );
+    // don't insert this entry more than once
+    if !previously_was_unconfirmed {
+        insert_predicate_expiration(
+            chain,
+            expired_at_block_height,
+            predicate_key,
+            predicates_db_conn,
+            &ctx,
+        );
+    }
 }
 
 pub fn set_confirmed_expiration_status(

components/chainhook-cli/src/service/tests/helpers/build_predicates.rs

@@ -0,0 +1,71 @@
+use rocket::serde::json::Value as JsonValue;
+
+pub const DEFAULT_UUID: &str = "4ecc-4ecc-435b-9948-d5eeca1c3ce6";
+
+pub fn get_random_uuid() -> String {
+    let mut rng = rand::thread_rng();
+    let random_digit: u64 = rand::Rng::gen(&mut rng);
+    format!("test-uuid-{random_digit}")
+}
+
+pub fn build_bitcoin_payload(
+    network: Option<&str>,
+    if_this: Option<JsonValue>,
+    then_that: Option<JsonValue>,
+    filter: Option<JsonValue>,
+    uuid: Option<&str>,
+) -> JsonValue {
+    let network = network.unwrap_or("mainnet");
+    let if_this = if_this.unwrap_or(json!({"scope":"block"}));
+    let then_that = then_that.unwrap_or(json!("noop"));
+    let filter = filter.unwrap_or(json!({}));
+
+    let filter = filter.as_object().unwrap();
+    let mut network_val = json!({
+        "if_this": if_this,
+        "then_that": then_that
+    });
+    for (k, v) in filter.iter() {
+        network_val[k] = v.to_owned();
+    }
+    json!({
+        "chain": "bitcoin",
+        "uuid": uuid.unwrap_or(DEFAULT_UUID),
+        "name": "test",
+        "version": 1,
+        "networks": {
+            network: network_val
+        }
+    })
+}
+
+pub fn build_stacks_payload(
+    network: Option<&str>,
+    if_this: Option<JsonValue>,
+    then_that: Option<JsonValue>,
+    filter: Option<JsonValue>,
+    uuid: Option<&str>,
+) -> JsonValue {
+    let network = network.unwrap_or("mainnet");
+    let if_this = if_this.unwrap_or(json!({"scope":"txid", "equals": "0xfaaac1833dc4883e7ec28f61e35b41f896c395f8d288b1a177155de2abd6052f"}));
+    let then_that = then_that.unwrap_or(json!("noop"));
+    let filter = filter.unwrap_or(json!({}));
+
+    let filter = filter.as_object().unwrap();
+    let mut network_val = json!({
+        "if_this": if_this,
+        "then_that": then_that
+    });
+    for (k, v) in filter.iter() {
+        network_val[k] = v.to_owned();
+    }
+    json!({
+        "chain": "stacks",
+        "uuid": uuid.unwrap_or(DEFAULT_UUID),
+        "name": "test",
+        "version": 1,
+        "networks": {
+            network: network_val
+        }
+    })
+}