build: run redis tests using a docker integration environment

.cargo/config.toml

@@ -1,2 +1,5 @@
 [alias]
 chainhook-install = "install --path components/chainhook-cli --locked --force --features cli --features debug --no-default-features"
+
+[env]
+RUST_TEST_THREADS = "1"

.github/workflows/ci.yaml

@@ -35,12 +35,6 @@ jobs:
           rustup toolchain install stable --profile minimal
           echo "RUST_VERSION_HASH=$(rustc --version | sha256sum | awk '{print $1}')" >> $GITHUB_ENV
 
-      - name: Install redis
-        if: matrix.suite == 'cli'
-        run: |
-          sudo apt-get install -y redis-server
-          echo "TARPAULIN_FLAGS=--features redis_tests" >> $GITHUB_ENV
-
       - name: Cache cargo
         uses: actions/cache@v4
         with:
@@ -59,6 +53,13 @@ jobs:
         run: |
           cargo install cargo-tarpaulin
 
+      - name: Setup integration environment
+        run: |
+          sudo ufw disable
+          docker compose -f ../../dockerfiles/docker-compose.dev.yml up -d
+          docker compose -f ../../dockerfiles/docker-compose.dev.yml logs -t -f --no-color &> docker-compose-logs.txt &
+        if: matrix.suite == 'cli'
+
       - name: Run tests
         run: |
           cargo tarpaulin --skip-clean --out lcov ${{ env.TARPAULIN_FLAGS }} -- --test-threads=1
@@ -69,6 +70,14 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           codecov_yml_path: .github/codecov.yml
 
+      - name: Print integration environment logs
+        run: cat docker-compose-logs.txt
+        if: matrix.suite == 'cli' && failure()
+
+      - name: Teardown integration environment
+        run: docker compose -f ../../dockerfiles/docker-compose.dev.yml down -v -t 0
+        if: matrix.suite == 'cli' && always()
+
   distributions:
     runs-on: ${{ matrix.os }}
 

.gitignore

@@ -19,6 +19,5 @@ components/chainhook-types-js/dist
 *.redb
 cache/
 Chainhook.toml
-
-components/chainhook-cli/src/service/tests/fixtures/tmp
-components/chainhook-cli/src/archive/tests/fixtures/tmp
+**/src/service/tests/fixtures/tmp
+**/src/archive/tests/fixtures/tmp

.vscode/launch.json

@@ -0,0 +1,47 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "lldb",
+      "request": "launch",
+      "name": "test: chainhook-sdk",
+      "cargo": {
+        "args": ["test", "--no-run", "--lib", "--package=chainhook-sdk"],
+        "filter": {
+          "name": "chainhook_sdk",
+          "kind": "lib"
+        }
+      },
+      "args": [],
+      "env": {
+        "RUST_TEST_THREADS": "1"
+      },
+      "cwd": "${workspaceFolder}"
+    },
+    {
+      "type": "lldb",
+      "request": "launch",
+      "name": "test: chainhook-cli",
+      "cargo": {
+        "args": [
+          "test",
+          "--no-run",
+          "--bin=chainhook",
+          "--package=chainhook",
+          "--features=redis_tests"
+        ],
+        "filter": {
+          "name": "chainhook",
+          "kind": "bin"
+        }
+      },
+      "args": [],
+      "env": {
+        "RUST_TEST_THREADS": "1"
+      },
+      "cwd": "${workspaceFolder}",
+      "preLaunchTask": "redis:start",
+      "postDebugTask": "redis:stop"
+    }
+  ]
+}

.vscode/tasks.json

@@ -0,0 +1,47 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "redis:start",
+      "type": "shell",
+      "command": "docker compose -f dockerfiles/docker-compose.dev.yml up --force-recreate -V",
+      "isBackground": true,
+      "problemMatcher": {
+        "pattern": { "regexp": ".", "file": 1, "location": 2, "message": 3 },
+        "background": {
+          "activeOnStart": true,
+          "beginsPattern": ".",
+          "endsPattern": "."
+        }
+      },
+      "presentation": {
+        "echo": true,
+        "reveal": "always",
+        "focus": false,
+        "panel": "dedicated",
+        "clear": false
+      }
+    },
+    {
+      "label": "redis:stop",
+      "type": "shell",
+      "command": "docker compose -f dockerfiles/docker-compose.dev.yml down -v -t 0",
+      "isBackground": true,
+      "problemMatcher": {
+        "pattern": { "regexp": ".", "file": 1, "location": 2, "message": 3 },
+        "background": {
+          "activeOnStart": true,
+          "beginsPattern": ".",
+          "endsPattern": "."
+        }
+      },
+      "presentation": {
+        "echo": true,
+        "reveal": "always",
+        "focus": false,
+        "panel": "dedicated",
+        "clear": false
+      }
+    }
+  ]
+}

components/chainhook-cli/src/service/tests/helpers/mock_service.rs

@@ -22,8 +22,6 @@ use reqwest::Method;
 use rocket::serde::json::Value as JsonValue;
 use rocket::Shutdown;
 use std::path::PathBuf;
-use std::process::Stdio;
-use std::process::{Child, Command};
 use std::sync::mpsc;
 use std::sync::mpsc::channel;
 use std::sync::mpsc::Receiver;
@@ -236,29 +234,24 @@ pub async fn build_predicate_api_server(port: u16) -> (Receiver<ObserverCommand>
     (rx, shutdown)
 }
 
-pub async fn start_redis(port: u16) -> Result<Child, String> {
-    let handle = Command::new("redis-server")
-        .arg(format!("--port {port}"))
-        .stdout(Stdio::null())
-        .spawn()
-        .map_err(|e| format!("failed to create start-redis command: {}", e))?;
+pub async fn wait_for_redis(port: u16) -> Result<(), String> {
     let mut attempts = 0;
     loop {
         match redis::Client::open(format!("redis://localhost:{port}/")) {
             Ok(client) => match client.get_connection() {
-                Ok(_) => return Ok(handle),
+                Ok(_) => return Ok(()),
                 Err(e) => {
                     attempts += 1;
                     if attempts == 10 {
-                        return Err(format!("failed to start redis service: {}", e));
+                        return Err(format!("failed to connect to redis service: {}", e));
                     }
                     tokio::time::sleep(std::time::Duration::from_secs(1)).await
                 }
             },
             Err(e) => {
                 attempts += 1;
                 if attempts == 10 {
-                    return Err(format!("failed to start redis service: {}", e));
+                    return Err(format!("failed to connect to redis service: {}", e));
                 }
                 tokio::time::sleep(std::time::Duration::from_secs(1)).await
             }
@@ -368,7 +361,6 @@ pub async fn start_chainhook_service(
 }
 
 pub struct TestSetupResult {
-    pub redis_process: Child,
     pub working_dir: String,
     pub chainhook_service_port: u16,
     pub redis_port: u16,
@@ -393,7 +385,7 @@ pub async fn setup_stacks_chainhook_test(
         prometheus_port,
     ) = setup_chainhook_service_ports().unwrap_or_else(|e| panic!("test failed with error: {e}"));
 
-    let mut redis_process = start_redis(redis_port)
+    wait_for_redis(redis_port)
         .await
         .unwrap_or_else(|e| panic!("test failed with error: {e}"));
     flush_redis(redis_port);
@@ -409,19 +401,16 @@ pub async fn setup_stacks_chainhook_test(
         let client = redis::Client::open(format!("redis://localhost:{redis_port}/"))
             .unwrap_or_else(|e| {
                 flush_redis(redis_port);
-                redis_process.kill().unwrap();
                 panic!("test failed with error: {e}");
             });
         let mut connection = client.get_connection().unwrap_or_else(|e| {
             flush_redis(redis_port);
-            redis_process.kill().unwrap();
             panic!("test failed with error: {e}");
         });
         let stacks_spec = predicate
             .into_specification_for_network(&StacksNetwork::Devnet)
             .unwrap_or_else(|e| {
                 flush_redis(redis_port);
-                redis_process.kill().unwrap();
                 panic!("test failed with error: {e}");
             });
 
@@ -432,14 +421,13 @@ pub async fn setup_stacks_chainhook_test(
 
     let (working_dir, tsv_dir) = create_tmp_working_dir().unwrap_or_else(|e| {
         flush_redis(redis_port);
-        redis_process.kill().unwrap();
         panic!("test failed with error: {e}");
     });
 
     write_stacks_blocks_to_tsv(starting_chain_tip, &tsv_dir).unwrap_or_else(|e| {
         std::fs::remove_dir_all(&working_dir).unwrap();
         flush_redis(redis_port);
-        redis_process.kill().unwrap();
+        // redis_process.kill().unwrap();
         panic!("test failed with error: {e}");
     });
 
@@ -459,7 +447,6 @@ pub async fn setup_stacks_chainhook_test(
         .unwrap_or_else(|e| {
             std::fs::remove_dir_all(&working_dir).unwrap();
             flush_redis(redis_port);
-            redis_process.kill().unwrap();
             panic!("test failed with error: {e}");
         });
 
@@ -469,11 +456,9 @@ pub async fn setup_stacks_chainhook_test(
             .unwrap_or_else(|e| {
                 std::fs::remove_dir_all(&working_dir).unwrap();
                 flush_redis(redis_port);
-                redis_process.kill().unwrap();
                 panic!("test failed with error: {e}");
             });
     TestSetupResult {
-        redis_process,
         working_dir,
         chainhook_service_port,
         redis_port,
@@ -495,14 +480,13 @@ pub async fn setup_bitcoin_chainhook_test(starting_chain_tip: u64) -> TestSetupR
         prometheus_port,
     ) = setup_chainhook_service_ports().unwrap_or_else(|e| panic!("test failed with error: {e}"));
 
-    let mut redis_process = start_redis(redis_port)
+    wait_for_redis(redis_port)
         .await
         .unwrap_or_else(|e| panic!("test failed with error: {e}"));
 
     flush_redis(redis_port);
     let (working_dir, tsv_dir) = create_tmp_working_dir().unwrap_or_else(|e| {
         flush_redis(redis_port);
-        redis_process.kill().unwrap();
         panic!("test failed with error: {e}");
     });
 
@@ -536,11 +520,9 @@ pub async fn setup_bitcoin_chainhook_test(starting_chain_tip: u64) -> TestSetupR
         .unwrap_or_else(|e| {
             std::fs::remove_dir_all(&working_dir).unwrap();
             flush_redis(redis_port);
-            redis_process.kill().unwrap();
             panic!("test failed with error: {e}");
         });
     TestSetupResult {
-        redis_process,
         working_dir,
         chainhook_service_port,
         redis_port,
@@ -553,7 +535,7 @@ pub async fn setup_bitcoin_chainhook_test(starting_chain_tip: u64) -> TestSetupR
 }
 
 pub fn setup_chainhook_service_ports() -> Result<(u16, u16, u16, u16, u16, u16), String> {
-    let redis_port = get_free_port()?;
+    let redis_port = 6379;
     let chainhook_service_port = get_free_port()?;
     let stacks_rpc_port = get_free_port()?;
     let stacks_ingestion_port = get_free_port()?;