feat: include signer messages in Stacks predicate payloads (#656)

Cargo.toml

@@ -8,4 +8,4 @@ default-members = ["components/chainhook-cli", "components/chainhook-sdk"]
 resolver = "2"
 
 [patch.crates-io]
-stacks-codec = { git = "https://github.com/hirosystems/clarinet.git", rev = "3a2f9136abd85b265e538fbe51c808e9c09a06cb" }
+stacks-codec = { git = "https://github.com/hirosystems/clarinet.git", rev = "b0683675115562d719ed4b5245f620e0990030a0" }

components/chainhook-cli/src/scan/stacks.rs

@@ -339,6 +339,8 @@ pub async fn scan_stacks_chainstate_via_rocksdb_using_predicate(
             chainhook: predicate_spec,
             apply: hits_per_blocks,
             rollback: vec![],
+            // TODO(rafaelcr): Query for non consensus events which fall between block timestamps to fill in here
+            events: vec![]
         };
         let res = match handle_stacks_hook_action(
             trigger,
@@ -533,6 +535,8 @@ pub async fn scan_stacks_chainstate_via_csv_using_predicate(
             chainhook: predicate_spec,
             apply: hits_per_blocks,
             rollback: vec![],
+            // TODO(rafaelcr): Consider StackerDB chunks that come from TSVs.
+            events: vec![]
         };
         match handle_stacks_hook_action(trigger, &proofs, &config.get_event_observer_config(), ctx)
         {

components/chainhook-cli/src/service/mod.rs

@@ -548,8 +548,9 @@ impl Service {
                             }
                             StacksChainEvent::ChainUpdatedWithMicroblocks(_)
                             | StacksChainEvent::ChainUpdatedWithMicroblocksReorg(_) => {},
-                            StacksChainEvent::ChainUpdatedWithStackerDbChunks(data) => {
+                            StacksChainEvent::ChainUpdatedWithNonConsensusEvents(data) => {
                                 // TODO(rafaelcr): Store signer data.
+                                println!("signer message: {:?}", data);
                             }
                         },
                         Err(e) => {
@@ -619,8 +620,8 @@ impl Service {
                             }
                             StacksChainEvent::ChainUpdatedWithMicroblocks(_)
                             | StacksChainEvent::ChainUpdatedWithMicroblocksReorg(_) => {},
-                            StacksChainEvent::ChainUpdatedWithStackerDbChunks(data) => {
-                                // TODO(rafaelcr): Send via HTTP payload.
+                            StacksChainEvent::ChainUpdatedWithNonConsensusEvents(_) => {
+                                // TODO(rafaelcr): Expire signer message predicates when appropriate
                             },
                         };
                         update_status_from_report(

components/chainhook-sdk/src/chainhooks/stacks/mod.rs

@@ -1,15 +1,15 @@
 use crate::observer::EventObserverConfig;
 use crate::utils::{AbstractStacksBlock, Context, MAX_BLOCK_HEIGHTS_ENTRIES};
 
+use super::types::validate_txid;
 use super::types::{
     append_error_context, BlockIdentifierIndexRule, ChainhookInstance, ExactMatchingRule,
     HookAction,
 };
-use super::types::validate_txid;
 use chainhook_types::{
-    BlockIdentifier, StacksChainEvent, StacksNetwork, StacksTransactionData,
-    StacksTransactionEvent, StacksTransactionEventPayload, StacksTransactionKind,
-    TransactionIdentifier,
+    BlockIdentifier, StacksChainEvent, StacksNetwork, StacksNonConsensusEventData,
+    StacksTransactionData, StacksTransactionEvent, StacksTransactionEventPayload,
+    StacksTransactionKind, TransactionIdentifier,
 };
 use clarity::codec::StacksMessageCodec;
 use clarity::vm::types::{
@@ -259,6 +259,8 @@ pub enum StacksPredicate {
     NftEvent(StacksNftEventBasedPredicate),
     StxEvent(StacksStxEventBasedPredicate),
     Txid(ExactMatchingRule),
+    #[cfg(feature = "stacks-signers")]
+    SignerMessage(StacksSignerMessagePredicate),
 }
 
 impl StacksPredicate {
@@ -307,11 +309,28 @@ impl StacksPredicate {
                     ));
                 }
             }
+            #[cfg(feature = "stacks-signers")]
+            StacksPredicate::SignerMessage(StacksSignerMessagePredicate::FromSignerPubKey(_)) => {
+                // TODO(rafaelcr): Validate pubkey format
+            }
+            #[cfg(feature = "stacks-signers")]
+            StacksPredicate::SignerMessage(StacksSignerMessagePredicate::AfterTimestamp(_)) => {}
         }
         Ok(())
     }
 }
 
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, JsonSchema)]
+#[serde(rename_all = "snake_case")]
+pub enum StacksSignerMessagePredicate {
+    AfterTimestamp(u64),
+    FromSignerPubKey(String),
+}
+
+impl StacksSignerMessagePredicate {
+    // TODO(rafaelcr): Write validators
+}
+
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, JsonSchema)]
 #[serde(rename_all = "snake_case")]
 pub struct StacksContractCallBasedPredicate {
@@ -457,6 +476,7 @@ pub struct StacksTriggerChainhook<'a> {
     pub chainhook: &'a StacksChainhookInstance,
     pub apply: Vec<(Vec<&'a StacksTransactionData>, &'a dyn AbstractStacksBlock)>,
     pub rollback: Vec<(Vec<&'a StacksTransactionData>, &'a dyn AbstractStacksBlock)>,
+    pub events: Vec<&'a StacksNonConsensusEventData>,
 }
 
 #[derive(Clone, Debug)]
@@ -480,21 +500,18 @@ pub struct StacksChainhookPayload {
 pub struct StacksChainhookOccurrencePayload {
     pub apply: Vec<StacksApplyTransactionPayload>,
     pub rollback: Vec<StacksRollbackTransactionPayload>,
+    pub events: Vec<StacksNonConsensusEventData>,
     pub chainhook: StacksChainhookPayload,
 }
 
 impl StacksChainhookOccurrencePayload {
-    pub fn from_trigger(
-        trigger: StacksTriggerChainhook<'_>,
-    ) -> StacksChainhookOccurrencePayload {
+    pub fn from_trigger(trigger: StacksTriggerChainhook<'_>) -> StacksChainhookOccurrencePayload {
         StacksChainhookOccurrencePayload {
             apply: trigger
                 .apply
                 .into_iter()
                 .map(|(transactions, block)| {
-                    let transactions = transactions
-                        .into_iter().cloned()
-                        .collect::<Vec<_>>();
+                    let transactions = transactions.into_iter().cloned().collect::<Vec<_>>();
                     StacksApplyTransactionPayload {
                         block_identifier: block.get_identifier().clone(),
                         transactions,
@@ -505,9 +522,7 @@ impl StacksChainhookOccurrencePayload {
                 .rollback
                 .into_iter()
                 .map(|(transactions, block)| {
-                    let transactions = transactions
-                        .into_iter().cloned()
-                        .collect::<Vec<_>>();
+                    let transactions = transactions.into_iter().cloned().collect::<Vec<_>>();
                     StacksRollbackTransactionPayload {
                         block_identifier: block.get_identifier().clone(),
                         transactions,
@@ -517,6 +532,7 @@ impl StacksChainhookOccurrencePayload {
             chainhook: StacksChainhookPayload {
                 uuid: trigger.chainhook.uuid.clone(),
             },
+            events: trigger.events.into_iter().cloned().collect::<Vec<_>>(),
         }
     }
 }
@@ -593,6 +609,7 @@ pub fn evaluate_stacks_chainhooks_on_chain_event<'a>(
                         chainhook,
                         apply,
                         rollback,
+                        events: vec![],
                     })
                 }
             }
@@ -621,6 +638,7 @@ pub fn evaluate_stacks_chainhooks_on_chain_event<'a>(
                         chainhook,
                         apply,
                         rollback,
+                        events: vec![],
                     })
                 }
             }
@@ -657,6 +675,7 @@ pub fn evaluate_stacks_chainhooks_on_chain_event<'a>(
                         chainhook,
                         apply,
                         rollback,
+                        events: vec![],
                     })
                 }
             }
@@ -718,13 +737,37 @@ pub fn evaluate_stacks_chainhooks_on_chain_event<'a>(
                         chainhook,
                         apply,
                         rollback,
+                        events: vec![],
                     })
                 }
             }
-        },
-        StacksChainEvent::ChainUpdatedWithStackerDbChunks(data) => {
-            // TODO: Support predicates to send this data
-        },
+        }
+        #[cfg(feature = "stacks-signers")]
+        StacksChainEvent::ChainUpdatedWithNonConsensusEvents(data) => {
+            if let Some(first_event) = data.events.first() {
+                for chainhook in active_chainhooks.iter() {
+                    evaluated_predicates
+                        .insert(chainhook.uuid.as_str(), &first_event.received_at_block);
+                    let (occurrences, mut expirations) =
+                        evaluate_stacks_predicate_on_non_consensus_events(
+                            &data.events,
+                            chainhook,
+                            ctx,
+                        );
+                    expired_predicates.append(&mut expirations);
+                    if occurrences.len() > 0 {
+                        triggered_predicates.push(StacksTriggerChainhook {
+                            chainhook,
+                            apply: vec![],
+                            rollback: vec![],
+                            events: occurrences,
+                        });
+                    }
+                }
+            }
+        }
+        #[cfg(not(feature = "stacks-signers"))]
+        StacksChainEvent::ChainUpdatedWithNonConsensusEvents(_) => {}
     }
     (
         triggered_predicates,
@@ -795,7 +838,45 @@ pub fn evaluate_stacks_predicate_on_block<'a>(
         | StacksPredicate::StxEvent(_)
         | StacksPredicate::PrintEvent(_)
         | StacksPredicate::Txid(_) => unreachable!(),
+        #[cfg(feature = "stacks-signers")]
+        StacksPredicate::SignerMessage(_) => false,
+    }
+}
+
+#[cfg(feature = "stacks-signers")]
+pub fn evaluate_stacks_predicate_on_non_consensus_events<'a>(
+    events: &'a Vec<StacksNonConsensusEventData>,
+    chainhook: &'a StacksChainhookInstance,
+    _ctx: &Context,
+) -> (
+    Vec<&'a StacksNonConsensusEventData>,
+    BTreeMap<&'a str, &'a BlockIdentifier>,
+) {
+    let mut occurrences = vec![];
+    let expired_predicates = BTreeMap::new();
+    for event in events {
+        match &chainhook.predicate {
+            StacksPredicate::SignerMessage(StacksSignerMessagePredicate::AfterTimestamp(
+                timestamp,
+            )) => {
+                if event.received_at_ms >= *timestamp {
+                    occurrences.push(event);
+                }
+            }
+            StacksPredicate::SignerMessage(StacksSignerMessagePredicate::FromSignerPubKey(_)) => {
+                // TODO(rafaelcr): Evaluate on pubkey
+            }
+            StacksPredicate::BlockHeight(_)
+            | StacksPredicate::ContractDeployment(_)
+            | StacksPredicate::ContractCall(_)
+            | StacksPredicate::FtEvent(_)
+            | StacksPredicate::NftEvent(_)
+            | StacksPredicate::StxEvent(_)
+            | StacksPredicate::PrintEvent(_)
+            | StacksPredicate::Txid(_) => unreachable!(),
+        };
     }
+    (occurrences, expired_predicates)
 }
 
 pub fn evaluate_stacks_predicate_on_transaction<'a>(
@@ -819,7 +900,7 @@ pub fn evaluate_stacks_predicate_on_transaction<'a>(
             _ => false,
         },
         StacksPredicate::ContractDeployment(StacksContractDeploymentPredicate::ImplementTrait(
-            stacks_trait,
+            _stacks_trait,
         )) => match &transaction.metadata.kind {
             StacksTransactionKind::ContractDeployment(_actual_deployment) => {
                 ctx.try_log(|logger| {
@@ -952,7 +1033,9 @@ pub fn evaluate_stacks_predicate_on_transaction<'a>(
         }
         StacksPredicate::PrintEvent(expected_event) => {
             for event in transaction.metadata.receipt.events.iter() {
-                if let StacksTransactionEventPayload::SmartContractEvent(actual) = &event.event_payload {
+                if let StacksTransactionEventPayload::SmartContractEvent(actual) =
+                    &event.event_payload
+                {
                     if actual.topic == "print" {
                         match expected_event {
                             StacksPrintEventBasedPredicate::Contains {
@@ -1006,9 +1089,29 @@ pub fn evaluate_stacks_predicate_on_transaction<'a>(
             txid.eq(&transaction.transaction_identifier.hash)
         }
         StacksPredicate::BlockHeight(_) => unreachable!(),
+        #[cfg(feature = "stacks-signers")]
+        StacksPredicate::SignerMessage(_) => false,
     }
 }
 
+fn serialize_stacks_non_consensus_event(
+    event: &StacksNonConsensusEventData,
+    _ctx: &Context,
+) -> serde_json::Value {
+    use chainhook_types::StacksNonConsensusEventPayloadData;
+
+    let payload = match &event.payload {
+        StacksNonConsensusEventPayloadData::SignerMessage(chunk) => {
+            json!({"type": "SignerMessage", "data": chunk})
+        }
+    };
+    json!({
+        "payload": payload,
+        "received_at": event.received_at_ms,
+        "received_at_block": event.received_at_block,
+    })
+}
+
 fn serialize_stacks_block(
     block: &dyn AbstractStacksBlock,
     transactions: Vec<&StacksTransactionData>,
@@ -1238,7 +1341,7 @@ pub fn serialized_decoded_clarity_value(hex_value: &str, ctx: &Context) -> serde
         Ok(bytes) => bytes,
         _ => return json!(hex_value.to_string()),
     };
-    
+
     match ClarityValue::consensus_deserialize(&mut Cursor::new(&value_bytes)) {
         Ok(value) => serialize_to_json(&value),
         Err(e) => {
@@ -1319,6 +1422,7 @@ pub fn serialize_stacks_payload_to_json<'a>(
         "rollback": trigger.rollback.into_iter().map(|(transactions, block)| {
             serialize_stacks_block(block, transactions, decode_clarity_values, include_contract_abi, ctx)
         }).collect::<Vec<_>>(),
+        "events": trigger.events.into_iter().map(|event| serialize_stacks_non_consensus_event(event, ctx)).collect::<Vec<_>>(),
         "chainhook": {
             "uuid": trigger.chainhook.uuid,
             "predicate": trigger.chainhook.predicate,

components/chainhook-sdk/src/chainhooks/tests/fixtures/stacks/testnet/occurrence.json

@@ -1211,5 +1211,6 @@
         }
       ]
     }
-  ]
+  ],
+  "events": []
 }

components/chainhook-sdk/src/chainhooks/tests/mod.rs

@@ -733,6 +733,7 @@ fn test_stacks_hook_action_noop() {
         chainhook: &chainhook,
         apply: vec![(apply_transactions, apply_blocks)],
         rollback: vec![(rollback_transactions, rollback_blocks)],
+        events: vec![]
     };
 
     let proofs = HashMap::new();
@@ -811,6 +812,7 @@ fn test_stacks_hook_action_file_append() {
         chainhook: &chainhook,
         apply,
         rollback: vec![(rollback_transactions, rollback_block)],
+        events: vec![]
     };
 
     let proofs = HashMap::new();