feat(builder): retry on incorrect host block error (#61)

* feat: retry on incorrect host block error

* chore: comments

crates/builder/src/tasks/submit.rs

@@ -12,18 +12,28 @@ use alloy::signers::Signer;
 use alloy::sol_types::SolCall;
 use alloy::transports::TransportError;
 use alloy_primitives::{FixedBytes, U256};
-use eyre::{bail, eyre};
+use alloy_sol_types::SolError;
+use eyre::eyre;
 use oauth2::{
     basic::BasicClient, basic::BasicTokenType, reqwest::http_client, AuthUrl, ClientId,
     ClientSecret, EmptyExtraTokenFields, StandardTokenResponse, TokenResponse, TokenUrl,
 };
 use tokio::{sync::mpsc, task::JoinHandle};
 use tracing::{debug, error, instrument, trace};
-use zenith_types::{SignRequest, SignResponse, Zenith};
+use zenith_types::{
+    SignRequest, SignResponse,
+    Zenith::{self, IncorrectHostBlock},
+};
 
 /// OAuth Audience Claim Name, required param by IdP for client credential grant
 const OAUTH_AUDIENCE_CLAIM: &str = "audience";
 
+pub enum ControlFlow {
+    Retry,
+    Skip,
+    Done,
+}
+
 /// Submits sidecars in ethereum txns to mainnet ethereum
 pub struct SubmitTask {
     /// Ethereum Provider
@@ -128,7 +138,7 @@ impl SubmitTask {
         &self,
         resp: &SignResponse,
         in_progress: &InProgressBlock,
-    ) -> eyre::Result<()> {
+    ) -> eyre::Result<ControlFlow> {
         let v: u8 = resp.sig.v().y_parity_byte() + 27;
         let r: FixedBytes<32> = resp.sig.r().into();
         let s: FixedBytes<32> = resp.sig.s().into();
@@ -157,7 +167,11 @@ impl SubmitTask {
                 "error in transaction submission"
             );
 
-            bail!("simulation failed, bailing transaction submission")
+            if e.as_revert_data() == Some(IncorrectHostBlock::SELECTOR.into()) {
+                return Ok(ControlFlow::Retry);
+            }
+
+            return Ok(ControlFlow::Skip);
         }
 
         tracing::debug!(
@@ -166,7 +180,13 @@ impl SubmitTask {
             "sending transaction to network"
         );
 
-        let result = self.provider.send_transaction(tx).await?;
+        let result = match self.provider.send_transaction(tx).await {
+            Ok(result) => result,
+            Err(e) => {
+                error!(error = %e, "error sending transaction");
+                return Ok(ControlFlow::Skip);
+            }
+        };
 
         let tx_hash = result.tx_hash();
 
@@ -177,13 +197,19 @@ impl SubmitTask {
             "dispatched to network"
         );
 
-        Ok(())
+        Ok(ControlFlow::Done)
     }
 
     #[instrument(skip_all, err)]
-    async fn handle_inbound(&self, in_progress: &InProgressBlock) -> eyre::Result<()> {
+    async fn handle_inbound(&self, in_progress: &InProgressBlock) -> eyre::Result<ControlFlow> {
         tracing::info!(txns = in_progress.len(), "handling inbound block");
-        let sig_request = self.construct_sig_request(in_progress).await?;
+        let sig_request = match self.construct_sig_request(in_progress).await {
+            Ok(sig_request) => sig_request,
+            Err(e) => {
+                tracing::error!(error = %e, "error constructing signature request");
+                return Ok(ControlFlow::Skip);
+            }
+        };
 
         tracing::debug!(
             host_block_number = %sig_request.host_block_number,
@@ -201,7 +227,13 @@ impl SubmitTask {
             );
             SignResponse { req: sig_request, sig }
         } else {
-            let resp: SignResponse = self.sup_quincey(&sig_request).await?;
+            let resp: SignResponse = match self.sup_quincey(&sig_request).await {
+                Ok(resp) => resp,
+                Err(e) => {
+                    tracing::error!(error = %e, "error acquiring signature from quincey");
+                    return Ok(ControlFlow::Retry);
+                }
+            };
             tracing::debug!(
                 sig = hex::encode(resp.sig.as_bytes()),
                 "acquired signature from quincey"
@@ -218,8 +250,33 @@ impl SubmitTask {
         let handle = tokio::spawn(async move {
             loop {
                 if let Some(in_progress) = inbound.recv().await {
-                    if let Err(e) = self.handle_inbound(&in_progress).await {
-                        error!(%e, "error in block submission. Dropping block.");
+                    let mut retries = 0;
+                    loop {
+                        match self.handle_inbound(&in_progress).await {
+                            Ok(ControlFlow::Retry) => {
+                                retries += 1;
+                                if retries > 3 {
+                                    tracing::error!(
+                                        "error handling inbound block: too many retries"
+                                    );
+                                    break;
+                                }
+                                tracing::error!("error handling inbound block: retrying");
+                                tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
+                            }
+                            Ok(ControlFlow::Skip) => {
+                                tracing::info!("skipping block");
+                                break;
+                            }
+                            Ok(ControlFlow::Done) => {
+                                tracing::info!("block landed successfully");
+                                break;
+                            }
+                            Err(e) => {
+                                tracing::error!(error = %e, "error handling inbound block");
+                                break;
+                            }
+                        }
                     }
                 } else {
                     tracing::debug!("upstream task gone");