feat: simpler verification retry to avoid inconclusives

blacklight-node/Cargo.toml

@@ -11,6 +11,7 @@ attestation-verification = { git = "https://github.com/NillionNetwork/nilcc", re
 dcap-qvl = "0.3.4"
 clap = { version = "4.5", features = ["derive", "env", "string"] }
 futures-util = "0.3"
+rand = "0.10.0"
 reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
 semver = "1.0"
 serde_json = "1.0"

blacklight-node/src/supervisor/htx.rs

@@ -14,6 +14,10 @@ use crate::supervisor::status::{check_minimum_balance, print_status};
 use crate::supervisor::version::validate_node_version;
 use crate::verification::HtxVerifier;
 
+const MAX_ATTEMPTS: u32 = 3;
+const BACKOFF_TIME: std::time::Duration = std::time::Duration::from_secs(10);
+const SPREAD: bool = true;
+
 #[derive(Clone)]
 pub struct HtxProcessor {
     client: BlacklightClient,
@@ -99,25 +103,15 @@ impl HtxProcessor {
         let htx_id = event.heartbeatKey;
         // Parse the HTX data - tries JSON first (nilCC/Phala), then ABI decoding (ERC-8004)
         let verification_result = match Htx::try_parse(&event.rawHTX) {
-            Ok(htx) => match htx {
-                Htx::Nillion(htx) => {
-                    info!(htx_id = ?htx_id, "Detected nilCC HTX");
-                    self.verifier.verify_nillion_htx(&htx).await
-                }
-                Htx::Phala(htx) => {
-                    info!(htx_id = ?htx_id, "Detected Phala HTX");
-                    self.verifier.verify_phala_htx(&htx).await
-                }
-                Htx::Erc8004(htx) => {
-                    info!(
-                        htx_id = ?htx_id,
-                        agent_id = %htx.agent_id,
-                        request_uri = %htx.request_uri,
-                        "Detected ERC-8004 validation HTX"
-                    );
-                    self.verifier.verify_erc8004_htx(&htx).await
-                }
-            },
+            Ok(htx) => {
+                self.verifier
+                    .retryable_verify(&htx)
+                    .with_max_attempts(MAX_ATTEMPTS)
+                    .with_backoff(BACKOFF_TIME)
+                    .with_spread(SPREAD)
+                    .execute()
+                    .await
+            }
             Err(e) => {
                 error!(htx_id = ?htx_id, error = %e, "Failed to parse HTX data");
                 // If we parse invalid data, it could be a malicious node, so Failure and it doesn't get rewarded

blacklight-node/src/verification.rs

@@ -12,8 +12,9 @@ use attestation_verification::{
 };
 use attestation_verification::{VerificationError as ExtVerificationError, VmType};
 use blacklight_contract_clients::heartbeat_manager::Verdict;
-use blacklight_contract_clients::htx::{Erc8004Htx, NillionHtx, PhalaHtx};
+use blacklight_contract_clients::htx::{Erc8004Htx, Htx, NillionHtx, PhalaHtx};
 use dcap_qvl::collateral::get_collateral_and_verify;
+use rand::RngExt;
 use reqwest::Client;
 use sha2::{Digest, Sha256};
 use std::path::{Path, PathBuf};
@@ -349,8 +350,120 @@ impl HtxVerifier {
 
         Ok(())
     }
+
+    /// Create a new `RetryableVerification` with the default values for a given HTX.
+    ///
+    /// # Arguments
+    ///
+    /// * `htx` - The HTX to verify.
+    ///
+    /// # Returns
+    ///
+    /// A new `RetryableVerification` with the default values.
+    pub fn retryable_verify<'a>(&'a self, htx: &'a Htx) -> RetryableVerification<'a> {
+        RetryableVerification::new(htx, self)
+    }
+}
+
+pub struct RetryableVerification<'a> {
+    pub htx: &'a Htx,
+    pub verifier: &'a HtxVerifier,
+    pub max_attempts: u32,
+    pub backoff: std::time::Duration,
+    pub with_spread: bool,
 }
 
+impl<'a> RetryableVerification<'a> {
+    /// Create a new `RetryableVerification` with the default values.
+    pub fn new(htx: &'a Htx, verifier: &'a HtxVerifier) -> Self {
+        Self {
+            htx,
+            verifier,
+            max_attempts: 1,
+            backoff: std::time::Duration::from_secs(10),
+            with_spread: false,
+        }
+    }
+
+    /// Set the maximum number of attempts to make.
+    ///
+    /// # Arguments
+    ///
+    /// * `max_attempts` - The maximum number of attempts to make.
+    ///
+    /// # Returns
+    ///
+    /// A new `RetryableVerification` with the maximum number of attempts set.
+    pub fn with_max_attempts(mut self, max_attempts: u32) -> Self {
+        self.max_attempts = max_attempts.max(1);
+        self
+    }
+
+    /// Set the backoff time between attempts.
+    ///
+    /// # Arguments
+    ///
+    /// * `backoff` - The backoff time between attempts.
+    ///
+    /// # Returns
+    ///
+    /// A new `RetryableVerification` with the backoff time set.
+    pub fn with_backoff(mut self, backoff: std::time::Duration) -> Self {
+        self.backoff = backoff;
+        self
+    }
+
+    /// Add a random jitter to the backoff time to prevent thundering herd effect.
+    /// Helps preventing 429 Too Many Requests errors from the server by spreading out the requests.
+    ///
+    /// # Arguments
+    ///
+    /// * `spread` - Whether to add a random jitter to the backoff time.
+    ///
+    /// # Returns
+    ///
+    /// A new `RetryableVerification` with the spread enabled/disabled.
+    pub fn with_spread(mut self, spread: bool) -> Self {
+        self.with_spread = spread;
+        self
+    }
+
+    /// Execute the verification.
+    ///
+    /// # Returns
+    ///
+    /// Ok(()) if the verification succeeds, Err(VerificationError) otherwise.
+    pub async fn execute(&self) -> Result<(), VerificationError> {
+        let mut last_err = None;
+        let max_attempts = self.max_attempts.max(1);
+        for attempt in 0..max_attempts {
+            let outcome = match self.htx {
+                Htx::Nillion(htx) => self.verifier.verify_nillion_htx(htx).await,
+                Htx::Phala(htx) => self.verifier.verify_phala_htx(htx).await,
+                Htx::Erc8004(htx) => self.verifier.verify_erc8004_htx(htx).await,
+            };
+
+            match outcome {
+                Ok(()) => return Ok(()),
+                Err(e) if e.verdict() == Verdict::Inconclusive => {
+                    last_err = Some(e);
+                    if attempt + 1 < max_attempts {
+                        let delay = if self.with_spread {
+                            let spread = rand::rng().random_range(0.5..1.5);
+                            self.backoff.mul_f64(spread)
+                        } else {
+                            self.backoff
+                        };
+                        tokio::time::sleep(delay).await;
+                    }
+                    continue;
+                }
+                Err(e) => return Err(e),
+            }
+        }
+        Err(last_err.expect("max_attempts must be >= 1"))
+    }
+}
 #[derive(Default)]
 struct LockedDownloader(Mutex<()>);