diff --git a/blacklight-node/Cargo.toml b/blacklight-node/Cargo.toml index c2303cd..96a4868 100644 --- a/blacklight-node/Cargo.toml +++ b/blacklight-node/Cargo.toml @@ -11,6 +11,7 @@ attestation-verification = { git = "https://github.com/NillionNetwork/nilcc", re dcap-qvl = "0.3.4" clap = { version = "4.5", features = ["derive", "env", "string"] } futures-util = "0.3" +rand = "0.10.0" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } semver = "1.0" serde_json = "1.0" diff --git a/blacklight-node/src/supervisor/htx.rs b/blacklight-node/src/supervisor/htx.rs index 402d20a..caae2ad 100644 --- a/blacklight-node/src/supervisor/htx.rs +++ b/blacklight-node/src/supervisor/htx.rs @@ -14,6 +14,10 @@ use crate::supervisor::status::{check_minimum_balance, print_status}; use crate::supervisor::version::validate_node_version; use crate::verification::HtxVerifier; +const MAX_ATTEMPTS: u32 = 3; +const BACKOFF_TIME: std::time::Duration = std::time::Duration::from_secs(10); +const SPREAD: bool = true; + #[derive(Clone)] pub struct HtxProcessor { client: BlacklightClient, @@ -99,25 +103,15 @@ impl HtxProcessor { let htx_id = event.heartbeatKey; // Parse the HTX data - tries JSON first (nilCC/Phala), then ABI decoding (ERC-8004) let verification_result = match Htx::try_parse(&event.rawHTX) { - Ok(htx) => match htx { - Htx::Nillion(htx) => { - info!(htx_id = ?htx_id, "Detected nilCC HTX"); - self.verifier.verify_nillion_htx(&htx).await - } - Htx::Phala(htx) => { - info!(htx_id = ?htx_id, "Detected Phala HTX"); - self.verifier.verify_phala_htx(&htx).await - } - Htx::Erc8004(htx) => { - info!( - htx_id = ?htx_id, - agent_id = %htx.agent_id, - request_uri = %htx.request_uri, - "Detected ERC-8004 validation HTX" - ); - self.verifier.verify_erc8004_htx(&htx).await - } - }, + Ok(htx) => { + self.verifier + .retryable_verify(&htx) + .with_max_attempts(MAX_ATTEMPTS) + .with_backoff(BACKOFF_TIME) + .with_spread(SPREAD) + .execute() + .await + } Err(e) => { error!(htx_id = ?htx_id, error = %e, "Failed to parse HTX data"); // If we parse invalid data, it could be a malicious node, so Failure and it doesn't get rewarded diff --git a/blacklight-node/src/verification.rs b/blacklight-node/src/verification.rs index aa1293b..9314313 100644 --- a/blacklight-node/src/verification.rs +++ b/blacklight-node/src/verification.rs @@ -12,8 +12,9 @@ use attestation_verification::{ }; use attestation_verification::{VerificationError as ExtVerificationError, VmType}; use blacklight_contract_clients::heartbeat_manager::Verdict; -use blacklight_contract_clients::htx::{Erc8004Htx, NillionHtx, PhalaHtx}; +use blacklight_contract_clients::htx::{Erc8004Htx, Htx, NillionHtx, PhalaHtx}; use dcap_qvl::collateral::get_collateral_and_verify; +use rand::RngExt; use reqwest::Client; use sha2::{Digest, Sha256}; use std::path::{Path, PathBuf}; @@ -349,8 +350,120 @@ impl HtxVerifier { Ok(()) } + + /// Create a new `RetryableVerification` with the default values for a given HTX. + /// + /// # Arguments + /// + /// * `htx` - The HTX to verify. + /// + /// # Returns + /// + /// A new `RetryableVerification` with the default values. + pub fn retryable_verify<'a>(&'a self, htx: &'a Htx) -> RetryableVerification<'a> { + RetryableVerification::new(htx, self) + } +} + +pub struct RetryableVerification<'a> { + pub htx: &'a Htx, + pub verifier: &'a HtxVerifier, + pub max_attempts: u32, + pub backoff: std::time::Duration, + pub with_spread: bool, } +impl<'a> RetryableVerification<'a> { + /// Create a new `RetryableVerification` with the default values. + pub fn new(htx: &'a Htx, verifier: &'a HtxVerifier) -> Self { + Self { + htx, + verifier, + max_attempts: 1, + backoff: std::time::Duration::from_secs(10), + with_spread: false, + } + } + + /// Set the maximum number of attempts to make. + /// + /// # Arguments + /// + /// * `max_attempts` - The maximum number of attempts to make. + /// + /// # Returns + /// + /// A new `RetryableVerification` with the maximum number of attempts set. + pub fn with_max_attempts(mut self, max_attempts: u32) -> Self { + self.max_attempts = max_attempts.max(1); + self + } + + /// Set the backoff time between attempts. + /// + /// # Arguments + /// + /// * `backoff` - The backoff time between attempts. + /// + /// # Returns + /// + /// A new `RetryableVerification` with the backoff time set. + pub fn with_backoff(mut self, backoff: std::time::Duration) -> Self { + self.backoff = backoff; + self + } + + /// Add a random jitter to the backoff time to prevent thundering herd effect. + /// Helps preventing 429 Too Many Requests errors from the server by spreading out the requests. + /// + /// # Arguments + /// + /// * `spread` - Whether to add a random jitter to the backoff time. + /// + /// # Returns + /// + /// A new `RetryableVerification` with the spread enabled/disabled. + pub fn with_spread(mut self, spread: bool) -> Self { + self.with_spread = spread; + self + } + + /// Execute the verification. + /// + /// # Returns + /// + /// Ok(()) if the verification succeeds, Err(VerificationError) otherwise. + pub async fn execute(&self) -> Result<(), VerificationError> { + let mut last_err = None; + let max_attempts = self.max_attempts.max(1); + for attempt in 0..max_attempts { + let outcome = match self.htx { + Htx::Nillion(htx) => self.verifier.verify_nillion_htx(htx).await, + Htx::Phala(htx) => self.verifier.verify_phala_htx(htx).await, + Htx::Erc8004(htx) => self.verifier.verify_erc8004_htx(htx).await, + }; + + match outcome { + Ok(()) => return Ok(()), + Err(e) if e.verdict() == Verdict::Inconclusive => { + last_err = Some(e); + if attempt + 1 < max_attempts { + let delay = if self.with_spread { + let spread = rand::rng().random_range(0.5..1.5); + self.backoff.mul_f64(spread) + } else { + self.backoff + }; + tokio::time::sleep(delay).await; + } + continue; + } + Err(e) => return Err(e), + } + } + Err(last_err.expect("max_attempts must be >= 1")) + } +} #[derive(Default)] struct LockedDownloader(Mutex<()>);