diff --git a/.github/workflows/node_test.yml b/.github/workflows/node_test.yml index a1015e9..8a8a21b 100644 --- a/.github/workflows/node_test.yml +++ b/.github/workflows/node_test.yml @@ -121,7 +121,7 @@ jobs: make compile-ttc-contract make create-schema export TTC_ADDRESS=$(make deploy | tee /dev/tty | tail -n 1) - TTC_ADDRESS="$TTC_ADDRESS" MAX_ACTORS=3 PROVER_TIMEOUT=1200 make run-node-tests + TTC_ADDRESS="$TTC_ADDRESS" NUM_ACTORS=3 PROVER_TIMEOUT=1200 make run-node-tests # Get short git SHA for tagging - name: Get short SHA diff --git a/Makefile b/Makefile index aeb4443..f7170d7 100644 --- a/Makefile +++ b/Makefile @@ -95,7 +95,7 @@ CHAIN_ID ?= 31337 OWNER_KEY ?= 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 MOCK_VERIFIER ?= false RISC0_DEV_MODE ?= true -MAX_ACTORS ?= 20 +NUM_ACTORS ?= 20 PROVER_TIMEOUT ?= 60 deploy-mock: ## Run node tests with mock verifier @@ -121,7 +121,7 @@ run-node-tests: ## Run node tests with mock verifier NODE_PORT=$(NODE_PORT) \ MONITOR_HOST=$(MONITOR_HOST) \ MONITOR_PORT=$(MONITOR_PORT) \ - MAX_ACTORS=$(MAX_ACTORS) \ + NUM_ACTORS=$(NUM_ACTORS) \ PROVER_TIMEOUT=$(PROVER_TIMEOUT) \ cargo run -p host --bin demo $(CARGO_BUILD_OPTIONS) -- e2e \ --chain-id $(CHAIN_ID) \ diff --git a/README.md b/README.md index 9fa9dd0..23fb6b2 100644 --- a/README.md +++ b/README.md @@ -11,25 +11,16 @@ Instead we create a pool, where the owners are asked to each rank the tokens in The [wikipedia article](https://en.wikipedia.org/wiki/Top_trading_cycle) does a good job explaining what the algorithm and setting is, and hints at various generalizations with links in the footnotes. ## Architecture -1. A Solidity smart contract capabale of - - holding NFTs in a custodial mannor (ideally with safe retrieval in case the participant wants to exit before completion) - - accepting trading preferences - - "locking down" for a period of time long enough to execute the trading algorithm off chain - - accepting and validating proofs for the results of the trading algorithm (a "re-allocation") - - allowing users to withdraw according to re-allocation -2. A rust implementation of the Top Trading Cycle (TTC) algorithm, and a compatibility layer for inputs/outputs expected by the contract. -3. Risc-Zero zkvm + [Steel](https://github.com/risc0/risc0-ethereum/tree/main/crates/steel) for generating Groth16 proofs of the TTC execution on smart contract data. -4. TODO: A server to monitor the contracts for proof requests / callbacks, and a gpu accelerated environment to construct the proofs. -5. TODO: A simple UI + testnet deployment for illustration purposes. +For architecture diagrams, see [here](./docs/README.md#architecture-diagram). For more details on the flows and service interaction, see [here](./docs/README.md#flows) -## Test against local node -The `host` crate contains an end-to-end test using a randomly generated allocation. See the [node_test](https://github.com/l-adic/ttc/blob/main/.github/workflows/node_test.yml) workflow for how you would set these up and run locally. +## Development Environment Setup -## Development Environment - -### Development Environment Setup +Two tmuxinator configurations are provided for development. Both configurations set up: +- Ethereum node and Postgres +- Prover and Monitor servers +- System monitoring (e.g htop, nvidia-smi) +- Command shell -Two tmuxinator configurations are provided for development: #### 1. Local Development (`.tmuxinator.yml`) Run services locally with cargo: @@ -43,25 +34,31 @@ Run all services in Docker containers: tmuxinator start -p .tmuxinator.docker.yml ``` -The Docker configuration automatically rebuilds the prover-server and monitor-server images before starting to ensure they reflect your current code. This is important when you've made changes to: -- Any Rust source files -- Cargo.toml dependencies -- Dockerfile configurations +## Testing +The `host` crate contains an end-to-end test using a randomly generated allocation. You can check the corresponding [github workflow](./.github/workflows/node_test.yml) for reference, +and view the [Makefile](./Makefile) for a complete set of config options + +1. Deploy the services (see [above](./README.md#development-environment-setup)) or use a hosted deployment. + +2. You must fetch the `ImageID.sol` contract and store it in the correct location. This contract is what cryptographically binds the solidity verifier to the rust TTC program. -If you need to manually rebuild the images: ```bash -docker compose build prover-server monitor-server +> make fetch-image-id-contract > contract/src/ImageID.sol ``` -Both configurations set up: -- Ethereum node and Postgres logs -- Prover and Monitor servers -- System monitoring (htop) -- Command shell +3. Deploy the contracts: + +``` +> make deploy +``` +NOTE: use the `deploy-mock` variant if you are running a mock verifier (recommended if you aren't using a cuda accelerated prover). +You should see the deploy address of the TTC contract printed to the console. There will be a json artifact written to `./deployments//deployed.json` + +4. Run the demo script with the desired config options, e.g. + +``` +> TTC_ADDRESS= NUM_ACTORS=10 PROVER_TIMEOUT=600 make run-node-tests +``` -#### Tmux Key Bindings -- Exit/detach from tmux: `Ctrl-b d` -- Switch between windows: `Ctrl-b [0-9]` or mouse click -- Switch between panes: `Ctrl-b arrow` or mouse click -- Scroll in a pane: Mouse wheel or `Ctrl-b [` then arrow keys (press `q` to exit scroll mode) -- Copy text: Select with mouse (may need to hold Shift depending on your terminal) +You can control the log level via `RUST_LOG`. This script creates checkpoints, writing the relevant state to `./deployments/`. If the script errors or halts at any +time, you can re-run from the last checkpoint using the same command as you used to start. \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index cacc928..79f2626 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,4 +1,4 @@ -## Architecture Diagram +# Architecture Diagram The TTC architecture consists of 2 hosted services (`Monitor` and `Prover`) and a shared Postgres database:

@@ -12,6 +12,8 @@ The `Monitor` runs on an extremely basic VM, the `Prover` runs on a GPU enabled (currently using an [L4](https://www.nvidia.com/en-us/data-center/l4/) instance on GCP) +# Flows + ## Deployment Flow. The `Operator` deploys the contracts and submits the address to the `Monitor` service. This spawns an event monitor loop looking for diff --git a/host/bin/demo.rs b/host/bin/demo.rs index 20d4379..9eb3d9e 100644 --- a/host/bin/demo.rs +++ b/host/bin/demo.rs @@ -6,6 +6,7 @@ use host::{ cli::{Command, DemoConfig}, contract::{nft::TestNFT, ttc::ITopTradingCycle}, env::{create_provider, init_console_subscriber}, + gas_metrics::{with_metrics, GasMetrics}, }; use jsonrpsee::http_client::{HttpClient, HttpClientBuilder}; use proptest::{ @@ -20,6 +21,7 @@ use risc0_steel::alloy::{ signers::local::PrivateKeySigner, }; use std::{collections::HashMap, path::Path, str::FromStr, thread::sleep, time::Duration}; +use tokio::sync::Mutex; use tracing::info; use ttc::strict::Preferences; use url::Url; @@ -33,6 +35,7 @@ struct TestSetup { monitor: HttpClient, timeout: Duration, checkpointer: Checkpointer, + gas_metrics: Mutex, } fn make_token_preferences( @@ -84,6 +87,7 @@ impl TestSetup { monitor, timeout: Duration::from_secs(config.prover_timeout), checkpointer, + gas_metrics: Mutex::new(GasMetrics::new()), }) } @@ -105,6 +109,7 @@ impl TestSetup { monitor, timeout: Duration::from_secs(config.prover_timeout), checkpointer, + gas_metrics: Mutex::new(GasMetrics::new()), }) } @@ -118,17 +123,29 @@ impl TestSetup { let nft = TestNFT::new(actor.token.collection, provider.clone()); let ttc = ITopTradingCycle::new(self.ttc, provider); async move { - nft.approve(self.ttc, actor.token.tokenId) + let approval_tx = nft + .approve(self.ttc, actor.token.tokenId) .send() .await? - .watch() + .get_receipt() .await?; - ttc.depositNFT(actor.token.clone()) + with_metrics(&self.gas_metrics, |m| { + m.inc_counter("approve"); + m.record_hist("approve", approval_tx.gas_used); + }) + .await; + let deposit_tx = ttc + .depositNFT(actor.token.clone()) .gas(self.config.base.max_gas) .send() .await? - .watch() + .get_receipt() .await?; + with_metrics(&self.gas_metrics, |m| { + m.inc_counter("approve"); + m.record_hist("approve", deposit_tx.gas_used); + }) + .await; Ok(()) } }) @@ -170,12 +187,18 @@ impl TestSetup { .map(|t| t.hash()) .collect::>(); async move { - ttc.setPreferences(actor.token.hash(), prefs.clone()) + let preferences_tx = ttc + .setPreferences(actor.token.hash(), prefs.clone()) .gas(self.config.base.max_gas) .send() .await? - .watch() + .get_receipt() .await?; + with_metrics(&self.gas_metrics, |m| { + m.inc_counter("setPreferences"); + m.record_hist("setPreferences", preferences_tx.gas_used); + }) + .await; let ps = ttc.getPreferences(actor.token.hash()).call().await?._0; assert_eq!(ps, prefs, "Preferences not set correctly in contract!"); info!( @@ -205,12 +228,18 @@ impl TestSetup { let provider = create_provider(self.node_url.clone(), self.owner.clone()); let ttc = ITopTradingCycle::new(self.ttc, provider); let journal_data = Bytes::from(proof.abi_encode()); - ttc.reallocateTokens(journal_data, Bytes::from(seal)) + let realloc_tx = ttc + .reallocateTokens(journal_data, Bytes::from(seal)) .gas(self.config.base.max_gas) .send() .await? - .watch() + .get_receipt() .await?; + with_metrics(&self.gas_metrics, |m| { + m.inc_counter("reallocateTokens"); + m.record_hist("reallocateTokens", realloc_tx.gas_used); + }) + .await; let stable: Vec = self .actors .iter() @@ -248,17 +277,23 @@ impl TestSetup { let provider = create_provider(self.node_url.clone(), actor.wallet.clone()); let ttc = ITopTradingCycle::new(self.ttc, provider.clone()); async move { - eprintln!( + info!( "Withdrawing token {:#} for existing owner {:#}", actor.token.hash(), actor.address() ); - ttc.withdrawNFT(actor.token.hash()) + let withdraw_tx = ttc + .withdrawNFT(actor.token.hash()) .gas(self.config.base.max_gas) .send() .await? - .watch() + .get_receipt() .await?; + with_metrics(&self.gas_metrics, |m| { + m.inc_counter("withdrawNFT"); + m.record_hist("withdrawNFT", withdraw_tx.gas_used); + }) + .await; Ok(()) } }) @@ -276,17 +311,23 @@ impl TestSetup { let provider = create_provider(self.node_url.clone(), actor.wallet.clone()); let ttc = ITopTradingCycle::new(self.ttc, provider.clone()); async move { - eprintln!( + info!( "Withdrawing token {:#} for new owner {:#}", new_token_hash, actor.address() ); - ttc.withdrawNFT(*new_token_hash) + let withdraw_tx = ttc + .withdrawNFT(*new_token_hash) .gas(self.config.base.max_gas) .send() .await? - .watch() + .get_receipt() .await?; + with_metrics(&self.gas_metrics, |m| { + m.inc_counter("withdrawNFT"); + m.record_hist("withdrawNFT", withdraw_tx.gas_used); + }) + .await; Ok(()) } }) @@ -329,12 +370,17 @@ impl TestSetup { async fn advance_phase(&self) -> Result<()> { let provider = create_provider(self.node_url.clone(), self.owner.clone()); let ttc = ITopTradingCycle::new(self.ttc, provider); - ttc.advancePhase().send().await?.watch().await?; + let advance_tx = ttc.advancePhase().send().await?.get_receipt().await?; + with_metrics(&self.gas_metrics, |m| { + m.inc_counter("advancePhase"); + m.record_hist("advancePhase", advance_tx.gas_used); + }) + .await; Ok(()) } } -async fn run_demo(setup: TestSetup) -> Result<()> { +async fn run_demo(setup: &TestSetup) -> Result<()> { let ttc = { let provider = create_provider(setup.node_url.clone(), setup.owner.clone()); ITopTradingCycle::new(setup.ttc, provider) @@ -430,8 +476,10 @@ async fn main() -> Result<()> { }; let test_case = { let mut runner = TestRunner::default(); - let strategy = (Preferences::::arbitrary_with(Some(2..=config.max_actors))) - .prop_map(|prefs| prefs.map(U256::from)); + let strategy = (Preferences::::arbitrary_with(Some( + config.num_actors..=config.num_actors, + ))) + .prop_map(|prefs| prefs.map(U256::from)); strategy.new_tree(&mut runner).unwrap().current() }; let setup = { @@ -447,7 +495,9 @@ async fn main() -> Result<()> { setup } }; - run_demo(setup).await + let res = run_demo(&setup).await; + info!("Metrics: {:}", &setup.gas_metrics.into_inner()); + res } Command::SubmitProof(config) => { info!("{}", serde_json::to_string_pretty(&config).unwrap()); diff --git a/host/src/cli.rs b/host/src/cli.rs index 5709408..f6d7ef1 100644 --- a/host/src/cli.rs +++ b/host/src/cli.rs @@ -60,8 +60,8 @@ pub struct DemoConfig { #[arg(long, env = "MONITOR_PORT", default_value = "3030")] pub monitor_port: String, - #[arg(long, env = "MAX_ACTORS", default_value_t = 10)] - pub max_actors: usize, + #[arg(long, env = "NUM_ACTORS", default_value_t = 10)] + pub num_actors: usize, /// Initial ETH balance for new accounts #[arg(long, env = "INITIAL_BALANCE", default_value = "5")] diff --git a/host/src/gas_metrics.rs b/host/src/gas_metrics.rs new file mode 100644 index 0000000..fcea7b9 --- /dev/null +++ b/host/src/gas_metrics.rs @@ -0,0 +1,79 @@ +use std::collections::HashMap; +use tokio::sync::Mutex; + +pub struct GasMetrics { + counter: HashMap, + histogram: HashMap>, +} + +impl GasMetrics { + pub fn new() -> Self { + Self { + counter: HashMap::new(), + histogram: HashMap::new(), + } + } + + pub fn inc_counter(&mut self, key: &str) { + let counter = self.counter.entry(key.to_string()).or_default(); + *counter += 1; + } + + pub fn record_hist(&mut self, key: &str, value: u64) { + let hist = self.histogram.entry(key.to_string()).or_default(); + hist.push(value); + } + + pub fn display(&self) { + println!("Gas Metrics:"); + } +} + +impl Default for GasMetrics { + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Display for GasMetrics { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.counter + .iter() + .try_for_each(|(key, value)| writeln!(f, "{}: {}", key, value))?; + + self.histogram.iter().try_for_each(|(key, hist)| { + let mean = if !hist.is_empty() { + hist.iter().sum::() / hist.len() as u64 + } else { + 0 + }; + + let median = if !hist.is_empty() { + let mut hist_clone = hist.clone(); + hist_clone.sort(); + hist_clone[hist_clone.len() / 2] + } else { + 0 + }; + + let max = hist.iter().max().unwrap_or(&0); + let min = hist.iter().min().unwrap_or(&0); + + writeln!( + f, + "{}: mean: {}, median: {}, max: {}, min: {}", + key, mean, median, max, min + ) + })?; + + Ok(()) + } +} + +pub async fn with_metrics(m: &Mutex, f: F) -> R +where + F: FnOnce(&mut GasMetrics) -> R, +{ + let mut metrics = m.lock().await; + f(&mut metrics) +} diff --git a/host/src/lib.rs b/host/src/lib.rs index b08fe9f..916141d 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -3,3 +3,4 @@ pub mod checkpoint; pub mod cli; pub mod contract; pub mod env; +pub mod gas_metrics;