From 3bee58c02d2dba15af5b8c44cc0430719050a9fe Mon Sep 17 00:00:00 2001 From: jalbrekt85 Date: Mon, 14 Jul 2025 22:35:54 -0500 Subject: [PATCH 1/5] etherscan class, new get_block_by_timestamp method --- .env.example | 1 + bal_tools/etherscan.py | 80 ++++++++++++++++++++++++++++++++++++++++++ bal_tools/subgraph.py | 47 +++++++++++++++++++------ tests/test_subgraph.py | 25 +++++++++++++ 4 files changed, 142 insertions(+), 11 deletions(-) create mode 100644 bal_tools/etherscan.py diff --git a/.env.example b/.env.example index 1ec24e2..138c679 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,4 @@ ETHNODEURL= DRPC_KEY= GRAPH_API_KEY= +ETHERSCAN_API_KEY= diff --git a/bal_tools/etherscan.py b/bal_tools/etherscan.py new file mode 100644 index 0000000..b8c500e --- /dev/null +++ b/bal_tools/etherscan.py @@ -0,0 +1,80 @@ +import os +import time +from typing import Optional, Dict, Any +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +from .utils import chain_ids_by_name + + +class EtherscanV2Client: + BASE_URL = "https://api.etherscan.io/v2/api" + + def __init__(self, api_key: Optional[str] = None): + self.api_key = api_key or os.getenv("ETHERSCAN_API_KEY") + if not self.api_key: + raise ValueError("Etherscan API key required. Set ETHERSCAN_API_KEY environment variable or pass api_key parameter") + + self.session = requests.Session() + retry_strategy = Retry( + total=3, + backoff_factor=0.5, + status_forcelist=[429, 500, 502, 503, 504], + ) + adapter = HTTPAdapter(max_retries=retry_strategy) + self.session.mount("http://", adapter) + self.session.mount("https://", adapter) + + self.last_request_time = 0 + self.rate_limit_delay = 0.2 # 200ms + + def _get_chain_id(self, chain: str) -> int: + chain_ids = chain_ids_by_name() + if chain not in chain_ids: + raise ValueError(f"Unsupported chain: {chain}. Supported chains: {list(chain_ids.keys())}") + return chain_ids[chain] + + def _rate_limit(self): + current_time = time.time() + time_since_last_request = current_time - self.last_request_time + if time_since_last_request < self.rate_limit_delay: + time.sleep(self.rate_limit_delay - time_since_last_request) + self.last_request_time = time.time() + + def _make_request(self, params: Dict[str, Any]) -> Dict[str, Any]: + self._rate_limit() + + params["apikey"] = self.api_key + + response = self.session.get(self.BASE_URL, params=params, timeout=30) + response.raise_for_status() + + data = response.json() + + if data.get("status") == "0" and data.get("message") != "No records found": + raise Exception(f"Etherscan API error: {data.get('message', 'Unknown error')}") + + return data + + def get_block_by_timestamp(self, chain: str, timestamp: int, closest: str = "before") -> Optional[int]: + chain_id = self._get_chain_id(chain) + + params = { + "chainid": chain_id, + "module": "block", + "action": "getblocknobytime", + "timestamp": timestamp, + "closest": closest + } + + try: + data = self._make_request(params) + + if data.get("status") == "1" and data.get("result"): + return int(data["result"]) + + return None + + except Exception as e: + raise Exception(f"Error fetching block for timestamp {timestamp} on {chain}: {str(e)}") \ No newline at end of file diff --git a/bal_tools/subgraph.py b/bal_tools/subgraph.py index 2d2f37c..99ce24a 100644 --- a/bal_tools/subgraph.py +++ b/bal_tools/subgraph.py @@ -16,6 +16,7 @@ from .models import * from .errors import NoPricesFoundError from .ts_config_loader import ts_config_loader +from .etherscan import EtherscanV2Client def url_dict_from_df(df): @@ -72,6 +73,7 @@ def __init__(self, chain: str = "mainnet", silence_warnings: bool = False): if silence_warnings: self.set_silence_warnings(True) self.custom_price_logic: Dict[str, Callable] = {} + self.etherscan_client = None def set_silence_warnings(self, silence_warnings: bool): if silence_warnings: @@ -325,20 +327,43 @@ def fetch_graphql_data( return result - def get_first_block_after_utc_timestamp(self, timestamp: int) -> int: + def get_first_block_after_utc_timestamp(self, timestamp: int, use_etherscan: bool = True) -> int: if timestamp > int(datetime.now().strftime("%s")): timestamp = int(datetime.now().strftime("%s")) - 2000 - data = self.fetch_graphql_data( - "blocks", - "first_block_after_ts", - { - "timestamp_gt": int(timestamp) - 200, - "timestamp_lt": int(timestamp) + 200, - }, - ) - data["blocks"].sort(key=lambda x: x["timestamp"], reverse=True) - return int(data["blocks"][0]["number"]) + if use_etherscan: + try: + if not self.etherscan_client: + self.etherscan_client = EtherscanV2Client() + + block_number = self.etherscan_client.get_block_by_timestamp( + chain=self.chain, + timestamp=timestamp, + closest="after" + ) + + if block_number: + return block_number + + except Exception as e: + warnings.warn( + f"Etherscan V2 block fetch failed for chain {self.chain}: {str(e)}. Falling back to subgraph.", + UserWarning + ) + + try: + data = self.fetch_graphql_data( + "blocks", + "first_block_after_ts", + { + "timestamp_gt": int(timestamp) - 200, + "timestamp_lt": int(timestamp) + 200, + }, + ) + data["blocks"].sort(key=lambda x: x["timestamp"], reverse=True) + return int(data["blocks"][0]["number"]) + except Exception as e: + raise Exception(f"Failed to fetch block for timestamp {timestamp} on {self.chain}: {str(e)}") def get_twap_price_token( self, diff --git a/tests/test_subgraph.py b/tests/test_subgraph.py index 51191d1..71e12a1 100644 --- a/tests/test_subgraph.py +++ b/tests/test_subgraph.py @@ -3,6 +3,8 @@ import json import warnings import time +import os +from datetime import datetime, timedelta from bal_tools.subgraph import Subgraph, GqlChain, Pool, PoolSnapshot from bal_tools.errors import NoPricesFoundError @@ -170,3 +172,26 @@ def test_get_pool_protocol_version(subgraph): ) == 2 ) + + +def test_get_first_block_after_utc_timestamp_with_etherscan(chain, subgraph_all_chains, chains_prod): + if not os.getenv("ETHERSCAN_API_KEY"): + pytest.skip("ETHERSCAN_API_KEY not set") + + if chain not in chains_prod or chain in ["fantom", "sonic"]: + pytest.skip(f"Skipping {chain}") + + test_timestamp = int((datetime.now() - timedelta(days=1)).timestamp()) + + try: + block = subgraph_all_chains.get_first_block_after_utc_timestamp( + test_timestamp, + use_etherscan=True + ) + assert isinstance(block, int) + assert block > 0 + except Exception as e: + if "Unsupported chain" in str(e) or "Error fetching block" in str(e): + pytest.skip(f"Chain {chain} not supported by Etherscan V2") + else: + raise From 209f41bfb66cd46230d73624c0777ae6c4853b79 Mon Sep 17 00:00:00 2001 From: jalbrekt85 <33009898+jalbrekt85@users.noreply.github.com> Date: Tue, 15 Jul 2025 03:38:04 +0000 Subject: [PATCH 2/5] style: ci lint with `black` --- bal_tools/etherscan.py | 54 +++++++++++++++++++++++++----------------- bal_tools/subgraph.py | 22 +++++++++-------- tests/test_subgraph.py | 13 +++++----- 3 files changed, 51 insertions(+), 38 deletions(-) diff --git a/bal_tools/etherscan.py b/bal_tools/etherscan.py index b8c500e..d92d079 100644 --- a/bal_tools/etherscan.py +++ b/bal_tools/etherscan.py @@ -10,12 +10,14 @@ class EtherscanV2Client: BASE_URL = "https://api.etherscan.io/v2/api" - + def __init__(self, api_key: Optional[str] = None): self.api_key = api_key or os.getenv("ETHERSCAN_API_KEY") if not self.api_key: - raise ValueError("Etherscan API key required. Set ETHERSCAN_API_KEY environment variable or pass api_key parameter") - + raise ValueError( + "Etherscan API key required. Set ETHERSCAN_API_KEY environment variable or pass api_key parameter" + ) + self.session = requests.Session() retry_strategy = Retry( total=3, @@ -25,56 +27,64 @@ def __init__(self, api_key: Optional[str] = None): adapter = HTTPAdapter(max_retries=retry_strategy) self.session.mount("http://", adapter) self.session.mount("https://", adapter) - + self.last_request_time = 0 self.rate_limit_delay = 0.2 # 200ms - + def _get_chain_id(self, chain: str) -> int: chain_ids = chain_ids_by_name() if chain not in chain_ids: - raise ValueError(f"Unsupported chain: {chain}. Supported chains: {list(chain_ids.keys())}") + raise ValueError( + f"Unsupported chain: {chain}. Supported chains: {list(chain_ids.keys())}" + ) return chain_ids[chain] - + def _rate_limit(self): current_time = time.time() time_since_last_request = current_time - self.last_request_time if time_since_last_request < self.rate_limit_delay: time.sleep(self.rate_limit_delay - time_since_last_request) self.last_request_time = time.time() - + def _make_request(self, params: Dict[str, Any]) -> Dict[str, Any]: self._rate_limit() - + params["apikey"] = self.api_key - + response = self.session.get(self.BASE_URL, params=params, timeout=30) response.raise_for_status() - + data = response.json() - + if data.get("status") == "0" and data.get("message") != "No records found": - raise Exception(f"Etherscan API error: {data.get('message', 'Unknown error')}") - + raise Exception( + f"Etherscan API error: {data.get('message', 'Unknown error')}" + ) + return data - - def get_block_by_timestamp(self, chain: str, timestamp: int, closest: str = "before") -> Optional[int]: + + def get_block_by_timestamp( + self, chain: str, timestamp: int, closest: str = "before" + ) -> Optional[int]: chain_id = self._get_chain_id(chain) - + params = { "chainid": chain_id, "module": "block", "action": "getblocknobytime", "timestamp": timestamp, - "closest": closest + "closest": closest, } try: data = self._make_request(params) - + if data.get("status") == "1" and data.get("result"): return int(data["result"]) - + return None - + except Exception as e: - raise Exception(f"Error fetching block for timestamp {timestamp} on {chain}: {str(e)}") \ No newline at end of file + raise Exception( + f"Error fetching block for timestamp {timestamp} on {chain}: {str(e)}" + ) diff --git a/bal_tools/subgraph.py b/bal_tools/subgraph.py index 99ce24a..c362884 100644 --- a/bal_tools/subgraph.py +++ b/bal_tools/subgraph.py @@ -327,7 +327,9 @@ def fetch_graphql_data( return result - def get_first_block_after_utc_timestamp(self, timestamp: int, use_etherscan: bool = True) -> int: + def get_first_block_after_utc_timestamp( + self, timestamp: int, use_etherscan: bool = True + ) -> int: if timestamp > int(datetime.now().strftime("%s")): timestamp = int(datetime.now().strftime("%s")) - 2000 @@ -335,22 +337,20 @@ def get_first_block_after_utc_timestamp(self, timestamp: int, use_etherscan: boo try: if not self.etherscan_client: self.etherscan_client = EtherscanV2Client() - + block_number = self.etherscan_client.get_block_by_timestamp( - chain=self.chain, - timestamp=timestamp, - closest="after" + chain=self.chain, timestamp=timestamp, closest="after" ) - + if block_number: return block_number - + except Exception as e: warnings.warn( f"Etherscan V2 block fetch failed for chain {self.chain}: {str(e)}. Falling back to subgraph.", - UserWarning + UserWarning, ) - + try: data = self.fetch_graphql_data( "blocks", @@ -363,7 +363,9 @@ def get_first_block_after_utc_timestamp(self, timestamp: int, use_etherscan: boo data["blocks"].sort(key=lambda x: x["timestamp"], reverse=True) return int(data["blocks"][0]["number"]) except Exception as e: - raise Exception(f"Failed to fetch block for timestamp {timestamp} on {self.chain}: {str(e)}") + raise Exception( + f"Failed to fetch block for timestamp {timestamp} on {self.chain}: {str(e)}" + ) def get_twap_price_token( self, diff --git a/tests/test_subgraph.py b/tests/test_subgraph.py index 71e12a1..d8b9124 100644 --- a/tests/test_subgraph.py +++ b/tests/test_subgraph.py @@ -174,19 +174,20 @@ def test_get_pool_protocol_version(subgraph): ) -def test_get_first_block_after_utc_timestamp_with_etherscan(chain, subgraph_all_chains, chains_prod): +def test_get_first_block_after_utc_timestamp_with_etherscan( + chain, subgraph_all_chains, chains_prod +): if not os.getenv("ETHERSCAN_API_KEY"): pytest.skip("ETHERSCAN_API_KEY not set") - + if chain not in chains_prod or chain in ["fantom", "sonic"]: pytest.skip(f"Skipping {chain}") - + test_timestamp = int((datetime.now() - timedelta(days=1)).timestamp()) - + try: block = subgraph_all_chains.get_first_block_after_utc_timestamp( - test_timestamp, - use_etherscan=True + test_timestamp, use_etherscan=True ) assert isinstance(block, int) assert block > 0 From 4dd2cf4a422bad855f9aebbb88ad6b0893dbe7b2 Mon Sep 17 00:00:00 2001 From: jalbrekt85 Date: Mon, 14 Jul 2025 22:41:19 -0500 Subject: [PATCH 3/5] update class name --- bal_tools/etherscan.py | 2 +- bal_tools/subgraph.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bal_tools/etherscan.py b/bal_tools/etherscan.py index d92d079..db37bf6 100644 --- a/bal_tools/etherscan.py +++ b/bal_tools/etherscan.py @@ -8,7 +8,7 @@ from .utils import chain_ids_by_name -class EtherscanV2Client: +class Etherscan: BASE_URL = "https://api.etherscan.io/v2/api" def __init__(self, api_key: Optional[str] = None): diff --git a/bal_tools/subgraph.py b/bal_tools/subgraph.py index c362884..eead922 100644 --- a/bal_tools/subgraph.py +++ b/bal_tools/subgraph.py @@ -16,7 +16,7 @@ from .models import * from .errors import NoPricesFoundError from .ts_config_loader import ts_config_loader -from .etherscan import EtherscanV2Client +from .etherscan import Etherscan def url_dict_from_df(df): @@ -336,7 +336,7 @@ def get_first_block_after_utc_timestamp( if use_etherscan: try: if not self.etherscan_client: - self.etherscan_client = EtherscanV2Client() + self.etherscan_client = Etherscan() block_number = self.etherscan_client.get_block_by_timestamp( chain=self.chain, timestamp=timestamp, closest="after" From 7faf01b54fba7e33c301eaf7da8ed468804a325c Mon Sep 17 00:00:00 2001 From: jalbrekt85 Date: Tue, 15 Jul 2025 12:55:27 -0500 Subject: [PATCH 4/5] add etherscan env var --- .github/workflows/python_package.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python_package.yaml b/.github/workflows/python_package.yaml index 18cc40b..a5d5a66 100644 --- a/.github/workflows/python_package.yaml +++ b/.github/workflows/python_package.yaml @@ -27,6 +27,7 @@ jobs: ETHNODEURL: ${{ secrets.ETHNODEURL }} DRPC_KEY: ${{ secrets.DRPC_KEY }} GRAPH_API_KEY: ${{ secrets.GRAPH_API_KEY }} + ETHERSCAN_API_KEY: ${{ secrets.ETHERSCAN_API_KEY }} run: | pip install -r bal_tools/requirements-dev.txt pytest -x From d9db9fe78e86a57c8889a9a914a859b0b0df40f7 Mon Sep 17 00:00:00 2001 From: jalbrekt85 Date: Tue, 15 Jul 2025 13:10:35 -0500 Subject: [PATCH 5/5] remove outdated fetch block mock test --- tests/mock/test_mock_subgraph.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/mock/test_mock_subgraph.py b/tests/mock/test_mock_subgraph.py index 2312c1d..a3113e2 100644 --- a/tests/mock/test_mock_subgraph.py +++ b/tests/mock/test_mock_subgraph.py @@ -15,13 +15,6 @@ def date_range(): return (1728190800, 1729400400) -@patch("bal_tools.subgraph.Subgraph.fetch_graphql_data", mock_fetch_graphql_data) -def test_get_first_block_after_utc_timestamp(subgraph): - timestamp = 1622505600 - result = subgraph.get_first_block_after_utc_timestamp(timestamp) - assert result == 12345678 - - @patch("bal_tools.subgraph.Subgraph.fetch_graphql_data", mock_fetch_graphql_data) # TODO: @pytest.mark.skip(