diff --git a/.env.example b/.env.example index 1ec24e2..138c679 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,4 @@ ETHNODEURL= DRPC_KEY= GRAPH_API_KEY= +ETHERSCAN_API_KEY= diff --git a/.github/workflows/python_package.yaml b/.github/workflows/python_package.yaml index 18cc40b..a5d5a66 100644 --- a/.github/workflows/python_package.yaml +++ b/.github/workflows/python_package.yaml @@ -27,6 +27,7 @@ jobs: ETHNODEURL: ${{ secrets.ETHNODEURL }} DRPC_KEY: ${{ secrets.DRPC_KEY }} GRAPH_API_KEY: ${{ secrets.GRAPH_API_KEY }} + ETHERSCAN_API_KEY: ${{ secrets.ETHERSCAN_API_KEY }} run: | pip install -r bal_tools/requirements-dev.txt pytest -x diff --git a/bal_tools/etherscan.py b/bal_tools/etherscan.py new file mode 100644 index 0000000..db37bf6 --- /dev/null +++ b/bal_tools/etherscan.py @@ -0,0 +1,90 @@ +import os +import time +from typing import Optional, Dict, Any +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +from .utils import chain_ids_by_name + + +class Etherscan: + BASE_URL = "https://api.etherscan.io/v2/api" + + def __init__(self, api_key: Optional[str] = None): + self.api_key = api_key or os.getenv("ETHERSCAN_API_KEY") + if not self.api_key: + raise ValueError( + "Etherscan API key required. Set ETHERSCAN_API_KEY environment variable or pass api_key parameter" + ) + + self.session = requests.Session() + retry_strategy = Retry( + total=3, + backoff_factor=0.5, + status_forcelist=[429, 500, 502, 503, 504], + ) + adapter = HTTPAdapter(max_retries=retry_strategy) + self.session.mount("http://", adapter) + self.session.mount("https://", adapter) + + self.last_request_time = 0 + self.rate_limit_delay = 0.2 # 200ms + + def _get_chain_id(self, chain: str) -> int: + chain_ids = chain_ids_by_name() + if chain not in chain_ids: + raise ValueError( + f"Unsupported chain: {chain}. Supported chains: {list(chain_ids.keys())}" + ) + return chain_ids[chain] + + def _rate_limit(self): + current_time = time.time() + time_since_last_request = current_time - self.last_request_time + if time_since_last_request < self.rate_limit_delay: + time.sleep(self.rate_limit_delay - time_since_last_request) + self.last_request_time = time.time() + + def _make_request(self, params: Dict[str, Any]) -> Dict[str, Any]: + self._rate_limit() + + params["apikey"] = self.api_key + + response = self.session.get(self.BASE_URL, params=params, timeout=30) + response.raise_for_status() + + data = response.json() + + if data.get("status") == "0" and data.get("message") != "No records found": + raise Exception( + f"Etherscan API error: {data.get('message', 'Unknown error')}" + ) + + return data + + def get_block_by_timestamp( + self, chain: str, timestamp: int, closest: str = "before" + ) -> Optional[int]: + chain_id = self._get_chain_id(chain) + + params = { + "chainid": chain_id, + "module": "block", + "action": "getblocknobytime", + "timestamp": timestamp, + "closest": closest, + } + + try: + data = self._make_request(params) + + if data.get("status") == "1" and data.get("result"): + return int(data["result"]) + + return None + + except Exception as e: + raise Exception( + f"Error fetching block for timestamp {timestamp} on {chain}: {str(e)}" + ) diff --git a/bal_tools/subgraph.py b/bal_tools/subgraph.py index 2d2f37c..eead922 100644 --- a/bal_tools/subgraph.py +++ b/bal_tools/subgraph.py @@ -16,6 +16,7 @@ from .models import * from .errors import NoPricesFoundError from .ts_config_loader import ts_config_loader +from .etherscan import Etherscan def url_dict_from_df(df): @@ -72,6 +73,7 @@ def __init__(self, chain: str = "mainnet", silence_warnings: bool = False): if silence_warnings: self.set_silence_warnings(True) self.custom_price_logic: Dict[str, Callable] = {} + self.etherscan_client = None def set_silence_warnings(self, silence_warnings: bool): if silence_warnings: @@ -325,20 +327,45 @@ def fetch_graphql_data( return result - def get_first_block_after_utc_timestamp(self, timestamp: int) -> int: + def get_first_block_after_utc_timestamp( + self, timestamp: int, use_etherscan: bool = True + ) -> int: if timestamp > int(datetime.now().strftime("%s")): timestamp = int(datetime.now().strftime("%s")) - 2000 - data = self.fetch_graphql_data( - "blocks", - "first_block_after_ts", - { - "timestamp_gt": int(timestamp) - 200, - "timestamp_lt": int(timestamp) + 200, - }, - ) - data["blocks"].sort(key=lambda x: x["timestamp"], reverse=True) - return int(data["blocks"][0]["number"]) + if use_etherscan: + try: + if not self.etherscan_client: + self.etherscan_client = Etherscan() + + block_number = self.etherscan_client.get_block_by_timestamp( + chain=self.chain, timestamp=timestamp, closest="after" + ) + + if block_number: + return block_number + + except Exception as e: + warnings.warn( + f"Etherscan V2 block fetch failed for chain {self.chain}: {str(e)}. Falling back to subgraph.", + UserWarning, + ) + + try: + data = self.fetch_graphql_data( + "blocks", + "first_block_after_ts", + { + "timestamp_gt": int(timestamp) - 200, + "timestamp_lt": int(timestamp) + 200, + }, + ) + data["blocks"].sort(key=lambda x: x["timestamp"], reverse=True) + return int(data["blocks"][0]["number"]) + except Exception as e: + raise Exception( + f"Failed to fetch block for timestamp {timestamp} on {self.chain}: {str(e)}" + ) def get_twap_price_token( self, diff --git a/tests/mock/test_mock_subgraph.py b/tests/mock/test_mock_subgraph.py index 2312c1d..a3113e2 100644 --- a/tests/mock/test_mock_subgraph.py +++ b/tests/mock/test_mock_subgraph.py @@ -15,13 +15,6 @@ def date_range(): return (1728190800, 1729400400) -@patch("bal_tools.subgraph.Subgraph.fetch_graphql_data", mock_fetch_graphql_data) -def test_get_first_block_after_utc_timestamp(subgraph): - timestamp = 1622505600 - result = subgraph.get_first_block_after_utc_timestamp(timestamp) - assert result == 12345678 - - @patch("bal_tools.subgraph.Subgraph.fetch_graphql_data", mock_fetch_graphql_data) # TODO: @pytest.mark.skip( diff --git a/tests/test_subgraph.py b/tests/test_subgraph.py index 51191d1..d8b9124 100644 --- a/tests/test_subgraph.py +++ b/tests/test_subgraph.py @@ -3,6 +3,8 @@ import json import warnings import time +import os +from datetime import datetime, timedelta from bal_tools.subgraph import Subgraph, GqlChain, Pool, PoolSnapshot from bal_tools.errors import NoPricesFoundError @@ -170,3 +172,27 @@ def test_get_pool_protocol_version(subgraph): ) == 2 ) + + +def test_get_first_block_after_utc_timestamp_with_etherscan( + chain, subgraph_all_chains, chains_prod +): + if not os.getenv("ETHERSCAN_API_KEY"): + pytest.skip("ETHERSCAN_API_KEY not set") + + if chain not in chains_prod or chain in ["fantom", "sonic"]: + pytest.skip(f"Skipping {chain}") + + test_timestamp = int((datetime.now() - timedelta(days=1)).timestamp()) + + try: + block = subgraph_all_chains.get_first_block_after_utc_timestamp( + test_timestamp, use_etherscan=True + ) + assert isinstance(block, int) + assert block > 0 + except Exception as e: + if "Unsupported chain" in str(e) or "Error fetching block" in str(e): + pytest.skip(f"Chain {chain} not supported by Etherscan V2") + else: + raise