diff --git a/docs/dev.md b/docs/dev.md index ebdbff8..b718589 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -26,10 +26,22 @@ uv run lefthook install This project uses [pytest](https://docs.pytest.org/en/stable/) as a testing framework. +### Unit Test + +Unit tests use a mock HTTP server ([csernazs/pytest-httpserver](https://github.com/csernazs/pytest-httpserver)) and are located under `/tests/unit/` + ```bash uv run pytest ``` +### Integration Test + +Integration tests require the environment variable `URLSCAN_API_KEY` and located under `/tests/integration/` + +```bash +uv run pytest --run-optional-tests=integration +``` + ## Docs This project uses [MkDocs](https://www.mkdocs.org/) as a documentation tool and uses [Mike](https://github.com/jimporter/mike) for versioning. diff --git a/pyproject.toml b/pyproject.toml index b0cc85d..2a4da55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,10 +23,12 @@ dev = [ "packaging>=26.0", "pytest-freezer>=0.4.9", "pytest-httpserver>=1.1.3", + "pytest-optional-tests==0.1.1", "pytest-pretty>=1.3.0", "pytest-randomly>=4.0.1", "pytest-timeout>=2.4.0", "pytest>=9.0.2", + "python-dotenv>=1.2.1", "ruff>=0.14.14", "uv-sort>=0.7.0", ] diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..15fdbb0 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +optional_tests= + integration: tests requiring real API access (require URLSCAN_API_KEY env variable) diff --git a/src/urlscan/pro/saved_search.py b/src/urlscan/pro/saved_search.py index b18280e..f449328 100644 --- a/src/urlscan/pro/saved_search.py +++ b/src/urlscan/pro/saved_search.py @@ -61,7 +61,7 @@ def create( https://docs.urlscan.io/apis/urlscan-openapi/saved-searches/savedsearches-post """ - data: dict[str, Any] = _compact( + search: dict[str, Any] = _compact( { "datasource": datasource, "query": query, @@ -73,6 +73,7 @@ def create( "permissions": permissions, } ) + data: dict[str, Any] = {"search": search} res = self._post("/api/v1/user/searches/", json=data) return self._response_to_json(res) @@ -119,7 +120,7 @@ def update( https://docs.urlscan.io/apis/urlscan-openapi/saved-searches/savedsearches-put """ - data: dict[str, Any] = _compact( + search: dict[str, Any] = _compact( { "datasource": datasource, "query": query, @@ -131,6 +132,7 @@ def update( "permissions": permissions, } ) + data: dict[str, Any] = {"search": search} res = self._put(f"/api/v1/user/searches/{search_id}/", json=data) return self._response_to_json(res) diff --git a/tests/pro/__init__.py b/tests/integration/__init__.py similarity index 100% rename from tests/pro/__init__.py rename to tests/integration/__init__.py diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..9b300a7 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,31 @@ +import os + +import pytest +from dotenv import load_dotenv + +from urlscan import Client, Pro + + +@pytest.fixture +def api_key() -> str: + load_dotenv() + key = os.getenv("URLSCAN_API_KEY") + assert key + return key + + +@pytest.fixture +def client(api_key: str): + with Client(api_key) as client: + yield client + + +@pytest.fixture +def pro(api_key: str): + with Pro(api_key) as client: + yield client + + +@pytest.fixture +def url() -> str: + return "https://httpbin.org/html" diff --git a/tests/integration/pro/__init__.py b/tests/integration/pro/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/pro/test_brand.py b/tests/integration/pro/test_brand.py new file mode 100644 index 0000000..bcbc0ff --- /dev/null +++ b/tests/integration/pro/test_brand.py @@ -0,0 +1,19 @@ +import pytest + +from urlscan import Pro + + +@pytest.mark.integration +def test_get_available_brands(pro: Pro): + brands = pro.brand.get_available_brands() + assert isinstance(brands, dict) + assert "kits" in brands + assert len(brands["kits"]) > 0 + + +@pytest.mark.integration +def test_get_brands(pro: Pro): + brands = pro.brand.get_brands() + assert isinstance(brands, dict) + assert "responses" in brands + assert len(brands["responses"]) > 0 diff --git a/tests/integration/pro/test_channel.py b/tests/integration/pro/test_channel.py new file mode 100644 index 0000000..3ab0420 --- /dev/null +++ b/tests/integration/pro/test_channel.py @@ -0,0 +1,56 @@ +import os + +import pytest + +from urlscan import Pro + + +@pytest.fixture +def channel_id() -> str: + """Get channel ID from environment variable. + + Set CHANNEL_ID environment variable to test channel operations. + """ + channel_id = os.getenv("CHANNEL_ID") + if not channel_id: + pytest.skip("CHANNEL_ID environment variable not set") + + return channel_id + + +@pytest.mark.integration +def test_get_channels(pro: Pro): + channels = pro.channel.get_channels() + assert isinstance(channels, dict) + assert "channels" in channels + assert len(channels["channels"]) > 0 + + +@pytest.mark.integration +def test_get_channel(pro: Pro, channel_id: str): + channel = pro.channel.get(channel_id) + assert channel is not None + assert "channel" in channel + + +@pytest.mark.integration +def test_update_channel(pro: Pro, channel_id: str): + # Get the current channel details + channel = pro.channel.get(channel_id) + original_name = channel["channel"]["name"] + channel_type = channel["channel"]["type"] + + try: + # Update the channel name + updated = pro.channel.update( + channel_id, channel_type=channel_type, name="integration-test" + ) + assert updated is not None + + # Verify the update + retrieved = pro.channel.get(channel_id) + assert retrieved["channel"]["name"] == "integration-test" + + finally: + # Revert the change + pro.channel.update(channel_id, channel_type=channel_type, name=original_name) diff --git a/tests/integration/pro/test_datadump.py b/tests/integration/pro/test_datadump.py new file mode 100644 index 0000000..e4bb09c --- /dev/null +++ b/tests/integration/pro/test_datadump.py @@ -0,0 +1,30 @@ +import datetime + +import pytest + +from urlscan import Pro + + +@pytest.fixture +def today() -> str: + return datetime.datetime.now(tz=datetime.timezone.utc).strftime("%Y%m%d") + + +@pytest.mark.integration +def test_datadump_list(pro: Pro, today: str): + result = pro.datadump.get_list(f"hours/api/{today}") + assert isinstance(result, dict) + assert "files" in result + assert len(result["files"]) > 0 + + +@pytest.mark.integration +def test_datadump_download(pro: Pro, today: str, tmp_path): + # Download a file from the hours/search path + output_path = tmp_path / f"{today}.gz" + with open(output_path, "wb") as f: + pro.datadump.download_file( + f"hours/search/{today}/{today}-00.gz", + file=f, + ) + assert output_path.exists() diff --git a/tests/integration/pro/test_hostname.py b/tests/integration/pro/test_hostname.py new file mode 100644 index 0000000..4a9efcc --- /dev/null +++ b/tests/integration/pro/test_hostname.py @@ -0,0 +1,17 @@ +import pytest + +from urlscan import Pro + + +@pytest.mark.integration +def test_hostname_with_limit(pro: Pro): + it = pro.hostname("example.com", limit=1) + results = list(it) + assert len(results) == 1 + + +@pytest.mark.integration +def test_hostname_with_limit_and_size(pro: Pro): + it = pro.hostname("example.com", limit=10, size=10) + results = list(it) + assert len(results) == 10 diff --git a/tests/integration/pro/test_livescan.py b/tests/integration/pro/test_livescan.py new file mode 100644 index 0000000..29d2468 --- /dev/null +++ b/tests/integration/pro/test_livescan.py @@ -0,0 +1,39 @@ +import pytest + +from urlscan import Pro + + +@pytest.mark.integration +def test_get_scanners(pro: Pro): + scanners = pro.livescan.get_scanners() + assert isinstance(scanners, dict) + assert "scanners" in scanners + assert len(scanners["scanners"]) > 0 + + +@pytest.fixture +def scanner_id() -> str: + return "us01" + + +@pytest.mark.integration +def test_scan_get_result_dom_and_purge(pro: Pro, url: str, scanner_id: str): + # Scan a URL + result = pro.livescan.scan(url, scanner_id=scanner_id) + uuid = result["uuid"] + + # Get the result + scan_result = pro.livescan.get_resource( + scanner_id=scanner_id, resource_type="result", resource_id=uuid + ) + assert scan_result is not None + + # Get the DOM + dom = pro.livescan.get_resource( + scanner_id=scanner_id, resource_type="dom", resource_id=uuid + ) + assert dom is not None + + # Purge the scan + purge_result = pro.livescan.purge(scanner_id=scanner_id, scan_id=uuid) + assert purge_result is not None diff --git a/tests/integration/pro/test_saved_search.py b/tests/integration/pro/test_saved_search.py new file mode 100644 index 0000000..5d80293 --- /dev/null +++ b/tests/integration/pro/test_saved_search.py @@ -0,0 +1,40 @@ +import pytest + +from urlscan import Pro + + +@pytest.mark.integration +def test_saved_search_crud(pro: Pro): + # Create a saved search + created = pro.saved_search.create( + datasource="scans", + query="page.domain:example.com", + name="integration-test", + ) + search_id = created["search"]["_id"] + + try: + # Get the saved search results + results = pro.saved_search.get_results(search_id) + assert results is not None + + # Update the saved search + updated = pro.saved_search.update( + search_id, + datasource="scans", + query="page.domain:example.net", + name="integration-test-updated", + ) + assert updated is not None + + finally: + # Delete the saved search + pro.saved_search.remove(search_id) + + +@pytest.mark.integration +def test_saved_search_list(pro: Pro): + result = pro.saved_search.get_list() + assert isinstance(result, dict) + assert "searches" in result + assert len(result["searches"]) > 0 diff --git a/tests/integration/pro/test_structure_search.py b/tests/integration/pro/test_structure_search.py new file mode 100644 index 0000000..8d4b9e3 --- /dev/null +++ b/tests/integration/pro/test_structure_search.py @@ -0,0 +1,21 @@ +import pytest + +from urlscan import Pro + +# Reference UUID from urlscan.io documentation +# https://docs.urlscan.io/apis/urlscan-openapi/search/similarsearch +REFERENCE_UUID = "68e26c59-2eae-437b-aeb1-cf750fafe7d7" + + +@pytest.mark.integration +def test_structure_search(pro: Pro): + it = pro.structure_search(REFERENCE_UUID, size=10, limit=10) + results = list(it) + assert len(results) >= 1 + + +@pytest.mark.integration +def test_structure_search_with_query(pro: Pro): + it = pro.structure_search(REFERENCE_UUID, q="page.domain:*", size=10, limit=10) + results = list(it) + assert len(results) >= 1 diff --git a/tests/integration/pro/test_subscription.py b/tests/integration/pro/test_subscription.py new file mode 100644 index 0000000..e4c79a0 --- /dev/null +++ b/tests/integration/pro/test_subscription.py @@ -0,0 +1,55 @@ +import pytest + +from urlscan import Pro + + +@pytest.mark.integration +def test_subscription_crud(pro: Pro): + # First, create a saved search to use for the subscription + search_result = pro.saved_search.create( + datasource="scans", + query="page.domain:example.com", + name="integration-test-subscription", + ) + search_id = search_result["search"]["_id"] + + try: + # Create a subscription + created = pro.subscription.create( + search_ids=[search_id], + frequency="daily", + name="integration-test", + email_addresses=["test@example.com"], + is_active=True, + ignore_time=False, + ) + subscription_id = created["subscription"]["_id"] + + try: + # Update the subscription + updated = pro.subscription.update( + subscription_id=subscription_id, + search_ids=[search_id], + frequency="hourly", + name="integration-test-updated", + email_addresses=["test@example.com"], + is_active=True, + ignore_time=False, + ) + assert updated is not None + + finally: + # Delete the subscription + pro.subscription.delete_subscription(subscription_id=subscription_id) + + finally: + # Clean up the saved search + pro.saved_search.remove(search_id) + + +@pytest.mark.integration +def test_subscription_list(pro: Pro): + result = pro.subscription.get_subscriptions() + assert isinstance(result, dict) + assert "subscriptions" in result + assert len(result["subscriptions"]) > 0 diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py new file mode 100644 index 0000000..2a2dc7a --- /dev/null +++ b/tests/integration/test_client.py @@ -0,0 +1,48 @@ +import pytest + +from urlscan import Client + + +@pytest.mark.integration +def test_scan_and_get_result(client: Client, url: str): + result = client.scan(url, visibility="private") + uuid: str = result["uuid"] + client.wait_for_result(uuid) + + assert client.get_dom(uuid) is not None + assert client.get_screenshot(uuid) is not None + assert client.get_result(uuid) is not None + + +@pytest.mark.integration +def test_search_with_limit(client: Client): + it = client.search("page.domain:example.com", limit=1) + results = list(it) + assert len(results) == 1 + + +@pytest.mark.integration +def test_search_with_limit_and_size(client: Client): + it = client.search("page.domain:example.com", limit=10, size=5) + results = list(it) + assert len(results) == 10 + + +@pytest.mark.integration +def test_quotas(client: Client): + quotas = client.get_quotas() + assert quotas["scope"] in ("team", "user") + + +@pytest.mark.integration +def test_get_available_countries(client: Client): + countries = client.get_available_countries() + assert isinstance(countries, dict) + assert len(countries) > 0 + + +@pytest.mark.integration +def test_get_user_agents(client: Client): + user_agents = client.get_user_agents() + assert isinstance(user_agents, dict) + assert len(user_agents) > 0 diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/unit/conftest.py similarity index 100% rename from tests/conftest.py rename to tests/unit/conftest.py diff --git a/tests/fixtures/test.gz b/tests/unit/fixtures/test.gz similarity index 100% rename from tests/fixtures/test.gz rename to tests/unit/fixtures/test.gz diff --git a/tests/fixtures/test.tar.gz b/tests/unit/fixtures/test.tar.gz similarity index 100% rename from tests/fixtures/test.tar.gz rename to tests/unit/fixtures/test.tar.gz diff --git a/tests/unit/pro/__init__.py b/tests/unit/pro/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/pro/test_brand.py b/tests/unit/pro/test_brand.py similarity index 100% rename from tests/pro/test_brand.py rename to tests/unit/pro/test_brand.py diff --git a/tests/pro/test_channel.py b/tests/unit/pro/test_channel.py similarity index 100% rename from tests/pro/test_channel.py rename to tests/unit/pro/test_channel.py diff --git a/tests/pro/test_datadump.py b/tests/unit/pro/test_datadump.py similarity index 100% rename from tests/pro/test_datadump.py rename to tests/unit/pro/test_datadump.py diff --git a/tests/pro/test_file.py b/tests/unit/pro/test_file.py similarity index 100% rename from tests/pro/test_file.py rename to tests/unit/pro/test_file.py diff --git a/tests/pro/test_hostname.py b/tests/unit/pro/test_hostname.py similarity index 100% rename from tests/pro/test_hostname.py rename to tests/unit/pro/test_hostname.py diff --git a/tests/pro/test_incident.py b/tests/unit/pro/test_incident.py similarity index 100% rename from tests/pro/test_incident.py rename to tests/unit/pro/test_incident.py diff --git a/tests/pro/test_livescan.py b/tests/unit/pro/test_livescan.py similarity index 100% rename from tests/pro/test_livescan.py rename to tests/unit/pro/test_livescan.py diff --git a/tests/pro/test_pro.py b/tests/unit/pro/test_pro.py similarity index 100% rename from tests/pro/test_pro.py rename to tests/unit/pro/test_pro.py diff --git a/tests/pro/test_saved_search.py b/tests/unit/pro/test_saved_search.py similarity index 83% rename from tests/pro/test_saved_search.py rename to tests/unit/pro/test_saved_search.py index 9955c1f..eebbadc 100644 --- a/tests/pro/test_saved_search.py +++ b/tests/unit/pro/test_saved_search.py @@ -40,9 +40,11 @@ def test_create_minimal(pro: Pro, httpserver: HTTPServer): "/api/v1/user/searches/", method="POST", json={ - "datasource": "scans", - "query": "page.domain:example.com", - "name": "Example Search", + "search": { + "datasource": "scans", + "query": "page.domain:example.com", + "name": "Example Search", + }, }, ).respond_with_json(data) @@ -72,14 +74,16 @@ def test_create_with_all_params(pro: Pro, httpserver: HTTPServer): "/api/v1/user/searches/", method="POST", json={ - "datasource": "hostnames", - "query": "hostname:*.example.com", - "name": "Hostname Search", - "description": "Short description", - "longDescription": "Detailed description of the search", - "tlp": "amber", - "userTags": ["tag1", "tag2"], - "permissions": ["public:read", "team:write"], + "search": { + "datasource": "hostnames", + "query": "hostname:*.example.com", + "name": "Hostname Search", + "description": "Short description", + "longDescription": "Detailed description of the search", + "tlp": "amber", + "userTags": ["tag1", "tag2"], + "permissions": ["public:read", "team:write"], + } }, ).respond_with_json(data) @@ -110,9 +114,11 @@ def test_update_minimal(pro: Pro, httpserver: HTTPServer): f"/api/v1/user/searches/{search_id}/", method="PUT", json={ - "datasource": "scans", - "query": "page.domain:updated.com", - "name": "Updated Search", + "search": { + "datasource": "scans", + "query": "page.domain:updated.com", + "name": "Updated Search", + } }, ).respond_with_json(data) @@ -144,14 +150,16 @@ def test_update_with_all_params(pro: Pro, httpserver: HTTPServer): f"/api/v1/user/searches/{search_id}/", method="PUT", json={ - "datasource": "hostnames", - "query": "hostname:*.updated.com", - "name": "Updated Hostname Search", - "description": "Updated short description", - "longDescription": "Updated detailed description", - "tlp": "green", - "userTags": ["updated-tag"], - "permissions": ["team:read"], + "search": { + "datasource": "hostnames", + "query": "hostname:*.updated.com", + "name": "Updated Hostname Search", + "description": "Updated short description", + "longDescription": "Updated detailed description", + "tlp": "green", + "userTags": ["updated-tag"], + "permissions": ["team:read"], + } }, ).respond_with_json(data) diff --git a/tests/pro/test_structure_search.py b/tests/unit/pro/test_structure_search.py similarity index 100% rename from tests/pro/test_structure_search.py rename to tests/unit/pro/test_structure_search.py diff --git a/tests/pro/test_subscription.py b/tests/unit/pro/test_subscription.py similarity index 100% rename from tests/pro/test_subscription.py rename to tests/unit/pro/test_subscription.py diff --git a/tests/test_client.py b/tests/unit/test_client.py similarity index 100% rename from tests/test_client.py rename to tests/unit/test_client.py diff --git a/tests/test_utils.py b/tests/unit/test_utils.py similarity index 93% rename from tests/test_utils.py rename to tests/unit/test_utils.py index 9c331dd..fc8deda 100644 --- a/tests/test_utils.py +++ b/tests/unit/test_utils.py @@ -23,7 +23,7 @@ def test_parse_datetime(input_str: str, expected: datetime.datetime): @pytest.fixture def gz(): - return "tests/fixtures/test.gz" + return "tests/unit/fixtures/test.gz" @pytest.fixture @@ -33,7 +33,7 @@ def tar_gz(): # $ gtar -cf 2.tar 2.txt # $ cat 1.tar 2.tar > test.tar # $ gzip test.tar - return "tests/fixtures/test.tar.gz" + return "tests/unit/fixtures/test.tar.gz" def test_extract_with_gz(gz: str): diff --git a/tests/test_version.py b/tests/unit/test_version.py similarity index 100% rename from tests/test_version.py rename to tests/unit/test_version.py diff --git a/uv.lock b/uv.lock index ec2ec2a..a11946f 100644 --- a/uv.lock +++ b/uv.lock @@ -933,6 +933,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0d/d2/dfc2f25f3905921c2743c300a48d9494d29032f1389fc142e718d6978fb2/pytest_httpserver-1.1.3-py3-none-any.whl", hash = "sha256:5f84757810233e19e2bb5287f3826a71c97a3740abe3a363af9155c0f82fdbb9", size = 21000, upload-time = "2025-04-10T08:17:13.906Z" }, ] +[[package]] +name = "pytest-optional-tests" +version = "0.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/0d/e0ed3c137ac8b604d0a90aa4a23e50ef21a66320b50a6a9b2c5f41e5a986/pytest-optional-tests-0.1.1.tar.gz", hash = "sha256:7f4411bf1551b556ae24a7e1385c4832d7520ccef109c374929aa71afe8c6754", size = 7569, upload-time = "2019-07-09T01:24:30.627Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/4a/7726bed4f1fda2f4af35d1b8d774169aa0103f0141abe3fb3ba7761efa86/pytest_optional_tests-0.1.1-py3-none-any.whl", hash = "sha256:ededc9d2aa7051d1af8ff5e757119b5758d86c7f24e73e1bb7dd5f19cd2031fa", size = 5466, upload-time = "2019-07-09T01:24:29.102Z" }, +] + [[package]] name = "pytest-pretty" version = "1.3.0" @@ -982,6 +994,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "python-dotenv" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f0/26/19cadc79a718c5edbec86fd4919a6b6d3f681039a2f6d66d14be94e75fb9/python_dotenv-1.2.1.tar.gz", hash = "sha256:42667e897e16ab0d66954af0e60a9caa94f0fd4ecf3aaf6d2d260eec1aa36ad6", size = 44221, upload-time = "2025-10-26T15:12:10.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230, upload-time = "2025-10-26T15:12:09.109Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1388,9 +1409,11 @@ dev = [ { name = "pytest" }, { name = "pytest-freezer" }, { name = "pytest-httpserver" }, + { name = "pytest-optional-tests" }, { name = "pytest-pretty" }, { name = "pytest-randomly" }, { name = "pytest-timeout" }, + { name = "python-dotenv" }, { name = "ruff" }, { name = "uv-sort" }, ] @@ -1415,9 +1438,11 @@ dev = [ { name = "pytest", specifier = ">=9.0.2" }, { name = "pytest-freezer", specifier = ">=0.4.9" }, { name = "pytest-httpserver", specifier = ">=1.1.3" }, + { name = "pytest-optional-tests", specifier = "==0.1.1" }, { name = "pytest-pretty", specifier = ">=1.3.0" }, { name = "pytest-randomly", specifier = ">=4.0.1" }, { name = "pytest-timeout", specifier = ">=2.4.0" }, + { name = "python-dotenv", specifier = ">=1.2.1" }, { name = "ruff", specifier = ">=0.14.14" }, { name = "uv-sort", specifier = ">=0.7.0" }, ]