From ef1608a760417bb3104c8f33554576826e2f6739 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Thu, 8 Jan 2026 09:24:56 +0100 Subject: [PATCH] [uss_qualifier] Record failled queries --- .basedpyright/baseline.json | 72 --------------- monitoring/mock_uss/versioning/routes.py | 0 .../clients/versioning/client_interuss.py | 0 monitoring/monitorlib/fetch/__init__.py | 88 ++++++++++++------- .../uss_qualifier/scenarios/scenario.py | 5 ++ schemas/manage_type_schemas.py | 5 +- uv.lock | 6 +- 7 files changed, 68 insertions(+), 108 deletions(-) mode change 100644 => 100755 monitoring/mock_uss/versioning/routes.py mode change 100644 => 100755 monitoring/monitorlib/clients/versioning/client_interuss.py diff --git a/.basedpyright/baseline.json b/.basedpyright/baseline.json index 1010366eff..4888a9f574 100644 --- a/.basedpyright/baseline.json +++ b/.basedpyright/baseline.json @@ -2576,78 +2576,6 @@ "endColumn": 67, "lineCount": 1 } - }, - { - "code": "reportAssignmentType", - "range": { - "startColumn": 17, - "endColumn": 35, - "lineCount": 1 - } - }, - { - "code": "reportOptionalMemberAccess", - "range": { - "startColumn": 23, - "endColumn": 30, - "lineCount": 1 - } - }, - { - "code": "reportOptionalMemberAccess", - "range": { - "startColumn": 28, - "endColumn": 49, - "lineCount": 1 - } - }, - { - "code": "reportOptionalMemberAccess", - "range": { - "startColumn": 25, - "endColumn": 40, - "lineCount": 1 - } - }, - { - "code": "reportPossiblyUnboundVariable", - "range": { - "startColumn": 46, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportOperatorIssue", - "range": { - "startColumn": 22, - "endColumn": 31, - "lineCount": 1 - } - }, - { - "code": "reportPossiblyUnboundVariable", - "range": { - "startColumn": 23, - "endColumn": 25, - "lineCount": 1 - } - }, - { - "code": "reportPossiblyUnboundVariable", - "range": { - "startColumn": 28, - "endColumn": 30, - "lineCount": 1 - } - }, - { - "code": "reportPossiblyUnboundVariable", - "range": { - "startColumn": 41, - "endColumn": 43, - "lineCount": 1 - } } ], "./monitoring/monitorlib/fetch/evaluation.py": [ diff --git a/monitoring/mock_uss/versioning/routes.py b/monitoring/mock_uss/versioning/routes.py old mode 100644 new mode 100755 diff --git a/monitoring/monitorlib/clients/versioning/client_interuss.py b/monitoring/monitorlib/clients/versioning/client_interuss.py old mode 100644 new mode 100755 diff --git a/monitoring/monitorlib/fetch/__init__.py b/monitoring/monitorlib/fetch/__init__.py index 455c9417d8..8f338d618e 100644 --- a/monitoring/monitorlib/fetch/__init__.py +++ b/monitoring/monitorlib/fetch/__init__.py @@ -1,3 +1,4 @@ +import copy import datetime import json import os @@ -5,7 +6,7 @@ import uuid from dataclasses import dataclass from enum import Enum -from typing import TypeVar +from typing import Self, TypeVar from urllib.parse import urlparse import flask @@ -458,6 +459,9 @@ class Query(ImplicitDict): query_type: QueryType | None """If specified, the recognized type of this query.""" + _previous_query: Self | None + """If specified, the previous, failling query that generated this query as a retry""" + @property def timestamp(self) -> datetime.datetime: """Safety property to prevent crashes when Query.timestamp is accessed. @@ -579,10 +583,12 @@ def describe_query( initiated_at: datetime.datetime, query_type: QueryType | None = None, participant_id: str | None = None, + previous_query: Query | None = None, ) -> Query: query = Query( request=describe_request(resp.request, initiated_at), response=describe_response(resp), + _previous_query=previous_query, ) if query_type is not None: query.query_type = query_type @@ -618,10 +624,9 @@ def query_and_describe( Query object describing the request and response/result. """ if client is None: - utm_session = False - client = requests.session() + _client = requests.session() else: - utm_session = True + _client = client req_kwargs = kwargs.copy() if "timeout" not in req_kwargs: req_kwargs["timeout"] = ( @@ -655,6 +660,36 @@ def get_location() -> str: .strip() ) + previous_query = None + + def build_failing_query(t0) -> Query: + _req_kwargs = copy.deepcopy(req_kwargs) + + if isinstance(_client, infrastructure.UTMClientSession): + _req_kwargs = _client.adjust_request_kwargs(_req_kwargs) + del _req_kwargs["timeout"] + + req = requests.Request(verb, url, **_req_kwargs) + prepped_req = _client.prepare_request(req) + + t1 = datetime.datetime.now(datetime.UTC) + + query = Query( + request=describe_request(prepped_req, t0), + response=ResponseDescription( + code=None, + failure="\n".join(failures), + elapsed_s=(t1 - t0).total_seconds(), + reported=StringBasedDateTime(t1), + ), + participant_id=participant_id, + _previous_query=previous_query, + ) + if query_type is not None: + query.query_type = query_type + + return query + # Note: retry logic could be attached to the `client` Session by `mount`ing an HTTPAdapter with custom # `max_retries`, however we do not want to mutate the provided Session. Instead, retry only on errors we explicitly # consider retryable. @@ -664,13 +699,14 @@ def get_location() -> str: if is_netloc_fake: failure_message = f"query_and_describe attempt {attempt + 1} from PID {os.getpid()} to {verb} {url} was not attempted because network location of {url} was identified as fake: {settings.fake_netlocs}\nAt {get_location()}" failures.append(failure_message) - break + return build_failing_query(t0) return describe_query( - client.request(verb, url, **req_kwargs), + _client.request(verb, url, **req_kwargs), t0, query_type=query_type, participant_id=participant_id, + previous_query=previous_query, ) except (requests.Timeout, urllib3.exceptions.ReadTimeoutError) as e: failure_message = f"query_and_describe attempt {attempt + 1} from PID {os.getpid()} to {verb} {url} failed with timeout {type(e).__name__}: {str(e)}\nAt {get_location()}" @@ -689,38 +725,28 @@ def get_location() -> str: if not expect_failure: logger.warning(failure_message) failures.append(failure_message) + if not retryable: - break + return build_failing_query(t0) + except requests.RequestException as e: failure_message = f"query_and_describe attempt {attempt + 1} from PID {os.getpid()} to {verb} {url} failed with non-retryable RequestException {type(e).__name__}: {str(e)}\nAt {get_location()}" if not expect_failure: logger.warning(failure_message) failures.append(failure_message) - break - finally: - t1 = datetime.datetime.now(datetime.UTC) - - # Reconstruct request similar to the one in the query (which is not - # accessible at this point) - if utm_session: - req_kwargs = client.adjust_request_kwargs(req_kwargs) - del req_kwargs["timeout"] - req = requests.Request(verb, url, **req_kwargs) - prepped_req = client.prepare_request(req) - result = Query( - request=describe_request(prepped_req, t0), - response=ResponseDescription( - code=None, - failure="\n".join(failures), - elapsed_s=(t1 - t0).total_seconds(), - reported=StringBasedDateTime(t1), - ), - participant_id=participant_id, - ) - if query_type is not None: - result.query_type = query_type - return result + return build_failing_query(t0) + + previous_query = build_failing_query( + t0 + ) # If we arrive there, query failled, but is retriable + + if not previous_query: + raise Exception( + "Internal error: arrived after retried without any expected failled query" + ) + + return previous_query # Previous query is the last failled one def describe_flask_query( diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index 5e4bec02fc..a635e86d09 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -423,6 +423,11 @@ def record_queries(self, queries: list[fetch.Query]) -> None: def record_query(self, query: fetch.Query) -> None: self._expect_phase({ScenarioPhase.RunningTestStep, ScenarioPhase.CleaningUp}) + + # If the query has a previous one, record it first + if "_previous_query" in query and query._previous_query: + self.record_query(query._previous_query) + if "queries" not in self._step_report: self._step_report.queries = [] for existing_query in self._step_report.queries: diff --git a/schemas/manage_type_schemas.py b/schemas/manage_type_schemas.py index 29e531c15b..f6678570f7 100644 --- a/schemas/manage_type_schemas.py +++ b/schemas/manage_type_schemas.py @@ -4,7 +4,7 @@ import json import os import sys -from typing import get_args, get_origin, get_type_hints +from typing import Self, get_args, get_origin, get_type_hints import implicitdict import implicitdict.jsonschema @@ -78,7 +78,8 @@ def _make_type_schemas( pending_types.extend(get_args(pending_type)) else: if ( - issubclass(pending_type, ImplicitDict) + pending_type != Self + and issubclass(pending_type, ImplicitDict) and fullname(pending_type) not in already_checked ): _make_type_schemas( diff --git a/uv.lock b/uv.lock index 714fb84c29..aaeb0ff42b 100644 --- a/uv.lock +++ b/uv.lock @@ -612,16 +612,16 @@ wheels = [ [[package]] name = "implicitdict" -version = "4.0.1" +version = "4.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "arrow" }, { name = "jsonschema" }, { name = "pytimeparse" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e8/92/335c435d15e2d198a679c3fbb56cd20907f204c386c1b8facdc06cea8256/implicitdict-4.0.1.tar.gz", hash = "sha256:6413da2faeb03f1d20a537f357c05a58cef8ab32e8923eae4e79ee2396546074", size = 57876, upload-time = "2025-10-01T15:07:54.467Z" } +sdist = { url = "https://files.pythonhosted.org/packages/09/42/657eac38a6f71090790a8399decb4c6a8243136cebc7929c18c8c0df0613/implicitdict-4.1.0.tar.gz", hash = "sha256:afa39b97180993c0c8a75bccb72fa05f14a8db7550ffc18d92a3a2717396701b", size = 53878, upload-time = "2026-01-08T04:33:24.524Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/02/ce/d947f5995b160d56a2e4a2ba48eacc59884cc2cabfb9eafe4ac49d80575d/implicitdict-4.0.1-py3-none-any.whl", hash = "sha256:71e16dc977a944c0279f5f383db63b44a14a9961b32b535e011fbba782e88a70", size = 13658, upload-time = "2025-10-01T15:07:53.603Z" }, + { url = "https://files.pythonhosted.org/packages/da/0e/9382537211f4009a99de7b6c005d52b55ee789c245b9e9a77015edd36958/implicitdict-4.1.0-py3-none-any.whl", hash = "sha256:e550f55aafe933e3132ec0dc9fe68e686dc2060d5ba676bf54f748cfd5fedfd0", size = 13737, upload-time = "2026-01-08T04:33:23.541Z" }, ] [[package]]