From e69b1be85690c001d65608ba3f5e1c0630648b5a Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Sun, 13 Apr 2025 10:34:49 +0330 Subject: [PATCH] feat: added retry policy for mediwiki activities! --- hivemind_etl/mediawiki/workflows.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/hivemind_etl/mediawiki/workflows.py b/hivemind_etl/mediawiki/workflows.py index 3cd946a..0d7bf87 100644 --- a/hivemind_etl/mediawiki/workflows.py +++ b/hivemind_etl/mediawiki/workflows.py @@ -2,6 +2,7 @@ from datetime import timedelta from temporalio import workflow +from temporalio.common import RetryPolicy with workflow.unsafe.imports_passed_through(): from hivemind_etl.mediawiki.activities import ( @@ -31,6 +32,10 @@ async def run(self, platform_id: str | None = None) -> None: get_hivemind_mediawiki_platforms, platform_id, start_to_close_timeout=timedelta(minutes=1), + retry_policy=RetryPolicy( + initial_interval=timedelta(minutes=1), + maximum_attempts=3, + ), ) for platform in platforms: @@ -45,6 +50,10 @@ async def run(self, platform_id: str | None = None) -> None: extract_mediawiki, mediawiki_platform, start_to_close_timeout=timedelta(days=5), + retry_policy=RetryPolicy( + initial_interval=timedelta(minutes=1), + maximum_attempts=3, + ), ) # Transform the extracted data @@ -52,6 +61,10 @@ async def run(self, platform_id: str | None = None) -> None: transform_mediawiki_data, platform["community_id"], start_to_close_timeout=timedelta(minutes=30), + retry_policy=RetryPolicy( + initial_interval=timedelta(minutes=1), + maximum_attempts=3, + ), ) # Load the transformed data @@ -60,6 +73,10 @@ async def run(self, platform_id: str | None = None) -> None: documents, platform["community_id"], start_to_close_timeout=timedelta(minutes=30), + retry_policy=RetryPolicy( + initial_interval=timedelta(minutes=1), + maximum_attempts=3, + ), ) logging.info(