From ca0e28cfa677d2b7dc3ba06b32ec0e541980e75e Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Sun, 13 Apr 2025 11:33:26 +0330 Subject: [PATCH] feat: added logs to wikiteam crawler! --- hivemind_etl/mediawiki/wikiteam_crawler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hivemind_etl/mediawiki/wikiteam_crawler.py b/hivemind_etl/mediawiki/wikiteam_crawler.py index 1ffa802..002a10c 100644 --- a/hivemind_etl/mediawiki/wikiteam_crawler.py +++ b/hivemind_etl/mediawiki/wikiteam_crawler.py @@ -59,6 +59,9 @@ def crawl(self, api_url: str, dump_path: str) -> None: else: params.extend([f"--{key}", str(value)]) + logging.info(f"Crawling mediawiki dump from {api_url} to {dump_path}") + logging.info(f"Parameters: {params}") + # Directly call the DumpGenerator static __init__ method which will parse these parameters, # execute the dump generation process, and run through the rest of the workflow. DumpGenerator(params)