From 86a9cdd46e2dd59223526cf45a2214e161b49672 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Tue, 1 Apr 2025 16:32:04 +0330 Subject: [PATCH] feat: adding more logs to crawler! --- hivemind_etl/website/website_etl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hivemind_etl/website/website_etl.py b/hivemind_etl/website/website_etl.py index 0f2853c..4d0f113 100644 --- a/hivemind_etl/website/website_etl.py +++ b/hivemind_etl/website/website_etl.py @@ -52,7 +52,9 @@ async def extract( extracted_data = [] for url in urls: logging.info(f"Crawling {url} and its routes!") - extracted_data.extend(await self.crawlee_client.crawl(links=[url])) + data = await self.crawlee_client.crawl(links=[url]) + logging.info(f"{len(data)} data is extracted.") + extracted_data.extend(data) logging.info(f"Extracted {len(extracted_data)} documents!")