diff --git a/hivemind_etl/website/website_etl.py b/hivemind_etl/website/website_etl.py index 0f2853c..4d0f113 100644 --- a/hivemind_etl/website/website_etl.py +++ b/hivemind_etl/website/website_etl.py @@ -52,7 +52,9 @@ async def extract( extracted_data = [] for url in urls: logging.info(f"Crawling {url} and its routes!") - extracted_data.extend(await self.crawlee_client.crawl(links=[url])) + data = await self.crawlee_client.crawl(links=[url]) + logging.info(f"{len(data)} data is extracted.") + extracted_data.extend(data) logging.info(f"Extracted {len(extracted_data)} documents!")