diff --git a/hivemind_etl/mediawiki/module.py b/hivemind_etl/mediawiki/module.py index 7dc4e70..3294e7b 100644 --- a/hivemind_etl/mediawiki/module.py +++ b/hivemind_etl/mediawiki/module.py @@ -62,16 +62,26 @@ def get_learning_platforms( platform_id=platform_id, metadata_name="path", ) + namespaces = self.get_platform_metadata( + platform_id=platform_id, + metadata_name="namespaces", + ) if not isinstance(path, str) and not isinstance(base_url, str): raise ValueError("Wrong format for `path` and `base_url`!") modules_options = platform["metadata"] - namespaces = modules_options.get("namespaces", []) + activated = modules_options["activated"] + + if not activated: + logging.warning( + f"Platform: {platform_id} is not activated! Skipping it..." + ) + continue if not namespaces: logging.warning( - f"No namespaces found for platform: {platform_id}" + f"No namespaces found for platform: {platform_id}. Skipping it..." ) continue diff --git a/requirements.txt b/requirements.txt index 144219e..bc6b6aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ python-dotenv>=1.0.0, <2.0.0 -tc-hivemind-backend==1.4.2.post2 +tc-hivemind-backend==1.4.3 llama-index-storage-docstore-redis==0.1.2 llama-index-storage-docstore-mongodb==0.1.3 crawlee[playwright]==0.3.8 diff --git a/tests/integration/test_mediawiki_modules.py b/tests/integration/test_mediawiki_modules.py index 1332fd2..c60a308 100644 --- a/tests/integration/test_mediawiki_modules.py +++ b/tests/integration/test_mediawiki_modules.py @@ -48,11 +48,12 @@ def test_get_single_data(self): "platform": platform_id, "name": "mediaWiki", "metadata": { - "namespaces": [0, 1, 2], + "activated": True, }, } ] }, + "activated": True, } ) @@ -85,18 +86,19 @@ def test_get_mediawiki_communities_data_multiple_platforms(self): "platform": platform_id1, "name": "mediaWiki", "metadata": { - "namespaces": [0, 1, 2], + "activated": True, }, }, { "platform": platform_id2, "name": "mediaWiki", "metadata": { - "namespaces": [3, 4, 5], + "activated": True, }, }, ] }, + "activated": True, } ) @@ -107,6 +109,7 @@ def test_get_mediawiki_communities_data_multiple_platforms(self): "metadata": { "baseURL": "http://example1.com", "path": "/api", + "namespaces": [0, 1, 2], }, "community": community_id, "disconnectedAt": None, @@ -123,6 +126,7 @@ def test_get_mediawiki_communities_data_multiple_platforms(self): "metadata": { "baseURL": "http://example2.com", "path": "/api", + "namespaces": [3, 4, 5], }, "community": community_id, "disconnectedAt": None, @@ -171,18 +175,19 @@ def test_get_mediawiki_communities_data_filtered_platforms(self): "platform": platform_id1, "name": "mediaWiki", "metadata": { - "namespaces": [0, 1, 2], + "activated": True, }, }, { "platform": platform_id2, "name": "mediaWiki", "metadata": { - "namespaces": [3, 4, 5], + "activated": True, }, }, ] }, + "activated": True, } ) @@ -193,6 +198,7 @@ def test_get_mediawiki_communities_data_filtered_platforms(self): "metadata": { "baseURL": "http://example1.com", "path": "/api", + "namespaces": [0, 1, 2], }, "community": community_id, "disconnectedAt": None, @@ -209,6 +215,7 @@ def test_get_mediawiki_communities_data_filtered_platforms(self): "metadata": { "baseURL": "http://example2.com", "path": "/api", + "namespaces": [3, 4, 5], }, "community": community_id, "disconnectedAt": None, @@ -232,3 +239,84 @@ def test_get_mediawiki_communities_data_filtered_platforms(self): "base_url": "http://example1.com/api", }, ) + + def test_get_mediawiki_communities_data_filtered_platforms_not_activated(self): + """ + Two mediawiki platforms for one community + """ + platform_id1 = ObjectId("6579c364f1120850414e0dc6") + platform_id2 = ObjectId("6579c364f1120850414e0dc7") + community_id = ObjectId("1009c364f1120850414e0dc5") + + self.client["Core"]["modules"].insert_one( + { + "name": "hivemind", + "community": community_id, + "options": { + "platforms": [ + { + "platform": platform_id1, + "name": "mediaWiki", + "metadata": { + "activated": False, + }, + }, + { + "platform": platform_id2, + "name": "mediaWiki", + "metadata": { + "activated": True, + }, + }, + ] + }, + "activated": True, + } + ) + + self.client["Core"]["platforms"].insert_one( + { + "_id": platform_id1, + "name": "mediaWiki", + "metadata": { + "baseURL": "http://example1.com", + "path": "/api", + "namespaces": [0, 1, 2], + }, + "community": community_id, + "disconnectedAt": None, + "connectedAt": datetime.now(), + "createdAt": datetime.now(), + "updatedAt": datetime.now(), + } + ) + + self.client["Core"]["platforms"].insert_one( + { + "_id": platform_id2, + "name": "mediaWiki", + "metadata": { + "baseURL": "http://example2.com", + "path": "/api", + "namespaces": [3, 4, 5], + }, + "community": community_id, + "disconnectedAt": None, + "connectedAt": datetime.now(), + "createdAt": datetime.now(), + "updatedAt": datetime.now(), + } + ) + + result = self.modules_mediawiki.get_learning_platforms() + + self.assertIsInstance(result, list) + self.assertEqual(len(result), 1) + self.assertEqual( + result[0], + { + "community_id": str(community_id), + "namespaces": [3, 4, 5], + "base_url": "http://example2.com/api", + }, + ) diff --git a/tests/integration/test_module_website.py b/tests/integration/test_module_website.py index d86529c..6ec0d4c 100644 --- a/tests/integration/test_module_website.py +++ b/tests/integration/test_module_website.py @@ -54,6 +54,7 @@ def test_get_website_communities_data_single_modules(self): } ] }, + "activated": True, } ) @@ -115,6 +116,7 @@ def test_get_website_communities_data_module_multiple_platforms(self): }, ] }, + "activated": True, } ) @@ -191,6 +193,7 @@ def test_get_website_communities_data_module_multiple_platforms_filtered_one(sel }, ] }, + "activated": True, } ) @@ -269,6 +272,7 @@ def test_get_website_communities_data_module_multiple_platforms_one_disabled(sel }, ] }, + "activated": True, } )