From e89d9f0c519a2239e09ac43d62089d00e49e96f1 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 16 Apr 2025 12:08:31 +0330 Subject: [PATCH 1/4] feat: update mediaWiki platform & modules schema! --- hivemind_etl/mediawiki/module.py | 14 ++- tests/integration/test_mediawiki_modules.py | 94 +++++++++++++++++++-- 2 files changed, 101 insertions(+), 7 deletions(-) diff --git a/hivemind_etl/mediawiki/module.py b/hivemind_etl/mediawiki/module.py index 7dc4e70..3294e7b 100644 --- a/hivemind_etl/mediawiki/module.py +++ b/hivemind_etl/mediawiki/module.py @@ -62,16 +62,26 @@ def get_learning_platforms( platform_id=platform_id, metadata_name="path", ) + namespaces = self.get_platform_metadata( + platform_id=platform_id, + metadata_name="namespaces", + ) if not isinstance(path, str) and not isinstance(base_url, str): raise ValueError("Wrong format for `path` and `base_url`!") modules_options = platform["metadata"] - namespaces = modules_options.get("namespaces", []) + activated = modules_options["activated"] + + if not activated: + logging.warning( + f"Platform: {platform_id} is not activated! Skipping it..." + ) + continue if not namespaces: logging.warning( - f"No namespaces found for platform: {platform_id}" + f"No namespaces found for platform: {platform_id}. Skipping it..." ) continue diff --git a/tests/integration/test_mediawiki_modules.py b/tests/integration/test_mediawiki_modules.py index 1332fd2..86cd0ee 100644 --- a/tests/integration/test_mediawiki_modules.py +++ b/tests/integration/test_mediawiki_modules.py @@ -48,7 +48,7 @@ def test_get_single_data(self): "platform": platform_id, "name": "mediaWiki", "metadata": { - "namespaces": [0, 1, 2], + "activated": True, }, } ] @@ -85,14 +85,14 @@ def test_get_mediawiki_communities_data_multiple_platforms(self): "platform": platform_id1, "name": "mediaWiki", "metadata": { - "namespaces": [0, 1, 2], + "activated": True, }, }, { "platform": platform_id2, "name": "mediaWiki", "metadata": { - "namespaces": [3, 4, 5], + "activated": True, }, }, ] @@ -107,6 +107,7 @@ def test_get_mediawiki_communities_data_multiple_platforms(self): "metadata": { "baseURL": "http://example1.com", "path": "/api", + "namespaces": [0, 1, 2], }, "community": community_id, "disconnectedAt": None, @@ -123,6 +124,7 @@ def test_get_mediawiki_communities_data_multiple_platforms(self): "metadata": { "baseURL": "http://example2.com", "path": "/api", + "namespaces": [3, 4, 5], }, "community": community_id, "disconnectedAt": None, @@ -171,14 +173,14 @@ def test_get_mediawiki_communities_data_filtered_platforms(self): "platform": platform_id1, "name": "mediaWiki", "metadata": { - "namespaces": [0, 1, 2], + "activated": True, }, }, { "platform": platform_id2, "name": "mediaWiki", "metadata": { - "namespaces": [3, 4, 5], + "activated": True, }, }, ] @@ -193,6 +195,7 @@ def test_get_mediawiki_communities_data_filtered_platforms(self): "metadata": { "baseURL": "http://example1.com", "path": "/api", + "namespaces": [0, 1, 2], }, "community": community_id, "disconnectedAt": None, @@ -209,6 +212,7 @@ def test_get_mediawiki_communities_data_filtered_platforms(self): "metadata": { "baseURL": "http://example2.com", "path": "/api", + "namespaces": [3, 4, 5], }, "community": community_id, "disconnectedAt": None, @@ -232,3 +236,83 @@ def test_get_mediawiki_communities_data_filtered_platforms(self): "base_url": "http://example1.com/api", }, ) + + def test_get_mediawiki_communities_data_filtered_platforms_not_activated(self): + """ + Two mediawiki platforms for one community + """ + platform_id1 = ObjectId("6579c364f1120850414e0dc6") + platform_id2 = ObjectId("6579c364f1120850414e0dc7") + community_id = ObjectId("1009c364f1120850414e0dc5") + + self.client["Core"]["modules"].insert_one( + { + "name": "hivemind", + "community": community_id, + "options": { + "platforms": [ + { + "platform": platform_id1, + "name": "mediaWiki", + "metadata": { + "activated": False, + }, + }, + { + "platform": platform_id2, + "name": "mediaWiki", + "metadata": { + "activated": True, + }, + }, + ] + }, + } + ) + + self.client["Core"]["platforms"].insert_one( + { + "_id": platform_id1, + "name": "mediaWiki", + "metadata": { + "baseURL": "http://example1.com", + "path": "/api", + "namespaces": [0, 1, 2], + }, + "community": community_id, + "disconnectedAt": None, + "connectedAt": datetime.now(), + "createdAt": datetime.now(), + "updatedAt": datetime.now(), + } + ) + + self.client["Core"]["platforms"].insert_one( + { + "_id": platform_id2, + "name": "mediaWiki", + "metadata": { + "baseURL": "http://example2.com", + "path": "/api", + "namespaces": [3, 4, 5], + }, + "community": community_id, + "disconnectedAt": None, + "connectedAt": datetime.now(), + "createdAt": datetime.now(), + "updatedAt": datetime.now(), + } + ) + + result = self.modules_mediawiki.get_learning_platforms() + + self.assertIsInstance(result, list) + self.assertEqual(len(result), 1) + self.assertEqual( + result[0], + { + "community_id": str(community_id), + "namespaces": [3, 4, 5], + "base_url": "http://example2.com/api", + }, + ) From 0062dcf0cc8af5cd29ac0d794cc6d83b3faca0e0 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 16 Apr 2025 12:25:54 +0330 Subject: [PATCH 2/4] feat: added support for modules doc activated field! --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 144219e..bc6b6aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ python-dotenv>=1.0.0, <2.0.0 -tc-hivemind-backend==1.4.2.post2 +tc-hivemind-backend==1.4.3 llama-index-storage-docstore-redis==0.1.2 llama-index-storage-docstore-mongodb==0.1.3 crawlee[playwright]==0.3.8 From b9b29cb815473c12fb8e760b700827620cf8565f Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 16 Apr 2025 13:26:00 +0330 Subject: [PATCH 3/4] feat: Added modules activated field support! --- tests/integration/test_module_website.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/test_module_website.py b/tests/integration/test_module_website.py index d86529c..6ec0d4c 100644 --- a/tests/integration/test_module_website.py +++ b/tests/integration/test_module_website.py @@ -54,6 +54,7 @@ def test_get_website_communities_data_single_modules(self): } ] }, + "activated": True, } ) @@ -115,6 +116,7 @@ def test_get_website_communities_data_module_multiple_platforms(self): }, ] }, + "activated": True, } ) @@ -191,6 +193,7 @@ def test_get_website_communities_data_module_multiple_platforms_filtered_one(sel }, ] }, + "activated": True, } ) @@ -269,6 +272,7 @@ def test_get_website_communities_data_module_multiple_platforms_one_disabled(sel }, ] }, + "activated": True, } ) From b5341fc494e72adbce874d7011173d14350f2ee6 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 16 Apr 2025 13:30:45 +0330 Subject: [PATCH 4/4] feat: support activated field for modules! --- tests/integration/test_mediawiki_modules.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/test_mediawiki_modules.py b/tests/integration/test_mediawiki_modules.py index 86cd0ee..c60a308 100644 --- a/tests/integration/test_mediawiki_modules.py +++ b/tests/integration/test_mediawiki_modules.py @@ -53,6 +53,7 @@ def test_get_single_data(self): } ] }, + "activated": True, } ) @@ -97,6 +98,7 @@ def test_get_mediawiki_communities_data_multiple_platforms(self): }, ] }, + "activated": True, } ) @@ -185,6 +187,7 @@ def test_get_mediawiki_communities_data_filtered_platforms(self): }, ] }, + "activated": True, } ) @@ -267,6 +270,7 @@ def test_get_mediawiki_communities_data_filtered_platforms_not_activated(self): }, ] }, + "activated": True, } )