add minio provider

docs/src/references/api.rst

@@ -76,6 +76,10 @@ Providers
    :members:
    :undoc-members:
 
+.. automodule:: multistorageclient.providers.minio
+   :members:
+   :undoc-members:
+
 *********
 Telemetry
 *********

docs/src/references/configuration.rst

@@ -169,6 +169,24 @@ Options: See parameters in :py:class:`multistorageclient.providers.s8k.S8KStorag
            region_name: us-east-1
            endpoint_url: https://s8k.example.com
 
+``minio``
+----------
+
+Minio provider.
+
+Options: See parameters in :py:class:`multistorageclient.providers.minio.MinioStorageProvider`.
+
+.. code-block:: yaml
+   :caption: Example configuration.
+
+   profiles:
+     my-profile:
+       storage_provider:
+         type: minio
+         options:
+           base_path: my-bucket
+           endpoint_url: https://play.min.io
+
 ``azure``
 ---------
 

src/multistorageclient/config.py

@@ -97,6 +97,7 @@ def create_implicit_profile_config(profile_name: str, protocol: str, base_path:
     "ais": "AIStoreStorageProvider",
     "s8k": "S8KStorageProvider",
     "gcs_s3": "GoogleS3StorageProvider",
+    "minio": "MinioStorageProvider",
 }
 
 CREDENTIALS_PROVIDER_MAPPING = {

src/multistorageclient/providers/__init__.py

@@ -50,6 +50,8 @@ def __getattr__(name: str) -> Any:
         # AIS
         "AIStoreStorageProvider": ".ais",
         "StaticAISCredentialProvider": ".ais",
+        # MinIO
+        "MinioStorageProvider": ".minio",
     }
 
     if name in module_map:
@@ -69,6 +71,7 @@ def __getattr__(name: str) -> Any:
                 ".s3": "boto3",
                 ".s8k": "boto3",
                 ".ais": "aistore",
+                ".minio": "boto3",
             }
 
             required_package = package_map.get(module_name, module_name.lstrip("."))

src/multistorageclient/providers/minio.py

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .s3 import S3StorageProvider
+
+PROVIDER = "minio"
+
+
+class MinioStorageProvider(S3StorageProvider):
+    """
+    A concrete implementation of the :py:class:`multistorageclient.types.StorageProvider` for interacting with Minio.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # override the provider name from "s3"
+        self._provider_name = PROVIDER

src/multistorageclient/rclone.py

@@ -210,7 +210,7 @@ def _parse_config_section(section: configparser.SectionProxy) -> dict[str, Any]:
     #   - rclone default storage type key (e.g. azureblob)
     #
     # Then, convert to storage type to MSC configuration storage key (e.g. azure).
-    if storage_type == "s3" or storage_type == "s8k":
+    if storage_type == "s3" or storage_type == "s8k" or storage_type == "minio":
         storage_provider_options, credentials_provider = _parse_s3_storage_provider_config(section)
     elif storage_type == "azure" or storage_type == "azureblob":
         storage_provider_options, credentials_provider = _parse_azure_storage_provider_config(section)

src/multistorageclient/schema.py

@@ -113,7 +113,7 @@
                 "properties": {
                     "type": {
                         "type": "string",
-                        "enum": ["ais", "azure", "file", "gcs", "gcs_s3", "oci", "s3", "s8k"],
+                        "enum": ["ais", "azure", "file", "gcs", "gcs_s3", "oci", "s3", "s8k", "minio"],
                     },
                     "options": {
                         "type": "object",

tests/test_multistorageclient/e2e/msc_config.template.yaml

@@ -123,6 +123,21 @@ profiles:
         multipart_threshold: 16777216 # 16MB
         multipart_chunksize: 4194304 # 4MB
         io_chunksize: 4194304 # 4MB
+  test-minio-play:
+    credentials_provider:
+      type: S3Credentials
+      options:
+        access_key: "*****"
+        secret_key: "*****"
+    storage_provider:
+      type: minio
+      options:
+        base_path: msc-integration-test-0001
+        endpoint_url: http://play.min.io
+        region_name: us-east-1
+        multipart_threshold: 16777216 # 16MB
+        multipart_chunksize: 4194304 # 4MB
+        io_chunksize: 4194304 # 4MB
   test-swift-pdx-rust:
     credentials_provider:
       type: S3Credentials

tests/test_multistorageclient/e2e/rclone.template.conf

@@ -39,6 +39,14 @@ base_path = msc-integration-test-0001
 access_key_id = *****
 secret_access_key = *****
 
+[test-minio-play-rclone]
+type = minio
+region = us-east-1
+endpoint = https://play.min.io
+base_path = msc-integration-test-0001
+access_key_id = *****
+secret_access_key = *****
+
 [test-swift-pdx-base-path-with-prefix-rclone]
 type = s8k
 region = us-east-1

tests/test_multistorageclient/unit/test_cache.py

@@ -628,10 +628,11 @@ def test_storage_provider_partial_cache_config(storage_provider_partial_cache_co
     argvalues=[
         [tempdatastore.TemporaryAWSS3Bucket, None],  # S3 should work
         [tempdatastore.TemporarySwiftStackBucket, None],  # SwiftStack (S8K) should work
+        [tempdatastore.TemporaryMinioBucket, None],  # Minio should work
         [tempdatastore.TemporaryAzureBlobStorageContainer, ValueError],  # Azure should fail
         [tempdatastore.TemporaryGoogleCloudStorageBucket, ValueError],  # GCS should fail
     ],
-    ids=["s3", "swiftstack", "azure", "gcs"],
+    ids=["s3", "swiftstack", "minio", "azure", "gcs"],
 )
 @pytest.fixture
 def no_eviction_cache_config(tmpdir):

tests/test_multistorageclient/unit/test_config.py

@@ -238,6 +238,28 @@ def test_swiftstack_storage_provider() -> None:
     assert isinstance(config.storage_provider, S3StorageProvider)
 
 
+def test_minio_storage_provider() -> None:
+    config = StorageClientConfig.from_json(
+        """{
+        "profiles": {
+            "minio_profile": {
+                "storage_provider": {
+                    "type": "minio",
+                    "options": {
+                        "base_path": "bucket",
+                        "endpoint_url": "https://play.min.io",
+                        "region_name": "us-east-1"
+                    }
+                }
+            }
+        }
+    }""",
+        profile="minio_profile",
+    )
+
+    assert isinstance(config.storage_provider, S3StorageProvider)
+
+
 def test_manifest_provider_bundle() -> None:
     sys.path.append(os.path.dirname(__file__))
 
@@ -699,6 +721,37 @@ def test_s8k_storage_provider_passthrough_options() -> None:
     )
 
 
+def test_minio_storage_provider_passthrough_options() -> None:
+    profile = "data"
+    StorageClient(
+        config=StorageClientConfig.from_dict(
+            config_dict={
+                "profiles": {
+                    profile: {
+                        "storage_provider": {
+                            "type": "minio",
+                            "options": {
+                                "base_path": "bucket",
+                                "endpoint_url": "https://play.min.io",
+                                # Passthrough options.
+                                "max_pool_connections": 1,
+                                "connect_timeout": 1,
+                                "read_timeout": 1,
+                                "retries": {
+                                    "total_max_attempts": 2,
+                                    "max_attempts": 1,
+                                    "mode": "adaptive",
+                                },
+                            },
+                        }
+                    }
+                }
+            },
+            profile=profile,
+        )
+    )
+
+
 def test_credentials_provider_with_base_path_endpoint_url() -> None:
     sys.path.append(os.path.dirname(__file__))
     from test_multistorageclient.unit.utils.mocks import (

tests/test_multistorageclient/unit/test_schema.py

@@ -56,7 +56,7 @@ def test_validate_profiles():
         )
 
     # Valid configurations for s3 and swiftstack with Rust client options
-    for provider in ("s3", "s8k"):
+    for provider in ("s3", "s8k", "minio"):
         validate_config(
             {
                 "profiles": {

tests/test_multistorageclient/unit/utils/tempdatastore.py

@@ -271,3 +271,15 @@ class TemporarySwiftStackBucket(TemporaryAWSS3Bucket):
     def __init__(self, enable_rust_client: bool = False):
         super().__init__(enable_rust_client=enable_rust_client)
         self._profile_config_dict["storage_provider"]["type"] = "s8k"
+
+
+class TemporaryMinioBucket(TemporaryAWSS3Bucket):
+    """
+    This class creates a temporary Minio bucket. The resulting object can be used as a context manager.
+    On completion of the context or destruction of the temporary data store object,
+    the newly created temporary data store and all its contents are removed.
+    """
+
+    def __init__(self, enable_rust_client: bool = False):
+        super().__init__(enable_rust_client=enable_rust_client)
+        self._profile_config_dict["storage_provider"]["type"] = "minio"