diff --git a/docs/src/references/api.rst b/docs/src/references/api.rst index 33c4509..8fd9757 100644 --- a/docs/src/references/api.rst +++ b/docs/src/references/api.rst @@ -76,6 +76,10 @@ Providers :members: :undoc-members: +.. automodule:: multistorageclient.providers.minio + :members: + :undoc-members: + ********* Telemetry ********* diff --git a/docs/src/references/configuration.rst b/docs/src/references/configuration.rst index 2eed1ea..5da7408 100644 --- a/docs/src/references/configuration.rst +++ b/docs/src/references/configuration.rst @@ -169,6 +169,24 @@ Options: See parameters in :py:class:`multistorageclient.providers.s8k.S8KStorag region_name: us-east-1 endpoint_url: https://s8k.example.com +``minio`` +---------- + +Minio provider. + +Options: See parameters in :py:class:`multistorageclient.providers.minio.MinioStorageProvider`. + +.. code-block:: yaml + :caption: Example configuration. + + profiles: + my-profile: + storage_provider: + type: minio + options: + base_path: my-bucket + endpoint_url: https://play.min.io + ``azure`` --------- diff --git a/src/multistorageclient/config.py b/src/multistorageclient/config.py index 2453115..e154b05 100644 --- a/src/multistorageclient/config.py +++ b/src/multistorageclient/config.py @@ -97,6 +97,7 @@ def create_implicit_profile_config(profile_name: str, protocol: str, base_path: "ais": "AIStoreStorageProvider", "s8k": "S8KStorageProvider", "gcs_s3": "GoogleS3StorageProvider", + "minio": "MinioStorageProvider", } CREDENTIALS_PROVIDER_MAPPING = { diff --git a/src/multistorageclient/providers/__init__.py b/src/multistorageclient/providers/__init__.py index 60ec0ca..a44c0ce 100644 --- a/src/multistorageclient/providers/__init__.py +++ b/src/multistorageclient/providers/__init__.py @@ -50,6 +50,8 @@ def __getattr__(name: str) -> Any: # AIS "AIStoreStorageProvider": ".ais", "StaticAISCredentialProvider": ".ais", + # MinIO + "MinioStorageProvider": ".minio", } if name in module_map: @@ -69,6 +71,7 @@ def __getattr__(name: str) -> Any: ".s3": "boto3", ".s8k": "boto3", ".ais": "aistore", + ".minio": "boto3", } required_package = package_map.get(module_name, module_name.lstrip(".")) diff --git a/src/multistorageclient/providers/minio.py b/src/multistorageclient/providers/minio.py new file mode 100644 index 0000000..94f5207 --- /dev/null +++ b/src/multistorageclient/providers/minio.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .s3 import S3StorageProvider + +PROVIDER = "minio" + + +class MinioStorageProvider(S3StorageProvider): + """ + A concrete implementation of the :py:class:`multistorageclient.types.StorageProvider` for interacting with Minio. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # override the provider name from "s3" + self._provider_name = PROVIDER diff --git a/src/multistorageclient/rclone.py b/src/multistorageclient/rclone.py index 0394d5a..dc7be60 100644 --- a/src/multistorageclient/rclone.py +++ b/src/multistorageclient/rclone.py @@ -210,7 +210,7 @@ def _parse_config_section(section: configparser.SectionProxy) -> dict[str, Any]: # - rclone default storage type key (e.g. azureblob) # # Then, convert to storage type to MSC configuration storage key (e.g. azure). - if storage_type == "s3" or storage_type == "s8k": + if storage_type == "s3" or storage_type == "s8k" or storage_type == "minio": storage_provider_options, credentials_provider = _parse_s3_storage_provider_config(section) elif storage_type == "azure" or storage_type == "azureblob": storage_provider_options, credentials_provider = _parse_azure_storage_provider_config(section) diff --git a/src/multistorageclient/schema.py b/src/multistorageclient/schema.py index 479f2c6..163d835 100644 --- a/src/multistorageclient/schema.py +++ b/src/multistorageclient/schema.py @@ -113,7 +113,7 @@ "properties": { "type": { "type": "string", - "enum": ["ais", "azure", "file", "gcs", "gcs_s3", "oci", "s3", "s8k"], + "enum": ["ais", "azure", "file", "gcs", "gcs_s3", "oci", "s3", "s8k", "minio"], }, "options": { "type": "object", diff --git a/tests/test_multistorageclient/e2e/msc_config.template.yaml b/tests/test_multistorageclient/e2e/msc_config.template.yaml index c8ea6ac..5ebbe0b 100644 --- a/tests/test_multistorageclient/e2e/msc_config.template.yaml +++ b/tests/test_multistorageclient/e2e/msc_config.template.yaml @@ -123,6 +123,21 @@ profiles: multipart_threshold: 16777216 # 16MB multipart_chunksize: 4194304 # 4MB io_chunksize: 4194304 # 4MB + test-minio-play: + credentials_provider: + type: S3Credentials + options: + access_key: "*****" + secret_key: "*****" + storage_provider: + type: minio + options: + base_path: msc-integration-test-0001 + endpoint_url: http://play.min.io + region_name: us-east-1 + multipart_threshold: 16777216 # 16MB + multipart_chunksize: 4194304 # 4MB + io_chunksize: 4194304 # 4MB test-swift-pdx-rust: credentials_provider: type: S3Credentials diff --git a/tests/test_multistorageclient/e2e/rclone.template.conf b/tests/test_multistorageclient/e2e/rclone.template.conf index 33fc634..5182ce2 100644 --- a/tests/test_multistorageclient/e2e/rclone.template.conf +++ b/tests/test_multistorageclient/e2e/rclone.template.conf @@ -39,6 +39,14 @@ base_path = msc-integration-test-0001 access_key_id = ***** secret_access_key = ***** +[test-minio-play-rclone] +type = minio +region = us-east-1 +endpoint = https://play.min.io +base_path = msc-integration-test-0001 +access_key_id = ***** +secret_access_key = ***** + [test-swift-pdx-base-path-with-prefix-rclone] type = s8k region = us-east-1 diff --git a/tests/test_multistorageclient/unit/test_cache.py b/tests/test_multistorageclient/unit/test_cache.py index 825ee3b..8682a01 100644 --- a/tests/test_multistorageclient/unit/test_cache.py +++ b/tests/test_multistorageclient/unit/test_cache.py @@ -628,10 +628,11 @@ def test_storage_provider_partial_cache_config(storage_provider_partial_cache_co argvalues=[ [tempdatastore.TemporaryAWSS3Bucket, None], # S3 should work [tempdatastore.TemporarySwiftStackBucket, None], # SwiftStack (S8K) should work + [tempdatastore.TemporaryMinioBucket, None], # Minio should work [tempdatastore.TemporaryAzureBlobStorageContainer, ValueError], # Azure should fail [tempdatastore.TemporaryGoogleCloudStorageBucket, ValueError], # GCS should fail ], - ids=["s3", "swiftstack", "azure", "gcs"], + ids=["s3", "swiftstack", "minio", "azure", "gcs"], ) @pytest.fixture def no_eviction_cache_config(tmpdir): diff --git a/tests/test_multistorageclient/unit/test_config.py b/tests/test_multistorageclient/unit/test_config.py index a1d0525..b379679 100644 --- a/tests/test_multistorageclient/unit/test_config.py +++ b/tests/test_multistorageclient/unit/test_config.py @@ -238,6 +238,28 @@ def test_swiftstack_storage_provider() -> None: assert isinstance(config.storage_provider, S3StorageProvider) +def test_minio_storage_provider() -> None: + config = StorageClientConfig.from_json( + """{ + "profiles": { + "minio_profile": { + "storage_provider": { + "type": "minio", + "options": { + "base_path": "bucket", + "endpoint_url": "https://play.min.io", + "region_name": "us-east-1" + } + } + } + } + }""", + profile="minio_profile", + ) + + assert isinstance(config.storage_provider, S3StorageProvider) + + def test_manifest_provider_bundle() -> None: sys.path.append(os.path.dirname(__file__)) @@ -699,6 +721,37 @@ def test_s8k_storage_provider_passthrough_options() -> None: ) +def test_minio_storage_provider_passthrough_options() -> None: + profile = "data" + StorageClient( + config=StorageClientConfig.from_dict( + config_dict={ + "profiles": { + profile: { + "storage_provider": { + "type": "minio", + "options": { + "base_path": "bucket", + "endpoint_url": "https://play.min.io", + # Passthrough options. + "max_pool_connections": 1, + "connect_timeout": 1, + "read_timeout": 1, + "retries": { + "total_max_attempts": 2, + "max_attempts": 1, + "mode": "adaptive", + }, + }, + } + } + } + }, + profile=profile, + ) + ) + + def test_credentials_provider_with_base_path_endpoint_url() -> None: sys.path.append(os.path.dirname(__file__)) from test_multistorageclient.unit.utils.mocks import ( diff --git a/tests/test_multistorageclient/unit/test_schema.py b/tests/test_multistorageclient/unit/test_schema.py index dc38e53..c3e66d4 100644 --- a/tests/test_multistorageclient/unit/test_schema.py +++ b/tests/test_multistorageclient/unit/test_schema.py @@ -56,7 +56,7 @@ def test_validate_profiles(): ) # Valid configurations for s3 and swiftstack with Rust client options - for provider in ("s3", "s8k"): + for provider in ("s3", "s8k", "minio"): validate_config( { "profiles": { diff --git a/tests/test_multistorageclient/unit/utils/tempdatastore.py b/tests/test_multistorageclient/unit/utils/tempdatastore.py index ad24f35..f79bde2 100644 --- a/tests/test_multistorageclient/unit/utils/tempdatastore.py +++ b/tests/test_multistorageclient/unit/utils/tempdatastore.py @@ -271,3 +271,15 @@ class TemporarySwiftStackBucket(TemporaryAWSS3Bucket): def __init__(self, enable_rust_client: bool = False): super().__init__(enable_rust_client=enable_rust_client) self._profile_config_dict["storage_provider"]["type"] = "s8k" + + +class TemporaryMinioBucket(TemporaryAWSS3Bucket): + """ + This class creates a temporary Minio bucket. The resulting object can be used as a context manager. + On completion of the context or destruction of the temporary data store object, + the newly created temporary data store and all its contents are removed. + """ + + def __init__(self, enable_rust_client: bool = False): + super().__init__(enable_rust_client=enable_rust_client) + self._profile_config_dict["storage_provider"]["type"] = "minio"