From a55cb9bcabc8ce9f1df6e95e9d1d9cbc02406db2 Mon Sep 17 00:00:00 2001 From: ivis-kondo Date: Thu, 11 Dec 2025 15:06:57 +0900 Subject: [PATCH 1/5] add cross-service copy and traceback --- .../invenio_files_rest/config.py | 6 ++ .../invenio_files_rest/utils.py | 36 ++++++--- .../weko-records-ui/weko_records_ui/api.py | 79 ++++++++++++++++--- .../weko-records-ui/weko_records_ui/views.py | 2 +- 4 files changed, 99 insertions(+), 24 deletions(-) diff --git a/modules/invenio-files-rest/invenio_files_rest/config.py b/modules/invenio-files-rest/invenio_files_rest/config.py index 01f76c7a75..6f4bb4d27e 100644 --- a/modules/invenio-files-rest/invenio_files_rest/config.py +++ b/modules/invenio-files-rest/invenio_files_rest/config.py @@ -144,3 +144,9 @@ 'INVENIO_ROLE_COMMUNITY' ] """The version update roles.""" + +FILES_REST_STORAGE_SERVICE_PATTERN = { + "aws_s3": r"^s3[.-](?P[a-z0-9-]+)\.amazonaws\.com$|^s3\.amazonaws\.com$", + "wasabi": r"wasabisys\.com", +} +"""Storage service patterns for parsing storage host.""" diff --git a/modules/invenio-files-rest/invenio_files_rest/utils.py b/modules/invenio-files-rest/invenio_files_rest/utils.py index 1c30beeafd..151c4d12fe 100644 --- a/modules/invenio-files-rest/invenio_files_rest/utils.py +++ b/modules/invenio-files-rest/invenio_files_rest/utils.py @@ -172,6 +172,26 @@ def update_ogp_image(ogp_image, file_uri): return src.uri if src else None +def parse_storage_host(host): + """Parse storage host to get service and parameters. + Args: + host (str): Storage host. + Returns: + dict: Dictionary with service and parameters. + """ + service_patterns = current_app.config.get( + "FILES_REST_STORAGE_SERVICE_PATTERN", {}) + for service, patterns in service_patterns.items(): + for pattern in patterns: + result = re.match(pattern, host) + if result: + return { + "service": service, + "params": result.groupdict() + } + return {"service": None, "params": {}} + + def create_boto3_s3_client(access_key, secret_key, region_name=None, endpoint_url=None, client_config={}): """Create boto3 S3 client. @@ -202,20 +222,16 @@ def create_boto3_s3_client(access_key, secret_key, region_name=None, # Check endpoint if endpoint_url: parsed_url = urlparse(endpoint_url) - is_aws = re.match(r'.*\.amazonaws\.com$', parsed_url.netloc) + service_info = parse_storage_host(parsed_url.netloc) current_app.logger.debug( - "Endpoint URL '%s' is AWS: %s", endpoint_url, is_aws) + "Storage service info: %s", service_info) # Only set endpoint_url if it matches the pattern # Note: AWS S3 requires no endpoint_url for standard regions - if not is_aws: + if service_info["service"] != "aws_s3": client_kwargs["endpoint_url"] = endpoint_url - elif not region_name: - # get region from endpoint url for aws s3 - # ex: s3.us-west-2.amazonaws.com - pattern = r'^s3[.-](?P[a-z0-9-]+)\.amazonaws\.com$' - result = re.match(pattern, parsed_url.netloc) - if result: - region_name = result.group('region') or None + elif service_info["service"] == "aws_s3" and \ + "region" in service_info["params"]: + region_name = region_name or service_info["params"]["region"] # Check region if region_name: diff --git a/modules/weko-records-ui/weko_records_ui/api.py b/modules/weko-records-ui/weko_records_ui/api.py index 67f6920568..6ce0e36d65 100644 --- a/modules/weko-records-ui/weko_records_ui/api.py +++ b/modules/weko-records-ui/weko_records_ui/api.py @@ -9,7 +9,7 @@ import shutil import copy import json -from urllib.parse import urlparse +from urllib.parse import urlparse, urljoin, uses_relative, uses_netloc from email_validator import validate_email from flask import current_app, request @@ -21,6 +21,7 @@ from invenio_mail.admin import _load_mail_cfg_from_db, _set_flask_mail_cfg from invenio_files_rest.models import Bucket, ObjectVersion, FileInstance +from invenio_files_rest.utils import parse_storage_host from invenio_pidstore.models import PersistentIdentifier from invenio_records.api import Record from weko_logging.activity_logger import UserActivityLogger @@ -245,8 +246,41 @@ def get_s3_bucket_list(): def copy_bucket_to_s3( - pid, filename, org_bucket_id, checked, bucket_name -): + pid, filename, org_bucket_id, checked, bucket_name + ): + """ + Copy file to S3 bucket. + Args: + pid (str): Persistent identifier value. + filename (str): File name. + org_bucket_id (int): Original bucket ID. + checked (str): Create bucket option. + bucket_name (str): Destination bucket name. + Returns: + str: File URI. + """ + + def _is_same_storage_service(source_s3_client, destination_s3_client): + """Check if source and destination storage service are the same. + Args: + source_s3_client (boto3.client): Boto3 S3 client instance for source. + destination_s3_client (boto3.client): Boto3 S3 client instance for destination. + Returns: + bool: True if same service, False otherwise. + """ + source_url_info = urlparse(source_s3_client.meta.endpoint_url) + destination_url_info = urlparse(destination_s3_client.meta.endpoint_url) + source_service = parse_storage_host(source_url_info.netloc) + destination_service = parse_storage_host(destination_url_info.netloc) + if source_service["service"] is None or destination_service["service"] is None: + return source_url_info.netloc == destination_url_info.netloc + return source_service["service"] == destination_service["service"] + + # Add s3 to uses_relative and uses_netloc + if not "s3" in uses_relative: + uses_relative.append("s3") + if not "s3" in uses_netloc: + uses_netloc.append("s3") # Get profile profile = UserProfile.get_by_userid(current_user.id) @@ -318,7 +352,7 @@ def copy_bucket_to_s3( bucket_name, filename ) - return os.path.join(uri, filename) + return urljoin(uri, filename) except Exception as e: traceback.print_exc() raise Exception(_('Uploading file failed.')) @@ -367,7 +401,7 @@ def copy_bucket_to_s3( "Bucket": source_bucket_name, "Key": source_file_key } - current_app.logger.debug(f'copy_source: {copy_source}') + current_app.logger.debug(f"copy_source: {copy_source}") try: s3_client_source.head_object( @@ -378,19 +412,38 @@ def copy_bucket_to_s3( traceback.print_exc() raise Exception(_('The source file cannot be found.')) try: - # Note: Allowed same storage service - destination_s3_client.copy( - copy_source, - destination_bucket, - destination_key, - SourceClient=s3_client_source - ) + if _is_same_storage_service( + s3_client_source, + destination_s3_client + ): + # Same storage service + destination_s3_client.copy( + copy_source, + destination_bucket, + destination_key, + SourceClient=s3_client_source + ) + else: + # Different storage service + # Check file size + source_file_size = source_file_instance.size + # Upload file size limit 256MiB + file_size_limit = 1024 * 1024 * 256 # 256MiB + if source_file_size < file_size_limit: + obj = s3_client_source.get_object(Bucket=copy_source['Bucket'], Key=copy_source['Key']) + body = obj['Body'] + destination_s3_client.upload_fileobj( + body, destination_bucket, destination_key + ) + else: + raise Exception(_('The source file size exceeds the limit for cross-service copy.')) except Exception as e: traceback.print_exc() current_app.logger.error(e) raise Exception(_('Uploading file failed.')) - return os.path.join(uri, filename) + # Return file URI + return urljoin(uri, filename) def create_storage_bucket(s3_client, endpoint_url, region_name, bucket_name): """Create storage bucket. diff --git a/modules/weko-records-ui/weko_records_ui/views.py b/modules/weko-records-ui/weko_records_ui/views.py index 46d0f8ded4..67322ad2de 100644 --- a/modules/weko-records-ui/weko_records_ui/views.py +++ b/modules/weko-records-ui/weko_records_ui/views.py @@ -1491,7 +1491,7 @@ def copy_bucket(): return jsonify(uri) except Exception as e: current_app.logger.error(str(e)) - traceback.print_exc() + current_app.logger.error(traceback.format_exc()) return jsonify({'error': str(e)}), 400 From 9463d1413772137b7f9c9a493c6458ece07ac396 Mon Sep 17 00:00:00 2001 From: ivis-kondo Date: Tue, 23 Dec 2025 19:17:05 +0900 Subject: [PATCH 2/5] add config and fix exception for cross service copy --- .../invenio_files_rest/config.py | 12 +- .../invenio-files-rest/tests/test_utils.py | 75 +++++- modules/weko-records-ui/tests/test_api.py | 252 ++++++++++++++++-- .../weko-records-ui/weko_records_ui/api.py | 31 ++- .../weko-records-ui/weko_records_ui/config.py | 3 + 5 files changed, 342 insertions(+), 31 deletions(-) diff --git a/modules/invenio-files-rest/invenio_files_rest/config.py b/modules/invenio-files-rest/invenio_files_rest/config.py index 6f4bb4d27e..df7b920a14 100644 --- a/modules/invenio-files-rest/invenio_files_rest/config.py +++ b/modules/invenio-files-rest/invenio_files_rest/config.py @@ -146,7 +146,15 @@ """The version update roles.""" FILES_REST_STORAGE_SERVICE_PATTERN = { - "aws_s3": r"^s3[.-](?P[a-z0-9-]+)\.amazonaws\.com$|^s3\.amazonaws\.com$", - "wasabi": r"wasabisys\.com", + "aws_s3": [ + r"^s3\.(?P[a-z0-9-]+)\.amazonaws\.com$", + r"^(?P.+)\.s3\.(?P[a-z0-9-]+)\.amazonaws\.com$", + r"^(?P.+)\.s3\.amazonaws\.com$", + r"^s3\.amazonaws\.com$", + ], + "wasabi": [ + r"^s3\.wasabisys\.com$", + r"^s3\.(?P[a-z0-9-]+)\.wasabisys\.com$", + ], } """Storage service patterns for parsing storage host.""" diff --git a/modules/invenio-files-rest/tests/test_utils.py b/modules/invenio-files-rest/tests/test_utils.py index 13e8655ad9..810ddf2518 100644 --- a/modules/invenio-files-rest/tests/test_utils.py +++ b/modules/invenio-files-rest/tests/test_utils.py @@ -9,8 +9,10 @@ """Files download/upload REST API similar to S3 for Invenio.""" import botocore.client -from invenio_files_rest.utils import create_boto3_s3_client +from invenio_files_rest.utils import create_boto3_s3_client, parse_storage_host + +# def create_boto3_s3_client(access_key, secret_key, region_name=None, endpoint_url=None, client_config={}): # .tox/c1/bin/pytest --cov=invenio_files_rest tests/test_utils.py::test_create_boto3_s3_client -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-files-rest/.tox/c1/tmp def test_create_boto3_s3_client(app): # Test case (Pos): S3 credentials are not configured @@ -27,6 +29,17 @@ def test_create_boto3_s3_client(app): assert client.meta.region_name == "eu-west-1" assert client.meta.endpoint_url == "https://s3.eu-west-1.amazonaws.com" + # Set app config for testing + app.config.update({ + "FILES_REST_STORAGE_SERVICE_PATTERN": { + "aws_s3": [ + r"^s3\.amazonaws\.com$", + r"^s3\.(?P.+)\.amazonaws\.com$", + r"^(?P.+)\.s3\.(?P.+)\.amazonaws\.com$" + ], + } + }) + # Test case (Pos): region_name is set client = create_boto3_s3_client( "access_key", "secret_key", @@ -61,3 +74,63 @@ def test_create_boto3_s3_client(app): assert isinstance(client, botocore.client.BaseClient) assert client.meta.service_model.service_name == "s3" assert client.meta.endpoint_url == "https://example.com" + + +# def parse_storage_host(host): +# .tox/c1/bin/pytest --cov=invenio_files_rest tests/test_utils.py::test_parse_storage_host_match_service -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-files-rest/.tox/c1/tmp +def test_parse_storage_host_match_service(app): + # サービスパターン設定 + patterns = { + "aws_s3": [ + r"^s3\.amazonaws\.com$", + r"^(?P.+)\.s3\.amazonaws\.com$" + ], + "wasabi": [ + r"^s3\.wasabisys\.com$", + r"^s3\.(?P[a-z0-9-]+)\.wasabisys\.com$" + ], + } + mock_config = {"FILES_REST_STORAGE_SERVICE_PATTERN": patterns} + app.config.update(mock_config) + + # Test case (Pos): Match aws_s3 pattern + result = parse_storage_host("s3.amazonaws.com") + assert result["service"] == "aws_s3" + assert result["params"] == {} + # Test case (Pos): Match aws_s3 pattern with bucket name + result2 = parse_storage_host("mybucket.s3.amazonaws.com") + assert result2["service"] == "aws_s3" + assert result2["params"]["bucket"] == "mybucket" + # Test case (Pos): Match wasabi pattern + result3 = parse_storage_host("s3.wasabisys.com") + assert result3["service"] == "wasabi" + assert result3["params"] == {} + # Test case (Pos): Match wasabi pattern with region + result4 = parse_storage_host("s3.us-west-1.wasabisys.com") + assert result4["service"] == "wasabi" + assert result4["params"]["region"] == "us-west-1" + + +# def parse_storage_host(host): +# .tox/c1/bin/pytest --cov=invenio_files_rest tests/test_utils.py::test_parse_storage_host_no_match -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-files-rest/.tox/c1/tmp +def test_parse_storage_host_no_match(app): + # Test case (Pos): No match any pattern + patterns = { + "aws_s3": [r"^s3\.amazonaws\.com$"], + } + mock_config = {"FILES_REST_STORAGE_SERVICE_PATTERN": patterns} + app.config.update(mock_config) + result = parse_storage_host("unknown.host.com") + assert result["service"] is None + assert result["params"] == {} + + +# def parse_storage_host(host): +# .tox/c1/bin/pytest --cov=invenio_files_rest tests/test_utils.py::test_parse_storage_host_empty_patterns -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-files-rest/.tox/c1/tmp +def test_parse_storage_host_empty_patterns(app): + # Test case (Pos): Empty patterns + mock_config = {"FILES_REST_STORAGE_SERVICE_PATTERN": {}} + app.config.update(mock_config) + result = parse_storage_host("s3.amazonaws.com") + assert result["service"] is None + assert result["params"] == {} diff --git a/modules/weko-records-ui/tests/test_api.py b/modules/weko-records-ui/tests/test_api.py index dc3f1530b0..5840095c74 100644 --- a/modules/weko-records-ui/tests/test_api.py +++ b/modules/weko-records-ui/tests/test_api.py @@ -3,6 +3,7 @@ from smtplib import SMTPException from mock import patch import pytest +from botocore.exceptions import ClientError from flask import url_for,current_app,make_response from flask_login import UserMixin import uuid @@ -370,6 +371,15 @@ def test_copy_bucket_to_s3( records_buckets = RecordsBuckets.query.filter_by( record_id=pid.object_uuid ).first() + app.config.update({ + "FILES_REST_STORAGE_SERVICE_PATTERN": { + "aws_s3": [ + r"^s3\.amazonaws\.com$", + r"^s3\.(?P.+)\.amazonaws\.com$", + r"^(?P.+)\.s3\.(?P.+)\.amazonaws\.com$" + ], + } + }) # Get user profile user_profile_s3 = users_storage_info["s3_storage_user"] # S3 storage profile (Repository) @@ -473,12 +483,35 @@ def test_copy_bucket_to_s3( mocker_boto3_client_src = mocker.Mock() mocker_boto3_client_src.head_object.return_value = True mocker_boto3_client_src.copy.return_value = None + mocker_boto3_client_src.meta.endpoint_url = "https://s3.us-west-2.amazonaws.com" # Prepare s3 storage file instance mock_file_instance_s3 = mocker.Mock() mock_file_instance_s3.uri = "s3://bucket-name/helloworld.pdf" + # Test Case (Pos): source_location is None + loc_s3_vh = Location( + name="testloc-s3-vh-path", + uri="https://s3.us-west-2.amazonaws.com/bucket-name/", + default=False, + type="s3_vh", + access_key="access_key", + secret_key="secret_key", + s3_region_name="us-west-2", + ) + with patch("invenio_files_rest.models.FileInstance.get", return_value=mock_file_instance_s3): + mocker.patch.object(loc_s3_vh, "create_s3_client", return_value=mocker_boto3_client_src) + uri = copy_bucket_to_s3( + pid=1, filename="helloworld.pdf", + org_bucket_id=records_buckets.bucket_id, + checked="create", bucket_name="sample1" + ) + assert uri == f"https://s3.{s3_storage_region_name}.amazonaws.com/sample1/helloworld.pdf" + mock_profile_boto3_client.copy.assert_not_called() + mock_profile_boto3_client.copy.reset_mock() + # Test Case (Pos): storage s3 to s3 (path style) + mock_get_default = mocker.patch("invenio_files_rest.models.Location.get_default") loc_s3 = Location( name="testloc-s3", uri="s3://bucket-name/", @@ -489,7 +522,7 @@ def test_copy_bucket_to_s3( secret_key="secret_key", s3_region_name="us-west-2", ) - mock_file_instance_s3.get_location_by_file_instance.return_value = loc_s3 + mock_get_default.return_value = loc_s3 with patch("invenio_files_rest.models.FileInstance.get", return_value=mock_file_instance_s3): mocker.patch.object(loc_s3, "create_s3_client", return_value=mocker_boto3_client_src) uri = copy_bucket_to_s3( @@ -511,7 +544,7 @@ def test_copy_bucket_to_s3( secret_key="secret_key", s3_region_name="us-west-2", ) - mock_file_instance_s3.get_location_by_file_instance.return_value = loc_s3_vh + mock_get_default.return_value = loc_s3_vh with patch("invenio_files_rest.models.FileInstance.get", return_value=mock_file_instance_s3): mocker.patch.object(loc_s3_vh, "create_s3_client", return_value=mocker_boto3_client_src) uri = copy_bucket_to_s3( @@ -533,7 +566,7 @@ def test_copy_bucket_to_s3( secret_key="secret_key", s3_region_name="us-west-2", ) - mock_file_instance_s3.get_location_by_file_instance.return_value = loc_s3_vh + mock_get_default.return_value = loc_s3_vh with patch("invenio_files_rest.models.FileInstance.get", return_value=mock_file_instance_s3): mocker.patch.object(loc_s3_vh, "create_s3_client", return_value=mocker_boto3_client_src) uri = copy_bucket_to_s3( @@ -545,21 +578,8 @@ def test_copy_bucket_to_s3( mock_profile_boto3_client.copy.assert_called_once() mock_profile_boto3_client.copy.reset_mock() - # Test Case (Pos): source_location is None - mock_file_instance_s3.get_location_by_file_instance.return_value = None - with patch("invenio_files_rest.models.FileInstance.get", return_value=mock_file_instance_s3): - mocker.patch.object(loc_s3_vh, "create_s3_client", return_value=mocker_boto3_client_src) - uri = copy_bucket_to_s3( - pid=1, filename="helloworld.pdf", - org_bucket_id=records_buckets.bucket_id, - checked="create", bucket_name="sample1" - ) - assert uri == f"https://s3.{s3_storage_region_name}.amazonaws.com/sample1/helloworld.pdf" - mock_profile_boto3_client.copy.assert_not_called() - mock_profile_boto3_client.copy.reset_mock() - mock_file_instance_s3.get_location_by_file_instance.return_value = loc_s3_vh - # Test Case (Pos): bucket_region is None + mock_get_default.return_value = loc_s3_vh mock_profile_boto3_client.get_bucket_location.return_value = { "LocationConstraint": None } @@ -615,7 +635,7 @@ def test_copy_bucket_to_s3( secret_key="secret_key", s3_region_name="us-west-2", ) - mock_file_instance_s3.get_location_by_file_instance.return_value = loc_s3_other + mock_get_default.return_value = loc_s3_other with patch("invenio_files_rest.models.FileInstance.get", return_value=mock_file_instance_s3): mocker.patch.object(loc_s3_vh, "create_s3_client", return_value=mocker_boto3_client_src) with pytest.raises(Exception) as e: @@ -628,7 +648,7 @@ def test_copy_bucket_to_s3( mock_profile_boto3_client.copy.reset_mock() # Test Case (Neg): s3 to s3, head object not found - mock_file_instance_s3.get_location_by_file_instance.return_value = loc_s3 + mock_get_default.return_value = loc_s3 mocker_boto3_client_src.head_object.side_effect = Exception("Not found") with patch("invenio_files_rest.models.FileInstance.get", return_value=mock_file_instance_s3): mocker.patch.object(loc_s3, "create_s3_client", return_value=mocker_boto3_client_src) @@ -641,7 +661,7 @@ def test_copy_bucket_to_s3( mocker_boto3_client_src.head_object.side_effect = None # Test Case (Neg): s3 to s3, head object not found - mock_file_instance_s3.get_location_by_file_instance.return_value = loc_s3 + mock_get_default.return_value = loc_s3 mocker_boto3_client_src.head_object.return_value = True mock_profile_boto3_client.copy.side_effect = Exception("Copy error") with patch("invenio_files_rest.models.FileInstance.get", return_value=mock_file_instance_s3): @@ -684,6 +704,198 @@ def test_copy_bucket_to_s3( assert uri +# def copy_bucket_to_s3(pid, filename, org_bucket_id, checked, bucket_name): +# .tox/c1/bin/pytest --cov=weko_records_ui tests/test_api.py::test_copy_bucket_to_s3_cross_service -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-records-ui/.tox/c1/tmp +def test_copy_bucket_to_s3_cross_service( + app, db, users_storage_info, client, records, mocker +): + # Prepare user profile with complete client credentials info + pid = PersistentIdentifier.query.filter_by( + pid_type="recid", pid_value='1' + ).first() + records_buckets = RecordsBuckets.query.filter_by( + record_id=pid.object_uuid + ).first() + app.config.update({ + "FILES_REST_STORAGE_SERVICE_PATTERN": { + "aws_s3": [ + r"^s3\.amazonaws\.com$", + r"^s3\.(?P.+)\.amazonaws\.com$", + r"^(?P.+)\.s3\.(?P.+)\.amazonaws\.com$" + ], + "other_s3": [ + r"^s3\.custom\.endpoint\.com$", + r"^s3\.(?P.+)\.custom\.endpoint\.com$", + r"^(?P.+)\.s3\.(?P.+)\.custom\.endpoint\.com$" + ] + } + }) + + # Get user profile + user_profile_s3 = users_storage_info["not_s3_storage_user"] # Other S3 storage profile (Repository) + non_s3_storage_user = user_profile_s3["user_info"]["obj"] + non_s3_storage_user_profile = user_profile_s3["profile_obj"] + non_s3_storage_region_name = non_s3_storage_user_profile.s3_region_name + login(client, obj=non_s3_storage_user) + + # Mock UserProfile.create_s3_client to return mock boto3 client + mock_profile_boto3_client = mocker.Mock() + mocker.patch("weko_user_profiles.models.UserProfile.create_s3_client", return_value=mock_profile_boto3_client) + mock_profile_boto3_client.meta.endpoint_url = non_s3_storage_user_profile.s3_endpoint_url + mock_profile_boto3_client.upload_file.return_value = None + mock_profile_boto3_client.get_bucket_location.return_value = { + "LocationConstraint": non_s3_storage_user_profile.s3_region_name, + } + mock_profile_boto3_client.upload_fileobj.return_value = None + + mocker_boto3_client_src = mocker.Mock() + mocker_boto3_client_src.head_object.return_value = True + mocker_boto3_client_src.copy.return_value = None + mocker_boto3_client_src.meta.endpoint_url = "https://s3.us-west-2.amazonaws.com" + mocker_boto3_client_src.get_object.return_value = {"Body": mocker.Mock()} + + # Prepare s3 storage file instance + mock_file_instance_s3 = mocker.Mock() + mock_file_instance_s3.uri = "s3://bucket-name/helloworld.pdf" + mock_file_instance_s3.size = 1234 + + mock_get_default = mocker.patch("invenio_files_rest.models.Location.get_default", return_value=non_s3_storage_user_profile) + + # Test Case (Pos): storage s3 to other s3 (path style) + loc_s3 = Location( + name="testloc-s3", + uri="s3://bucket-name/", + default=False, + type="s3", + s3_endpoint_url="https://s3.us-west-2.amazonaws.com/", + access_key="access_key", + secret_key="secret_key", + s3_region_name="us-west-2", + ) + mock_get_default.return_value = loc_s3 + with patch("invenio_files_rest.models.FileInstance.get", return_value=mock_file_instance_s3): + mocker.patch.object(loc_s3, "create_s3_client", return_value=mocker_boto3_client_src) + uri = copy_bucket_to_s3( + pid=1, filename="helloworld.pdf", + org_bucket_id=records_buckets.bucket_id, + checked="select", bucket_name="sample1" + ) + assert uri == non_s3_storage_user_profile.s3_endpoint_url + "sample1/helloworld.pdf" + mock_profile_boto3_client.copy.assert_not_called() + mock_profile_boto3_client.upload_fileobj.assert_called_once() + mock_profile_boto3_client.upload_fileobj.reset_mock() + + mock_create_storage_bucket = mocker.patch("weko_records_ui.api.create_storage_bucket", return_value=None) + + # Test Case (Pos): s3 to other s3 (virtual host style) + loc_s3_vh = Location( + name="testloc-s3-vh", + uri="https://bucket-name.s3.us-west-2.amazonaws.com/", + default=False, + type="s3_vh", + access_key="access_key", + secret_key="secret_key", + s3_region_name="us-west-2", + ) + mock_get_default.return_value = loc_s3_vh + with patch("invenio_files_rest.models.FileInstance.get", return_value=mock_file_instance_s3): + mocker.patch.object(loc_s3_vh, "create_s3_client", return_value=mocker_boto3_client_src) + uri = copy_bucket_to_s3( + pid=1, filename="helloworld.pdf", + org_bucket_id=records_buckets.bucket_id, + checked="create", bucket_name="sample1" + ) + assert uri == non_s3_storage_user_profile.s3_endpoint_url + "sample1/helloworld.pdf" + mock_create_storage_bucket.assert_called_once() + mock_profile_boto3_client.upload_fileobj.assert_called_once() + mock_profile_boto3_client.upload_fileobj.reset_mock() + + # Test Case (Pos): s3 to other s3 (virtual host style, path style uri) + loc_s3_vh = Location( + name="testloc-s3-vh-path", + uri="https://s3.us-west-2.amazonaws.com/bucket-name/", + default=False, + type="s3_vh", + access_key="access_key", + secret_key="secret_key", + s3_region_name="us-west-2", + ) + mock_get_default.return_value = loc_s3_vh + with patch("invenio_files_rest.models.FileInstance.get", return_value=mock_file_instance_s3): + mocker.patch.object(loc_s3_vh, "create_s3_client", return_value=mocker_boto3_client_src) + uri = copy_bucket_to_s3( + pid=1, filename="helloworld.pdf", + org_bucket_id=records_buckets.bucket_id, + checked="create", bucket_name="sample1" + ) + assert uri == non_s3_storage_user_profile.s3_endpoint_url + "sample1/helloworld.pdf" + mock_profile_boto3_client.upload_fileobj.assert_called_once() + mock_profile_boto3_client.upload_fileobj.reset_mock() + + # Test Case (Neg): uploading file ClientError + loc_s3_vh = Location( + name="testloc-s3-vh-path", + uri="https://s3.us-west-2.amazonaws.com/bucket-name/", + default=False, + type="s3_vh", + access_key="access_key", + secret_key="secret_key", + s3_region_name="us-west-2", + ) + mock_get_default.return_value = loc_s3_vh + with patch("invenio_files_rest.models.FileInstance.get", return_value=mock_file_instance_s3): + mocker.patch.object(loc_s3_vh, "create_s3_client", return_value=mocker_boto3_client_src) + mock_profile_boto3_client.upload_fileobj.side_effect = ClientError( + error_response={"Error": {"Code": "403", "Message": "Forbidden"}}, + operation_name="UploadFileObj" + ) + with pytest.raises(Exception) as e: + uri = copy_bucket_to_s3( + pid=1, filename="helloworld.pdf", + org_bucket_id=records_buckets.bucket_id, + checked="create", bucket_name="sample1" + ) + assert "Uploading file failed. Please make sure you have write permissions or that the bucket is writable." == str(e.value) + mock_profile_boto3_client.upload_fileobj.reset_mock() + mock_profile_boto3_client.upload_fileobj.side_effect = None + + # Test Case (Neg): file size is too large + loc_s3_vh = Location( + name="testloc-s3-vh-path", + uri="https://s3.us-west-2.amazonaws.com/bucket-name/", + default=False, + type="s3_vh", + access_key="access_key", + secret_key="secret_key", + s3_region_name="us-west-2", + ) + mock_get_default.return_value = loc_s3_vh + mock_file_instance_s3.size = 20 * 1024 * 1024 * 1024 + 1 + with patch("invenio_files_rest.models.FileInstance.get", return_value=mock_file_instance_s3): + mocker.patch.object(loc_s3_vh, "create_s3_client", return_value=mocker_boto3_client_src) + with pytest.raises(Exception) as e: + uri = copy_bucket_to_s3( + pid=1, filename="helloworld.pdf", + org_bucket_id=records_buckets.bucket_id, + checked="create", bucket_name="sample1" + ) + assert "The source file size exceeds the limit for cross-service copy." == str(e.value) + mock_profile_boto3_client.upload_fileobj.reset_mock() + + # Test Case (Pos): WEKO_RECORDS_UI_S3_CROSS_COPY_MAX_FILE_SIZE is bigger than config value + app.config["WEKO_RECORDS_UI_S3_CROSS_COPY_MAX_FILE_SIZE"] = 40 * 1024 * 1024 * 1024 # 40 GB + with patch("invenio_files_rest.models.FileInstance.get", return_value=mock_file_instance_s3): + mocker.patch.object(loc_s3_vh, "create_s3_client", return_value=mocker_boto3_client_src) + uri = copy_bucket_to_s3( + pid=1, filename="helloworld.pdf", + org_bucket_id=records_buckets.bucket_id, + checked="create", bucket_name="sample1" + ) + assert uri == non_s3_storage_user_profile.s3_endpoint_url + "sample1/helloworld.pdf" + mock_profile_boto3_client.upload_fileobj.assert_called_once() + mock_profile_boto3_client.upload_fileobj.reset_mock() + + # def create_storage_bucket(s3_client, endpoint_url, region_name, bucket_name): # .tox/c1/bin/pytest --cov=weko_records_ui tests/test_api.py::test_create_storage_bucket_bucket_exists -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-records-ui/.tox/c1/tmp def test_create_storage_bucket_bucket_exists(mocker): diff --git a/modules/weko-records-ui/weko_records_ui/api.py b/modules/weko-records-ui/weko_records_ui/api.py index 6ce0e36d65..5da713fcfe 100644 --- a/modules/weko-records-ui/weko_records_ui/api.py +++ b/modules/weko-records-ui/weko_records_ui/api.py @@ -11,6 +11,7 @@ import json from urllib.parse import urlparse, urljoin, uses_relative, uses_netloc +from botocore.exceptions import BotoCoreError, ClientError from email_validator import validate_email from flask import current_app, request @@ -292,7 +293,11 @@ def _is_same_storage_service(source_s3_client, destination_s3_client): source_content_info = ObjectVersion.get(bucket=source_bucket, key=filename) source_file_instance = FileInstance.get(source_content_info.file_id) - source_location = source_file_instance.get_location_by_file_instance() + # source_location = source_file_instance.get_location_by_file_instance() + source_location = db.session.query(Location).filter( + func.cast(source_file_instance.uri, String).like(Location.uri + "%")) \ + .order_by(func.length(Location.uri).desc()) \ + .first() if source_location is None: source_location = Location.get_default() @@ -427,20 +432,30 @@ def _is_same_storage_service(source_s3_client, destination_s3_client): # Different storage service # Check file size source_file_size = source_file_instance.size - # Upload file size limit 256MiB - file_size_limit = 1024 * 1024 * 256 # 256MiB + # Upload file size limit from config + file_size_limit = current_app.config.get( + "WEKO_RECORDS_UI_S3_CROSS_COPY_MAX_FILE_SIZE", + 20 * 1024 * 1024 * 1024 + ) if source_file_size < file_size_limit: - obj = s3_client_source.get_object(Bucket=copy_source['Bucket'], Key=copy_source['Key']) - body = obj['Body'] + obj = s3_client_source.get_object( + Bucket=copy_source["Bucket"], + Key=copy_source["Key"] + ) + body = obj["Body"] destination_s3_client.upload_fileobj( body, destination_bucket, destination_key ) else: - raise Exception(_('The source file size exceeds the limit for cross-service copy.')) - except Exception as e: - traceback.print_exc() + raise Exception(_("The source file size exceeds the limit for cross-service copy.")) + except (ClientError, BotoCoreError) as e: current_app.logger.error(e) + current_app.logger.error(traceback.format_exc()) raise Exception(_('Uploading file failed.')) + except Exception as e: + current_app.logger.error(e) + current_app.logger.error(traceback.format_exc()) + raise # Return file URI return urljoin(uri, filename) diff --git a/modules/weko-records-ui/weko_records_ui/config.py b/modules/weko-records-ui/weko_records_ui/config.py index 1cf40b5c86..2226090047 100644 --- a/modules/weko-records-ui/weko_records_ui/config.py +++ b/modules/weko-records-ui/weko_records_ui/config.py @@ -796,6 +796,9 @@ WEKO_RECORDS_UI_OA_API_CODE = "oaa" +WEKO_RECORDS_UI_S3_CROSS_COPY_MAX_FILE_SIZE = 20 * 1024 * 1024 * 1024 # 20GB +"""Max file size for s3 compatible service cross copy.""" + class EXTERNAL_SYSTEM(Enum): OA = "OA" From 4f072fb8fdb354dd4502183ec41c47a28dfe374a Mon Sep 17 00:00:00 2001 From: ivis-kondo Date: Wed, 24 Dec 2025 10:38:55 +0900 Subject: [PATCH 3/5] add translation file --- modules/weko-records-ui/tests/test_api.py | 4 +- .../weko-records-ui/weko_records_ui/api.py | 4 +- .../translations/en/LC_MESSAGES/messages.po | 236 +++++++++--------- .../translations/ja/LC_MESSAGES/messages.po | 235 +++++++++-------- .../weko_records_ui/translations/messages.pot | 203 ++++++++------- 5 files changed, 344 insertions(+), 338 deletions(-) diff --git a/modules/weko-records-ui/tests/test_api.py b/modules/weko-records-ui/tests/test_api.py index 5840095c74..092a2992f0 100644 --- a/modules/weko-records-ui/tests/test_api.py +++ b/modules/weko-records-ui/tests/test_api.py @@ -644,7 +644,7 @@ def test_copy_bucket_to_s3( org_bucket_id=records_buckets.bucket_id, checked="create", bucket_name="sample1" ) - assert "The source bucket or file key is not set." in str(e.value) + assert "The source bucket or file cannot be found." in str(e.value) mock_profile_boto3_client.copy.reset_mock() # Test Case (Neg): s3 to s3, head object not found @@ -1268,7 +1268,7 @@ def test_get_file_place_info(app, db, users, client, records, mocker): org_bucket_id=records_buckets.bucket_id, file_name='helloworld.pdf' ) - assert "The file cannot be found." in str(e.value) + assert "The source bucket or file cannot be found." in str(e.value) # .tox/c1/bin/pytest --cov=weko_records_ui tests/test_api.py::test_replace_file_bucket -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-records-ui/.tox/c1/tmp diff --git a/modules/weko-records-ui/weko_records_ui/api.py b/modules/weko-records-ui/weko_records_ui/api.py index 5da713fcfe..a30f8f8aa3 100644 --- a/modules/weko-records-ui/weko_records_ui/api.py +++ b/modules/weko-records-ui/weko_records_ui/api.py @@ -400,7 +400,7 @@ def _is_same_storage_service(source_s3_client, destination_s3_client): # Validate source file bucket and key if not source_bucket_name or not source_file_key: - raise Exception(_("The source bucket or file key is not set.")) + raise Exception(_("The source bucket or file cannot be found.")) copy_source = { "Bucket": source_bucket_name, @@ -657,7 +657,7 @@ def get_file_place_info(org_pid, org_bucket_id, file_name): prev_key_parts[:-new_key_depth]) + new_file_key if not bucket_name or not new_file_key: - raise Exception(_("The file cannot be found.")) + raise Exception(_("The source bucket or file cannot be found.")) try: s3_client = location.create_s3_client() diff --git a/modules/weko-records-ui/weko_records_ui/translations/en/LC_MESSAGES/messages.po b/modules/weko-records-ui/weko_records_ui/translations/en/LC_MESSAGES/messages.po index 05e43f6058..e10a237902 100644 --- a/modules/weko-records-ui/weko_records_ui/translations/en/LC_MESSAGES/messages.po +++ b/modules/weko-records-ui/weko_records_ui/translations/en/LC_MESSAGES/messages.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: weko-records-ui 0.1.0.dev20170000\n" "Report-Msgid-Bugs-To: wekosoftware@nii.ac.jp\n" -"POT-Creation-Date: 2025-10-21 14:06+0900\n" +"POT-Creation-Date: 2025-12-24 10:03+0900\n" "PO-Revision-Date: 2018-04-12 18:06+0900\n" "Last-Translator: FULL NAME \n" "Language: en\n" @@ -19,16 +19,16 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 2.5.1\n" -#: tests/test_utils.py:712 weko_records_ui/api.py:634 weko_records_ui/fd.py:651 -#: weko_records_ui/fd.py:729 weko_records_ui/utils.py:1213 +#: tests/test_utils.py:717 weko_records_ui/api.py:678 weko_records_ui/fd.py:650 +#: weko_records_ui/fd.py:728 weko_records_ui/utils.py:1214 msgid "Unexpected error occurred." msgstr "" -#: tests/test_utils.py:716 weko_records_ui/utils.py:1215 +#: tests/test_utils.py:721 weko_records_ui/utils.py:1216 msgid "Failed to send mail." msgstr "" -#: tests/test_views.py:1303 weko_records_ui/views.py:1247 +#: tests/test_views.py:1342 weko_records_ui/views.py:1261 msgid "MSG_WEKO_RECORDS_UI_IS_EDITING_TRUE" msgstr "Cannot delete because it is being edited." @@ -63,35 +63,51 @@ msgstr "" msgid "Bulk Update" msgstr "" -#: weko_records_ui/api.py:214 weko_records_ui/api.py:221 -#: weko_records_ui/api.py:252 weko_records_ui/api.py:263 -msgid "S3 setting none. Please check your profile." +#: weko_records_ui/api.py:220 +msgid "Not authenticated user." msgstr "" -#: weko_records_ui/api.py:244 -msgid "Getting Bucket List failed." +#: weko_records_ui/api.py:224 weko_records_ui/api.py:227 +#: weko_records_ui/api.py:289 +msgid "S3 setting none. Please check your profile." msgstr "" -#: weko_records_ui/api.py:379 -msgid "Creating Bucket failed." +#: weko_records_ui/api.py:246 +msgid "Getting Bucket List failed." msgstr "" -#: weko_records_ui/api.py:396 +#: weko_records_ui/api.py:325 msgid "Getting region failed." msgstr "" -#: weko_records_ui/api.py:425 weko_records_ui/api.py:491 +#: weko_records_ui/api.py:363 weko_records_ui/api.py:454 msgid "Uploading file failed." msgstr "" "Uploading file failed. Please make sure you have write permissions or " "that the bucket is writable." -#: weko_records_ui/api.py:480 +#: weko_records_ui/api.py:403 weko_records_ui/api.py:660 +msgid "The source bucket or file cannot be found." +msgstr "" + +#: weko_records_ui/api.py:418 msgid "The source file cannot be found." msgstr "" -#: weko_records_ui/api.py:507 weko_records_ui/api.py:667 -#: weko_records_ui/api.py:668 +#: weko_records_ui/api.py:450 +msgid "The source file size exceeds the limit for cross-service copy." +msgstr "" + +#: weko_records_ui/api.py:476 +msgid "Bucket already exists." +msgstr "" + +#: weko_records_ui/api.py:525 +msgid "Creating Bucket failed." +msgstr "" + +#: weko_records_ui/api.py:551 weko_records_ui/api.py:711 +#: weko_records_ui/api.py:712 msgid "Cannot update because the corresponding item is being edited." msgstr "" @@ -195,21 +211,21 @@ msgstr "" msgid "Restricted access is disabled." msgstr "" -#: weko_records_ui/fd.py:501 weko_records_ui/fd.py:621 -#: weko_records_ui/fd.py:714 +#: weko_records_ui/fd.py:500 weko_records_ui/fd.py:620 +#: weko_records_ui/fd.py:713 #, python-format msgid "The file \"%s\" does not exist." msgstr "" -#: weko_records_ui/fd.py:513 weko_records_ui/fd.py:568 +#: weko_records_ui/fd.py:512 weko_records_ui/fd.py:567 msgid "Invalid token." msgstr "" -#: weko_records_ui/fd.py:577 +#: weko_records_ui/fd.py:576 msgid "Could not download file." msgstr "" -#: weko_records_ui/fd.py:586 +#: weko_records_ui/fd.py:585 msgid "Invalid verification info." msgstr "" @@ -238,83 +254,83 @@ msgstr "" msgid "Allow read file." msgstr "" -#: weko_records_ui/utils.py:452 +#: weko_records_ui/utils.py:453 msgid "Item cannot be deleted because the import is in progress." msgstr "" #: weko_records_ui/templates/weko_records_ui/body_contents.html:218 #: weko_records_ui/templates/weko_records_ui/box/preview_carousel.html:151 -#: weko_records_ui/utils.py:988 +#: weko_records_ui/utils.py:989 msgid "Restricted Access" msgstr "" -#: weko_records_ui/utils.py:997 +#: weko_records_ui/utils.py:998 msgid "Download is available from {}/{}/{}." msgstr "" -#: weko_records_ui/utils.py:1000 +#: weko_records_ui/utils.py:1001 msgid "Download / Preview is available from {}/{}/{}." msgstr "" -#: weko_records_ui/utils.py:1260 weko_records_ui/utils.py:1296 +#: weko_records_ui/utils.py:1261 weko_records_ui/utils.py:1297 msgid "Token is invalid." msgstr "" -#: weko_records_ui/utils.py:1314 weko_records_ui/utils.py:2347 +#: weko_records_ui/utils.py:1315 weko_records_ui/utils.py:2353 msgid "The expiration date for download has been exceeded." msgstr "" -#: weko_records_ui/utils.py:1321 weko_records_ui/utils.py:2344 +#: weko_records_ui/utils.py:1322 weko_records_ui/utils.py:2350 msgid "The download limit has been exceeded." msgstr "" -#: weko_records_ui/utils.py:1580 +#: weko_records_ui/utils.py:1581 msgid "Guest" msgstr "" -#: weko_records_ui/utils.py:1591 +#: weko_records_ui/utils.py:1592 msgid "Free Input" msgstr "" -#: weko_records_ui/utils.py:2324 +#: weko_records_ui/utils.py:2330 msgid "The type of URL is not specified." msgstr "" -#: weko_records_ui/utils.py:2328 +#: weko_records_ui/utils.py:2334 msgid "The provided token is invalid." msgstr "" -#: weko_records_ui/utils.py:2332 +#: weko_records_ui/utils.py:2338 msgid "This feature is currently disabled." msgstr "" -#: weko_records_ui/utils.py:2337 +#: weko_records_ui/utils.py:2343 msgid "This file is currently not available for this feature." msgstr "" -#: weko_records_ui/utils.py:2342 +#: weko_records_ui/utils.py:2348 msgid "This URL has been deactivated." msgstr "" -#: weko_records_ui/views.py:904 +#: weko_records_ui/views.py:914 msgid "Secret URL generated successfully" msgstr "" -#: weko_records_ui/views.py:909 +#: weko_records_ui/views.py:923 msgid ", please check your email inbox" msgstr "" -#: weko_records_ui/views.py:911 +#: weko_records_ui/views.py:925 msgid "" ", but there was an error while sending the email. To use the URL, please " "refresh the page and copy it from the issued URL list" msgstr "" -#: weko_records_ui/views.py:914 +#: weko_records_ui/views.py:928 msgid "." msgstr "" -#: weko_records_ui/views.py:1144 +#: weko_records_ui/views.py:1158 msgid "PDF cover page settings have been updated." msgstr "Updated PDF cover settings" @@ -364,34 +380,34 @@ msgid "Next" msgstr "" #: weko_records_ui/templates/weko_records_ui/body_contents.html:25 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:553 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:580 #: weko_records_ui/templates/weko_records_ui/box/request.html:71 msgid "Send" msgstr "" #: weko_records_ui/templates/weko_records_ui/body_contents.html:25 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:561 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:588 msgid "Please input email address." msgstr "" #: weko_records_ui/templates/weko_records_ui/body_contents.html:35 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:455 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:476 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:498 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:482 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:503 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:525 #: weko_records_ui/templates/weko_records_ui/file_details.html:110 msgid "Confirm" msgstr "" #: weko_records_ui/templates/weko_records_ui/body_contents.html:36 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:462 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:484 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:489 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:511 msgid "OK" msgstr "" #: weko_records_ui/templates/weko_records_ui/body_contents.html:37 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:463 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:485 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:558 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:490 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:512 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:585 #: weko_records_ui/templates/weko_records_ui/box/request.html:74 msgid "Cancel" msgstr "" @@ -442,110 +458,127 @@ msgstr "" msgid "Original" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:270 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:276 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:289 +msgid "Import to GakuNin RDM" +msgstr "Import to GakuNin RDM" + +#: weko_records_ui/templates/weko_records_ui/body_contents.html:281 +msgid "Item is not published and file is not publicly accessible" +msgstr "Item is not published and file is not publicly accessible" + +#: weko_records_ui/templates/weko_records_ui/body_contents.html:283 +msgid "Item is not published" +msgstr "Item is not published" + +#: weko_records_ui/templates/weko_records_ui/body_contents.html:285 +msgid "File is not publicly accessible" +msgstr "File is not publicly accessible" + +#: weko_records_ui/templates/weko_records_ui/body_contents.html:297 #: weko_records_ui/templates/weko_records_ui/box/preview.html:61 msgid "Download" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:290 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:317 msgid "Information" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:305 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:314 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:322 #: weko_records_ui/templates/weko_records_ui/body_contents.html:332 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:341 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:349 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:359 #: weko_records_ui/templates/weko_records_ui/box/preview.html:34 #: weko_records_ui/templates/weko_records_ui/box/preview.html:61 #: weko_records_ui/templates/weko_records_ui/box/preview_carousel.html:34 msgid "Preview" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:352 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:379 msgid "Jupyter" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:377 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:404 msgid "Back" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:382 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:409 msgid "Edit" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:384 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:411 #: weko_records_ui/templates/weko_records_ui/file_details_contents.html:270 #: weko_records_ui/templates/weko_records_ui/file_details_contents.html:317 msgid "Delete" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:388 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:415 msgid "Delete this version" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:391 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:401 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:418 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:428 msgid "The workflow is being edited." msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:393 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:420 msgid "The item cannot be deleted because it has a DOI." msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:395 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:422 msgid "You cannot keep an item private because it has a DOI." msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:407 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:434 msgid "Please wait a moment." msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:502 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:529 #: weko_records_ui/templates/weko_records_ui/file_details.html:113 msgid "This file is a Billing file. (Price: XXXXX). Do you want to download it?" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:506 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:533 #: weko_records_ui/templates/weko_records_ui/file_details.html:117 msgid "Yes" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:507 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:534 #: weko_records_ui/templates/weko_records_ui/file_details.html:118 msgid "No" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:523 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:550 msgid "" "To download the files below, you will need to enter your email address " "and password." msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:525 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:552 msgid "To download the files below, you will need to enter your email address." msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:528 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:555 msgid "File Name" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:533 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:536 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:560 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:563 msgid "Email address" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:541 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:544 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:568 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:571 msgid "Password" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:545 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:572 msgid "If you have not set a password, please apply again." msgstr "" -#: weko_records_ui/templates/weko_records_ui/creator_detail_template.html:110 #: weko_records_ui/templates/weko_records_ui/creator_detail_template.html:115 -#: weko_records_ui/templates/weko_records_ui/creator_detail_template.html:116 +#: weko_records_ui/templates/weko_records_ui/creator_detail_template.html:120 +#: weko_records_ui/templates/weko_records_ui/creator_detail_template.html:121 msgid "Search repository" msgstr "" @@ -1147,47 +1180,6 @@ msgstr "" msgid "Show All versions" msgstr "" -msgid "MSG_WEKO_RECORDS_UI_IS_EDITING_TRUE" -msgstr "Cannot delete because it is being edited." +#~ msgid "Email address(reconfirmation)" +#~ msgstr "" -msgid "Email address" -msgstr "" - -msgid "Print Terms and Conditions" -msgstr "" - -msgid "Could not download file." -msgstr "" - -msgid "To download the files below, you will need to enter your email address and password." -msgstr "" - -msgid "To download the files below, you will need to enter your email address." -msgstr "" - -msgid "If you have not set a password, please apply again." -msgstr "" - -msgid "File Name" -msgstr "" - -msgid "Download is available from {}/{}/{}." -msgstr "Download is available from {}/{}/{}." - -msgid "Download / Preview is available from {}/{}/{}." -msgstr "Download / Preview is available from {}/{}/{}." - -msgid "Email address(reconfirmation)" -msgstr "" - -msgid "Import to GakuNin RDM" -msgstr "Import to GakuNin RDM" - -msgid "Item is not published" -msgstr "Item is not published" - -msgid "File is not publicly accessible" -msgstr "File is not publicly accessible" - -msgid "Item is not published and file is not publicly accessible" -msgstr "Item is not published and file is not publicly accessible" diff --git a/modules/weko-records-ui/weko_records_ui/translations/ja/LC_MESSAGES/messages.po b/modules/weko-records-ui/weko_records_ui/translations/ja/LC_MESSAGES/messages.po index 6703a02204..0fc92c5d76 100644 --- a/modules/weko-records-ui/weko_records_ui/translations/ja/LC_MESSAGES/messages.po +++ b/modules/weko-records-ui/weko_records_ui/translations/ja/LC_MESSAGES/messages.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: weko-records-ui 0.1.0.dev20170000\n" "Report-Msgid-Bugs-To: wekosoftware@nii.ac.jp\n" -"POT-Creation-Date: 2025-10-21 14:06+0900\n" +"POT-Creation-Date: 2025-12-24 10:03+0900\n" "PO-Revision-Date: 2021-02-02 03:25+0000\n" "Last-Translator: FULL NAME \n" "Language: ja\n" @@ -19,16 +19,16 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 2.5.1\n" -#: tests/test_utils.py:712 weko_records_ui/api.py:634 weko_records_ui/fd.py:651 -#: weko_records_ui/fd.py:729 weko_records_ui/utils.py:1213 +#: tests/test_utils.py:717 weko_records_ui/api.py:678 weko_records_ui/fd.py:650 +#: weko_records_ui/fd.py:728 weko_records_ui/utils.py:1214 msgid "Unexpected error occurred." msgstr "予期しないエラーが発生しました" -#: tests/test_utils.py:716 weko_records_ui/utils.py:1215 +#: tests/test_utils.py:721 weko_records_ui/utils.py:1216 msgid "Failed to send mail." msgstr "" -#: tests/test_views.py:1303 weko_records_ui/views.py:1247 +#: tests/test_views.py:1342 weko_records_ui/views.py:1261 msgid "MSG_WEKO_RECORDS_UI_IS_EDITING_TRUE" msgstr "該当アイテムは編集中のため、削除できません。" @@ -62,35 +62,52 @@ msgstr "" msgid "Bulk Update" msgstr "" -#: weko_records_ui/api.py:214 weko_records_ui/api.py:221 -#: weko_records_ui/api.py:252 weko_records_ui/api.py:263 +#: weko_records_ui/api.py:220 +msgid "Not authenticated user." +msgstr "" + +#: weko_records_ui/api.py:224 weko_records_ui/api.py:227 +#: weko_records_ui/api.py:289 msgid "S3 setting none. Please check your profile." msgstr "S3に関する設定がありません。あなたのプロフィールを確認してください。" -#: weko_records_ui/api.py:244 +#: weko_records_ui/api.py:246 msgid "Getting Bucket List failed." msgstr "バケットリストの取得に失敗しました。" -#: weko_records_ui/api.py:379 -msgid "Creating Bucket failed." -msgstr "バケットの作成に失敗しました。" - -#: weko_records_ui/api.py:396 +#: weko_records_ui/api.py:325 msgid "Getting region failed." msgstr "リージョンの取得に失敗しました。" -#: weko_records_ui/api.py:425 weko_records_ui/api.py:491 +#: weko_records_ui/api.py:363 weko_records_ui/api.py:454 msgid "Uploading file failed." msgstr "ファイルのアップロードに失敗しました。書き込み権限や書き込み可能なバケットであることを確認してください。" -#: weko_records_ui/api.py:480 +#: weko_records_ui/api.py:403 weko_records_ui/api.py:660 +#, fuzzy +msgid "The source bucket or file cannot be found." +msgstr "コピー元のファイル、バケットが見つかりません。" + +#: weko_records_ui/api.py:418 msgid "The source file cannot be found." msgstr "コピー元のファイルが見つかりません。" -#: weko_records_ui/api.py:507 weko_records_ui/api.py:667 -#: weko_records_ui/api.py:668 +#: weko_records_ui/api.py:450 +msgid "The source file size exceeds the limit for cross-service copy." +msgstr "S3互換サービス間でファイルコピー可能なサイズを超過しています" + +#: weko_records_ui/api.py:476 +msgid "Bucket already exists." +msgstr "指定されたバケットはすでに存在しています。" + +#: weko_records_ui/api.py:525 +msgid "Creating Bucket failed." +msgstr "バケットの作成に失敗しました。" + +#: weko_records_ui/api.py:551 weko_records_ui/api.py:711 +#: weko_records_ui/api.py:712 msgid "Cannot update because the corresponding item is being edited." -msgstr "" +msgstr "該当アイテムが編集中のため更新できません。" #: weko_records_ui/config.py:435 msgid "write your own license" @@ -192,21 +209,21 @@ msgstr "" msgid "Restricted access is disabled." msgstr "制限公開機能が無効です" -#: weko_records_ui/fd.py:501 weko_records_ui/fd.py:621 -#: weko_records_ui/fd.py:714 +#: weko_records_ui/fd.py:500 weko_records_ui/fd.py:620 +#: weko_records_ui/fd.py:713 #, python-format msgid "The file \"%s\" does not exist." msgstr "指定されたファイル「\"%s\"」が存在しません" -#: weko_records_ui/fd.py:513 weko_records_ui/fd.py:568 +#: weko_records_ui/fd.py:512 weko_records_ui/fd.py:567 msgid "Invalid token." msgstr "" -#: weko_records_ui/fd.py:577 +#: weko_records_ui/fd.py:576 msgid "Could not download file." msgstr "ファイルをダウンロードできませんでした" -#: weko_records_ui/fd.py:586 +#: weko_records_ui/fd.py:585 msgid "Invalid verification info." msgstr "" @@ -235,83 +252,83 @@ msgstr "アイテムタイプ" msgid "Allow read file." msgstr "" -#: weko_records_ui/utils.py:452 +#: weko_records_ui/utils.py:453 msgid "Item cannot be deleted because the import is in progress." msgstr "" #: weko_records_ui/templates/weko_records_ui/body_contents.html:218 #: weko_records_ui/templates/weko_records_ui/box/preview_carousel.html:151 -#: weko_records_ui/utils.py:988 +#: weko_records_ui/utils.py:989 msgid "Restricted Access" msgstr "アクセス制限" -#: weko_records_ui/utils.py:997 +#: weko_records_ui/utils.py:998 msgid "Download is available from {}/{}/{}." msgstr "" -#: weko_records_ui/utils.py:1000 +#: weko_records_ui/utils.py:1001 msgid "Download / Preview is available from {}/{}/{}." msgstr "" -#: weko_records_ui/utils.py:1260 weko_records_ui/utils.py:1296 +#: weko_records_ui/utils.py:1261 weko_records_ui/utils.py:1297 msgid "Token is invalid." msgstr "トークンが無効です。" -#: weko_records_ui/utils.py:1314 weko_records_ui/utils.py:2347 +#: weko_records_ui/utils.py:1315 weko_records_ui/utils.py:2353 msgid "The expiration date for download has been exceeded." msgstr "ダウンロード有効期限を超過しています。" -#: weko_records_ui/utils.py:1321 weko_records_ui/utils.py:2344 +#: weko_records_ui/utils.py:1322 weko_records_ui/utils.py:2350 msgid "The download limit has been exceeded." msgstr "ダウンロード制限回数を超過しています。" -#: weko_records_ui/utils.py:1580 +#: weko_records_ui/utils.py:1581 msgid "Guest" msgstr "" -#: weko_records_ui/utils.py:1591 +#: weko_records_ui/utils.py:1592 msgid "Free Input" msgstr "" -#: weko_records_ui/utils.py:2324 +#: weko_records_ui/utils.py:2330 msgid "The type of URL is not specified." msgstr "" -#: weko_records_ui/utils.py:2328 +#: weko_records_ui/utils.py:2334 msgid "The provided token is invalid." msgstr "トークンが無効です。" -#: weko_records_ui/utils.py:2332 +#: weko_records_ui/utils.py:2338 msgid "This feature is currently disabled." msgstr "この機能は現在ご利用頂けません。" -#: weko_records_ui/utils.py:2337 +#: weko_records_ui/utils.py:2343 msgid "This file is currently not available for this feature." msgstr "このファイルは現在ダウンロードできません。" -#: weko_records_ui/utils.py:2342 +#: weko_records_ui/utils.py:2348 msgid "This URL has been deactivated." msgstr "このURLは削除されました。" -#: weko_records_ui/views.py:904 +#: weko_records_ui/views.py:914 msgid "Secret URL generated successfully" msgstr "シークレットURLの作成に成功しました" -#: weko_records_ui/views.py:909 +#: weko_records_ui/views.py:923 msgid ", please check your email inbox" msgstr "。メールをご確認ください" -#: weko_records_ui/views.py:911 +#: weko_records_ui/views.py:925 msgid "" ", but there was an error while sending the email. To use the URL, please " "refresh the page and copy it from the issued URL list" msgstr "が、メール送信エラーが発生しました。ページを更新し、URL一覧表からご利用ください" -#: weko_records_ui/views.py:914 +#: weko_records_ui/views.py:928 msgid "." msgstr "。" -#: weko_records_ui/views.py:1144 +#: weko_records_ui/views.py:1158 msgid "PDF cover page settings have been updated." msgstr "" @@ -361,34 +378,34 @@ msgid "Next" msgstr "次へ" #: weko_records_ui/templates/weko_records_ui/body_contents.html:25 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:553 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:580 #: weko_records_ui/templates/weko_records_ui/box/request.html:71 msgid "Send" msgstr "送信" #: weko_records_ui/templates/weko_records_ui/body_contents.html:25 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:561 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:588 msgid "Please input email address." msgstr "メールアドレスを入力してください。" #: weko_records_ui/templates/weko_records_ui/body_contents.html:35 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:455 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:476 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:498 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:482 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:503 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:525 #: weko_records_ui/templates/weko_records_ui/file_details.html:110 msgid "Confirm" msgstr "確認" #: weko_records_ui/templates/weko_records_ui/body_contents.html:36 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:462 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:484 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:489 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:511 msgid "OK" msgstr "" #: weko_records_ui/templates/weko_records_ui/body_contents.html:37 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:463 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:485 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:558 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:490 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:512 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:585 #: weko_records_ui/templates/weko_records_ui/box/request.html:74 msgid "Cancel" msgstr "キャンセル" @@ -437,110 +454,127 @@ msgstr "" msgid "Original" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:270 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:276 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:289 +msgid "Import to GakuNin RDM" +msgstr "GakuNin RDMにインポート" + +#: weko_records_ui/templates/weko_records_ui/body_contents.html:281 +msgid "Item is not published and file is not publicly accessible" +msgstr "アイテムとファイルが公開されていません" + +#: weko_records_ui/templates/weko_records_ui/body_contents.html:283 +msgid "Item is not published" +msgstr "アイテムが公開されていません" + +#: weko_records_ui/templates/weko_records_ui/body_contents.html:285 +msgid "File is not publicly accessible" +msgstr "ファイルが公開されていません" + +#: weko_records_ui/templates/weko_records_ui/body_contents.html:297 #: weko_records_ui/templates/weko_records_ui/box/preview.html:61 msgid "Download" msgstr "ダウンロード" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:290 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:317 msgid "Information" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:305 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:314 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:322 #: weko_records_ui/templates/weko_records_ui/body_contents.html:332 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:341 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:349 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:359 #: weko_records_ui/templates/weko_records_ui/box/preview.html:34 #: weko_records_ui/templates/weko_records_ui/box/preview.html:61 #: weko_records_ui/templates/weko_records_ui/box/preview_carousel.html:34 msgid "Preview" msgstr "プレビュー" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:352 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:379 msgid "Jupyter" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:377 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:404 msgid "Back" msgstr "戻る" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:382 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:409 msgid "Edit" msgstr "編集" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:384 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:411 #: weko_records_ui/templates/weko_records_ui/file_details_contents.html:270 #: weko_records_ui/templates/weko_records_ui/file_details_contents.html:317 msgid "Delete" msgstr "削除" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:388 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:415 msgid "Delete this version" msgstr "バージョン削除" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:391 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:401 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:418 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:428 msgid "The workflow is being edited." msgstr "このアイテムは編集中です。" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:393 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:420 msgid "The item cannot be deleted because it has a DOI." msgstr "このアイテムはDOIが付与されているため削除できません。" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:395 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:422 msgid "You cannot keep an item private because it has a DOI." msgstr "このアイテムはDOIが付与されているため非公開にできません。" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:407 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:434 msgid "Please wait a moment." msgstr "少々お待ちください。" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:502 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:529 #: weko_records_ui/templates/weko_records_ui/file_details.html:113 msgid "This file is a Billing file. (Price: XXXXX). Do you want to download it?" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:506 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:533 #: weko_records_ui/templates/weko_records_ui/file_details.html:117 msgid "Yes" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:507 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:534 #: weko_records_ui/templates/weko_records_ui/file_details.html:118 msgid "No" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:523 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:550 msgid "" "To download the files below, you will need to enter your email address " "and password." msgstr "以下のファイルのダウンロードには、メールアドレスとパスワードの入力が必要です。" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:525 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:552 msgid "To download the files below, you will need to enter your email address." msgstr "以下のファイルのダウンロードには、メールアドレスの入力が必要です。" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:528 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:555 msgid "File Name" msgstr "ファイル名" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:533 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:536 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:560 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:563 msgid "Email address" msgstr "メールアドレス" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:541 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:544 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:568 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:571 msgid "Password" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:545 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:572 msgid "If you have not set a password, please apply again." msgstr "パスワード未設定の場合は、再度申請を行ってください。" -#: weko_records_ui/templates/weko_records_ui/creator_detail_template.html:110 #: weko_records_ui/templates/weko_records_ui/creator_detail_template.html:115 -#: weko_records_ui/templates/weko_records_ui/creator_detail_template.html:116 +#: weko_records_ui/templates/weko_records_ui/creator_detail_template.html:120 +#: weko_records_ui/templates/weko_records_ui/creator_detail_template.html:121 msgid "Search repository" msgstr "" @@ -1139,44 +1173,3 @@ msgstr "" msgid "Show All versions" msgstr "" -msgid "This data is not available for undergraduate students or those who do not register their positions." -msgstr "このデータは利用できません(学部生または役職が登録されていないため)" - -msgid "Email address" -msgstr "メールアドレス" - -msgid "Email address(reconfirmation)" -msgstr "メールアドレス(確認用)" - -msgid "MSG_WEKO_RECORDS_UI_IS_EDITING_TRUE" -msgstr "該当アイテムは編集中のため、削除できません。" - -msgid "Print Terms and Conditions" -msgstr "利用規約を印刷" - -msgid "Could not download file." -msgstr "ファイルをダウンロードできませんでした" - -msgid "To download the files below, you will need to enter your email address and password." -msgstr "以下のファイルのダウンロードには、メールアドレスとパスワードの入力が必要です。" - -msgid "To download the files below, you will need to enter your email address." -msgstr "以下のファイルのダウンロードには、メールアドレスの入力が必要です。" - -msgid "If you have not set a password, please apply again." -msgstr "パスワード未設定の場合は、再度申請を行ってください。" - -msgid "File Name" -msgstr "ファイル名" - -msgid "Import to GakuNin RDM" -msgstr "GakuNin RDMにインポート" - -msgid "Item is not published" -msgstr "アイテムが公開されていません" - -msgid "File is not publicly accessible" -msgstr "ファイルが公開されていません" - -msgid "Item is not published and file is not publicly accessible" -msgstr "アイテムとファイルが公開されていません" diff --git a/modules/weko-records-ui/weko_records_ui/translations/messages.pot b/modules/weko-records-ui/weko_records_ui/translations/messages.pot index 2040f487d3..a70b0ed986 100644 --- a/modules/weko-records-ui/weko_records_ui/translations/messages.pot +++ b/modules/weko-records-ui/weko_records_ui/translations/messages.pot @@ -9,7 +9,7 @@ msgid "" msgstr "" "Project-Id-Version: weko-records-ui 0.1.0.dev20170000\n" "Report-Msgid-Bugs-To: wekosoftware@nii.ac.jp\n" -"POT-Creation-Date: 2025-10-21 14:06+0900\n" +"POT-Creation-Date: 2025-12-24 10:03+0900\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -18,16 +18,16 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 2.5.1\n" -#: tests/test_utils.py:712 weko_records_ui/api.py:634 weko_records_ui/fd.py:651 -#: weko_records_ui/fd.py:729 weko_records_ui/utils.py:1213 +#: tests/test_utils.py:717 weko_records_ui/api.py:678 weko_records_ui/fd.py:650 +#: weko_records_ui/fd.py:728 weko_records_ui/utils.py:1214 msgid "Unexpected error occurred." msgstr "" -#: tests/test_utils.py:716 weko_records_ui/utils.py:1215 +#: tests/test_utils.py:721 weko_records_ui/utils.py:1216 msgid "Failed to send mail." msgstr "" -#: tests/test_views.py:1303 weko_records_ui/views.py:1247 +#: tests/test_views.py:1342 weko_records_ui/views.py:1261 msgid "MSG_WEKO_RECORDS_UI_IS_EDITING_TRUE" msgstr "" @@ -61,33 +61,49 @@ msgstr "" msgid "Bulk Update" msgstr "" -#: weko_records_ui/api.py:214 weko_records_ui/api.py:221 -#: weko_records_ui/api.py:252 weko_records_ui/api.py:263 -msgid "S3 setting none. Please check your profile." +#: weko_records_ui/api.py:220 +msgid "Not authenticated user." msgstr "" -#: weko_records_ui/api.py:244 -msgid "Getting Bucket List failed." +#: weko_records_ui/api.py:224 weko_records_ui/api.py:227 +#: weko_records_ui/api.py:289 +msgid "S3 setting none. Please check your profile." msgstr "" -#: weko_records_ui/api.py:379 -msgid "Creating Bucket failed." +#: weko_records_ui/api.py:246 +msgid "Getting Bucket List failed." msgstr "" -#: weko_records_ui/api.py:396 +#: weko_records_ui/api.py:325 msgid "Getting region failed." msgstr "" -#: weko_records_ui/api.py:425 weko_records_ui/api.py:491 +#: weko_records_ui/api.py:363 weko_records_ui/api.py:454 msgid "Uploading file failed." msgstr "" -#: weko_records_ui/api.py:480 +#: weko_records_ui/api.py:403 weko_records_ui/api.py:660 +msgid "The source bucket or file cannot be found." +msgstr "" + +#: weko_records_ui/api.py:418 msgid "The source file cannot be found." msgstr "" -#: weko_records_ui/api.py:507 weko_records_ui/api.py:667 -#: weko_records_ui/api.py:668 +#: weko_records_ui/api.py:450 +msgid "The source file size exceeds the limit for cross-service copy." +msgstr "" + +#: weko_records_ui/api.py:476 +msgid "Bucket already exists." +msgstr "" + +#: weko_records_ui/api.py:525 +msgid "Creating Bucket failed." +msgstr "" + +#: weko_records_ui/api.py:551 weko_records_ui/api.py:711 +#: weko_records_ui/api.py:712 msgid "Cannot update because the corresponding item is being edited." msgstr "" @@ -191,21 +207,21 @@ msgstr "" msgid "Restricted access is disabled." msgstr "" -#: weko_records_ui/fd.py:501 weko_records_ui/fd.py:621 -#: weko_records_ui/fd.py:714 +#: weko_records_ui/fd.py:500 weko_records_ui/fd.py:620 +#: weko_records_ui/fd.py:713 #, python-format msgid "The file \"%s\" does not exist." msgstr "" -#: weko_records_ui/fd.py:513 weko_records_ui/fd.py:568 +#: weko_records_ui/fd.py:512 weko_records_ui/fd.py:567 msgid "Invalid token." msgstr "" -#: weko_records_ui/fd.py:577 +#: weko_records_ui/fd.py:576 msgid "Could not download file." msgstr "" -#: weko_records_ui/fd.py:586 +#: weko_records_ui/fd.py:585 msgid "Invalid verification info." msgstr "" @@ -234,83 +250,83 @@ msgstr "" msgid "Allow read file." msgstr "" -#: weko_records_ui/utils.py:452 +#: weko_records_ui/utils.py:453 msgid "Item cannot be deleted because the import is in progress." msgstr "" #: weko_records_ui/templates/weko_records_ui/body_contents.html:218 #: weko_records_ui/templates/weko_records_ui/box/preview_carousel.html:151 -#: weko_records_ui/utils.py:988 +#: weko_records_ui/utils.py:989 msgid "Restricted Access" msgstr "" -#: weko_records_ui/utils.py:997 +#: weko_records_ui/utils.py:998 msgid "Download is available from {}/{}/{}." msgstr "" -#: weko_records_ui/utils.py:1000 +#: weko_records_ui/utils.py:1001 msgid "Download / Preview is available from {}/{}/{}." msgstr "" -#: weko_records_ui/utils.py:1260 weko_records_ui/utils.py:1296 +#: weko_records_ui/utils.py:1261 weko_records_ui/utils.py:1297 msgid "Token is invalid." msgstr "" -#: weko_records_ui/utils.py:1314 weko_records_ui/utils.py:2347 +#: weko_records_ui/utils.py:1315 weko_records_ui/utils.py:2353 msgid "The expiration date for download has been exceeded." msgstr "" -#: weko_records_ui/utils.py:1321 weko_records_ui/utils.py:2344 +#: weko_records_ui/utils.py:1322 weko_records_ui/utils.py:2350 msgid "The download limit has been exceeded." msgstr "" -#: weko_records_ui/utils.py:1580 +#: weko_records_ui/utils.py:1581 msgid "Guest" msgstr "" -#: weko_records_ui/utils.py:1591 +#: weko_records_ui/utils.py:1592 msgid "Free Input" msgstr "" -#: weko_records_ui/utils.py:2324 +#: weko_records_ui/utils.py:2330 msgid "The type of URL is not specified." msgstr "" -#: weko_records_ui/utils.py:2328 +#: weko_records_ui/utils.py:2334 msgid "The provided token is invalid." msgstr "" -#: weko_records_ui/utils.py:2332 +#: weko_records_ui/utils.py:2338 msgid "This feature is currently disabled." msgstr "" -#: weko_records_ui/utils.py:2337 +#: weko_records_ui/utils.py:2343 msgid "This file is currently not available for this feature." msgstr "" -#: weko_records_ui/utils.py:2342 +#: weko_records_ui/utils.py:2348 msgid "This URL has been deactivated." msgstr "" -#: weko_records_ui/views.py:904 +#: weko_records_ui/views.py:914 msgid "Secret URL generated successfully" msgstr "" -#: weko_records_ui/views.py:909 +#: weko_records_ui/views.py:923 msgid ", please check your email inbox" msgstr "" -#: weko_records_ui/views.py:911 +#: weko_records_ui/views.py:925 msgid "" ", but there was an error while sending the email. To use the URL, please " "refresh the page and copy it from the issued URL list" msgstr "" -#: weko_records_ui/views.py:914 +#: weko_records_ui/views.py:928 msgid "." msgstr "" -#: weko_records_ui/views.py:1144 +#: weko_records_ui/views.py:1158 msgid "PDF cover page settings have been updated." msgstr "" @@ -360,34 +376,34 @@ msgid "Next" msgstr "" #: weko_records_ui/templates/weko_records_ui/body_contents.html:25 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:553 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:580 #: weko_records_ui/templates/weko_records_ui/box/request.html:71 msgid "Send" msgstr "" #: weko_records_ui/templates/weko_records_ui/body_contents.html:25 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:561 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:588 msgid "Please input email address." msgstr "" #: weko_records_ui/templates/weko_records_ui/body_contents.html:35 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:455 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:476 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:498 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:482 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:503 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:525 #: weko_records_ui/templates/weko_records_ui/file_details.html:110 msgid "Confirm" msgstr "" #: weko_records_ui/templates/weko_records_ui/body_contents.html:36 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:462 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:484 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:489 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:511 msgid "OK" msgstr "" #: weko_records_ui/templates/weko_records_ui/body_contents.html:37 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:463 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:485 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:558 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:490 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:512 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:585 #: weko_records_ui/templates/weko_records_ui/box/request.html:74 msgid "Cancel" msgstr "" @@ -436,110 +452,127 @@ msgstr "" msgid "Original" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:270 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:276 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:289 +msgid "Import to GakuNin RDM" +msgstr "" + +#: weko_records_ui/templates/weko_records_ui/body_contents.html:281 +msgid "Item is not published and file is not publicly accessible" +msgstr "" + +#: weko_records_ui/templates/weko_records_ui/body_contents.html:283 +msgid "Item is not published" +msgstr "" + +#: weko_records_ui/templates/weko_records_ui/body_contents.html:285 +msgid "File is not publicly accessible" +msgstr "" + +#: weko_records_ui/templates/weko_records_ui/body_contents.html:297 #: weko_records_ui/templates/weko_records_ui/box/preview.html:61 msgid "Download" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:290 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:317 msgid "Information" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:305 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:314 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:322 #: weko_records_ui/templates/weko_records_ui/body_contents.html:332 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:341 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:349 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:359 #: weko_records_ui/templates/weko_records_ui/box/preview.html:34 #: weko_records_ui/templates/weko_records_ui/box/preview.html:61 #: weko_records_ui/templates/weko_records_ui/box/preview_carousel.html:34 msgid "Preview" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:352 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:379 msgid "Jupyter" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:377 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:404 msgid "Back" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:382 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:409 msgid "Edit" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:384 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:411 #: weko_records_ui/templates/weko_records_ui/file_details_contents.html:270 #: weko_records_ui/templates/weko_records_ui/file_details_contents.html:317 msgid "Delete" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:388 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:415 msgid "Delete this version" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:391 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:401 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:418 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:428 msgid "The workflow is being edited." msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:393 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:420 msgid "The item cannot be deleted because it has a DOI." msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:395 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:422 msgid "You cannot keep an item private because it has a DOI." msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:407 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:434 msgid "Please wait a moment." msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:502 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:529 #: weko_records_ui/templates/weko_records_ui/file_details.html:113 msgid "This file is a Billing file. (Price: XXXXX). Do you want to download it?" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:506 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:533 #: weko_records_ui/templates/weko_records_ui/file_details.html:117 msgid "Yes" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:507 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:534 #: weko_records_ui/templates/weko_records_ui/file_details.html:118 msgid "No" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:523 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:550 msgid "" "To download the files below, you will need to enter your email address " "and password." msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:525 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:552 msgid "To download the files below, you will need to enter your email address." msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:528 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:555 msgid "File Name" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:533 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:536 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:560 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:563 msgid "Email address" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:541 -#: weko_records_ui/templates/weko_records_ui/body_contents.html:544 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:568 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:571 msgid "Password" msgstr "" -#: weko_records_ui/templates/weko_records_ui/body_contents.html:545 +#: weko_records_ui/templates/weko_records_ui/body_contents.html:572 msgid "If you have not set a password, please apply again." msgstr "" -#: weko_records_ui/templates/weko_records_ui/creator_detail_template.html:110 #: weko_records_ui/templates/weko_records_ui/creator_detail_template.html:115 -#: weko_records_ui/templates/weko_records_ui/creator_detail_template.html:116 +#: weko_records_ui/templates/weko_records_ui/creator_detail_template.html:120 +#: weko_records_ui/templates/weko_records_ui/creator_detail_template.html:121 msgid "Search repository" msgstr "" @@ -1138,15 +1171,3 @@ msgstr "" msgid "Show All versions" msgstr "" - -msgid "Import to GakuNin RDM" -msgstr "" - -msgid "Item is not published" -msgstr "" - -msgid "File is not publicly accessible" -msgstr "" - -msgid "Item is not published and file is not publicly accessible" -msgstr "" From 75372308d19f32430f0d588bd55009fffbf81828 Mon Sep 17 00:00:00 2001 From: ivis-kondo Date: Fri, 26 Dec 2025 10:10:52 +0900 Subject: [PATCH 4/5] add TransferConfig settings --- modules/weko-records-ui/weko_records_ui/api.py | 14 +++++++++++--- modules/weko-records-ui/weko_records_ui/config.py | 9 +++++++++ scripts/instance.cfg | 9 +++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/modules/weko-records-ui/weko_records_ui/api.py b/modules/weko-records-ui/weko_records_ui/api.py index a30f8f8aa3..ed26e0b0aa 100644 --- a/modules/weko-records-ui/weko_records_ui/api.py +++ b/modules/weko-records-ui/weko_records_ui/api.py @@ -12,6 +12,7 @@ from urllib.parse import urlparse, urljoin, uses_relative, uses_netloc from botocore.exceptions import BotoCoreError, ClientError +from boto3.s3.transfer import TransferConfig from email_validator import validate_email from flask import current_app, request @@ -347,6 +348,10 @@ def _is_same_storage_service(source_s3_client, destination_s3_client): current_app.logger.debug(f"Profile endpoint URL: {profile.s3_endpoint_url}") current_app.logger.debug(f"Profile region name: {profile.s3_region_name}") + # Get TransferConfig from config + transfer_setting = current_app.config.get("WEKO_RECORDS_UI_S3_TRANSFER_CONFIG", {}) + s3_transfer_config = TransferConfig(**transfer_setting) + if source_location.type is None: # local to S3 @@ -355,7 +360,8 @@ def _is_same_storage_service(source_s3_client, destination_s3_client): destination_s3_client.upload_file( source_file_instance.uri, bucket_name, - filename + filename, + Config=s3_transfer_config ) return urljoin(uri, filename) except Exception as e: @@ -426,7 +432,8 @@ def _is_same_storage_service(source_s3_client, destination_s3_client): copy_source, destination_bucket, destination_key, - SourceClient=s3_client_source + SourceClient=s3_client_source, + Config=s3_transfer_config ) else: # Different storage service @@ -444,7 +451,8 @@ def _is_same_storage_service(source_s3_client, destination_s3_client): ) body = obj["Body"] destination_s3_client.upload_fileobj( - body, destination_bucket, destination_key + body, destination_bucket, destination_key, + Config=s3_transfer_config ) else: raise Exception(_("The source file size exceeds the limit for cross-service copy.")) diff --git a/modules/weko-records-ui/weko_records_ui/config.py b/modules/weko-records-ui/weko_records_ui/config.py index 2226090047..56e59a6ee8 100644 --- a/modules/weko-records-ui/weko_records_ui/config.py +++ b/modules/weko-records-ui/weko_records_ui/config.py @@ -799,6 +799,15 @@ WEKO_RECORDS_UI_S3_CROSS_COPY_MAX_FILE_SIZE = 20 * 1024 * 1024 * 1024 # 20GB """Max file size for s3 compatible service cross copy.""" +WEKO_RECORDS_UI_S3_TRANSFER_CONFIG = { + "multipart_threshold": 8 * 1024 * 1024, # threshold to use multipart upload (default: 8 MiB) + "multipart_chunksize": 8 * 1024 * 1024, # size of each part for multipart upload (default: 8 MiB) + "use_threads": True, # enable multithreading (default: True) + "max_concurrency": 10, # number of threads for multipart upload/download (default: 10) + # "max_bandwidth": None, # throttle bandwidth in bytes/second (default: None) Available Boto3 Version 1.20+ +} +"""Configuration for S3 compatible service transfer.""" + class EXTERNAL_SYSTEM(Enum): OA = "OA" diff --git a/scripts/instance.cfg b/scripts/instance.cfg index 576c5d2d01..835693078c 100644 --- a/scripts/instance.cfg +++ b/scripts/instance.cfg @@ -900,3 +900,12 @@ WEKO_ACCOUNTS_IDP_ENTITY_ID = '' WEKO_ACCOUNTS_SHIB_BIND_GAKUNIN_MAP_GROUPS = False """Bind Gakunin mAP groups to WEKO groups.""" + +WEKO_RECORDS_UI_S3_TRANSFER_CONFIG = { + "multipart_threshold": 8 * 1024 * 1024, # threshold to use multipart upload (default: 8 MiB) + "multipart_chunksize": 8 * 1024 * 1024, # size of each part for multipart upload (default: 8 MiB) + "use_threads": True, # enable multithreading (default: True) + "max_concurrency": 10, # number of threads for multipart upload/download (default: 10) + # "max_bandwidth": None, # throttle bandwidth in bytes/second (default: None) Available Boto3 Version 1.20.0+ +} +"""Configuration for S3 compatible service transfer.""" From 1bc5f0e0543901206fa5f592cb3bf5a294028b3a Mon Sep 17 00:00:00 2001 From: ivis-kondo Date: Fri, 26 Dec 2025 10:43:54 +0900 Subject: [PATCH 5/5] S3 transfer configuration refactor: separate settings into individual constants --- .../weko-records-ui/weko_records_ui/api.py | 7 ++++++- .../weko-records-ui/weko_records_ui/config.py | 21 +++++++++++-------- scripts/instance.cfg | 19 ++++++++++------- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/modules/weko-records-ui/weko_records_ui/api.py b/modules/weko-records-ui/weko_records_ui/api.py index ed26e0b0aa..03bbf3f68c 100644 --- a/modules/weko-records-ui/weko_records_ui/api.py +++ b/modules/weko-records-ui/weko_records_ui/api.py @@ -349,7 +349,12 @@ def _is_same_storage_service(source_s3_client, destination_s3_client): current_app.logger.debug(f"Profile region name: {profile.s3_region_name}") # Get TransferConfig from config - transfer_setting = current_app.config.get("WEKO_RECORDS_UI_S3_TRANSFER_CONFIG", {}) + transfer_setting = { + "multipart_threshold": current_app.config.get("WEKO_RECORDS_UI_S3_TRANSFER_MULTIPART_THRESHOLD", 8 * 1024 * 1024), + "multipart_chunksize": current_app.config.get("WEKO_RECORDS_UI_S3_TRANSFER_MULTIPART_CHUNKSIZE", 8 * 1024 * 1024), + "use_threads": current_app.config.get("WEKO_RECORDS_UI_S3_TRANSFER_USE_THREADS", True), + "max_concurrency": current_app.config.get("WEKO_RECORDS_UI_S3_TRANSFER_MAX_CONCURRENCY", 10), + } s3_transfer_config = TransferConfig(**transfer_setting) if source_location.type is None: diff --git a/modules/weko-records-ui/weko_records_ui/config.py b/modules/weko-records-ui/weko_records_ui/config.py index 56e59a6ee8..ac792124e2 100644 --- a/modules/weko-records-ui/weko_records_ui/config.py +++ b/modules/weko-records-ui/weko_records_ui/config.py @@ -796,17 +796,20 @@ WEKO_RECORDS_UI_OA_API_CODE = "oaa" -WEKO_RECORDS_UI_S3_CROSS_COPY_MAX_FILE_SIZE = 20 * 1024 * 1024 * 1024 # 20GB +WEKO_RECORDS_UI_S3_CROSS_COPY_MAX_FILE_SIZE = 20 * 1024 * 1024 * 1024 # 20GiB """Max file size for s3 compatible service cross copy.""" -WEKO_RECORDS_UI_S3_TRANSFER_CONFIG = { - "multipart_threshold": 8 * 1024 * 1024, # threshold to use multipart upload (default: 8 MiB) - "multipart_chunksize": 8 * 1024 * 1024, # size of each part for multipart upload (default: 8 MiB) - "use_threads": True, # enable multithreading (default: True) - "max_concurrency": 10, # number of threads for multipart upload/download (default: 10) - # "max_bandwidth": None, # throttle bandwidth in bytes/second (default: None) Available Boto3 Version 1.20+ -} -"""Configuration for S3 compatible service transfer.""" +WEKO_RECORDS_UI_S3_TRANSFER_MULTIPART_THRESHOLD = 8 * 1024 * 1024 +"""Threshold to use multipart upload for S3 compatible service transfer (byte). Default is 8 MiB.""" + +WEKO_RECORDS_UI_S3_TRANSFER_MULTIPART_CHUNKSIZE = 8 * 1024 * 1024 +"""Size of each part for multipart upload for S3 compatible service transfer (byte). Default is 8 MiB.""" + +WEKO_RECORDS_UI_S3_TRANSFER_USE_THREADS = True +"""Enable multithreading for S3 compatible service transfer. True if using multithreading.""" + +WEKO_RECORDS_UI_S3_TRANSFER_MAX_CONCURRENCY = 10 +"""Number of threads for multipart upload/download for S3 compatible service transfer. Default is 10.""" class EXTERNAL_SYSTEM(Enum): OA = "OA" diff --git a/scripts/instance.cfg b/scripts/instance.cfg index 835693078c..e895234ad8 100644 --- a/scripts/instance.cfg +++ b/scripts/instance.cfg @@ -901,11 +901,14 @@ WEKO_ACCOUNTS_IDP_ENTITY_ID = '' WEKO_ACCOUNTS_SHIB_BIND_GAKUNIN_MAP_GROUPS = False """Bind Gakunin mAP groups to WEKO groups.""" -WEKO_RECORDS_UI_S3_TRANSFER_CONFIG = { - "multipart_threshold": 8 * 1024 * 1024, # threshold to use multipart upload (default: 8 MiB) - "multipart_chunksize": 8 * 1024 * 1024, # size of each part for multipart upload (default: 8 MiB) - "use_threads": True, # enable multithreading (default: True) - "max_concurrency": 10, # number of threads for multipart upload/download (default: 10) - # "max_bandwidth": None, # throttle bandwidth in bytes/second (default: None) Available Boto3 Version 1.20.0+ -} -"""Configuration for S3 compatible service transfer.""" +WEKO_RECORDS_UI_S3_TRANSFER_MULTIPART_THRESHOLD = 8 * 1024 * 1024 +"""Threshold to use multipart upload for S3 compatible service transfer (byte). Default is 8 MiB.""" + +WEKO_RECORDS_UI_S3_TRANSFER_MULTIPART_CHUNKSIZE = 8 * 1024 * 1024 +"""Size of each part for multipart upload for S3 compatible service transfer (byte). Default is 8 MiB.""" + +WEKO_RECORDS_UI_S3_TRANSFER_USE_THREADS = True +"""Enable multithreading for S3 compatible service transfer. True if using multithreading.""" + +WEKO_RECORDS_UI_S3_TRANSFER_MAX_CONCURRENCY = 10 +"""Number of threads for multipart upload/download for S3 compatible service transfer. Default is 10."""