From 90fecf8f7ccaa092484397e07677bf914ef16384 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Fri, 6 Feb 2026 18:31:32 -0500 Subject: [PATCH] [PULP-817] Django5 support and adjustments to make imp/exp work with django-imp-exp 4.x. (cherry picked from commit 793c0ad638254ef76fb379e44d49a5dea0613751) closes #5324. closes #6988. Co-authored-by: Grant Gainey Assisted by: cursor agent Co-authored-by: Cursor --- .ci/assets/ci_constraints.txt | 1 + .github/workflows/scripts/install.sh | 6 +-- CHANGES/5324.feature | 1 + CHANGES/6988.feature | 1 + lint_requirements.txt | 3 ++ pulpcore/app/apps.py | 15 ++++-- pulpcore/app/checks.py | 2 +- pulpcore/app/importexport.py | 6 ++- pulpcore/app/modelresource.py | 5 +- pulpcore/app/models/domain.py | 5 +- pulpcore/app/serializers/domain.py | 7 ++- pulpcore/app/settings.py | 38 +++++++++++--- pulpcore/app/tasks/export.py | 2 +- pulpcore/app/tasks/importer.py | 6 +++ pulpcore/app/util.py | 6 ++- pulpcore/app/views/status.py | 2 +- pulpcore/plugin/importexport.py | 3 ++ pulpcore/tests/functional/__init__.py | 41 +++++++++------ .../api/test_artifact_distribution.py | 7 +-- .../functional/api/test_crd_artifacts.py | 7 ++- .../tests/functional/api/test_crud_domains.py | 22 ++++---- pulpcore/tests/functional/api/test_status.py | 5 +- pulpcore/tests/unit/models/test_content.py | 2 +- .../tests/unit/viewsets/test_viewset_base.py | 6 +-- requirements.txt | 10 ++-- template_config.yml | 52 ++++++++++--------- 26 files changed, 164 insertions(+), 97 deletions(-) create mode 100644 CHANGES/5324.feature create mode 100644 CHANGES/6988.feature diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt index c8070aff546..5e526bb5b79 100644 --- a/.ci/assets/ci_constraints.txt +++ b/.ci/assets/ci_constraints.txt @@ -16,3 +16,4 @@ azure-storage-blob!=12.28.* pycares<5 # older aiodns versions don't pin pycares UB, and are broken by pycares>=5 + diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 3aeed5f2c40..50950fc89f3 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -79,7 +79,7 @@ cat >> vars/main.yaml << VARSYAML pulp_env: {"PULP_CA_BUNDLE": "/etc/pulp/certs/pulp_webserver.crt"} pulp_settings: {"allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "orphan_protection_time": 0} pulp_scheme: https -pulp_default_container: ghcr.io/pulp/pulp-ci-centos:latest +pulp_default_container: ghcr.io/pulp/pulp-ci-centos9:latest VARSYAML SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "generate-bindings" "lowerbounds") @@ -105,7 +105,7 @@ if [ "$TEST" = "s3" ]; then sed -i -e '$a s3_test: true\ minio_access_key: "'$MINIO_ACCESS_KEY'"\ minio_secret_key: "'$MINIO_SECRET_KEY'"\ -pulp_scenario_settings: {"AWS_ACCESS_KEY_ID": "AKIAIT2Z5TDYPX3ARJBA", "AWS_DEFAULT_ACL": "@none None", "AWS_S3_ADDRESSING_STYLE": "path", "AWS_S3_ENDPOINT_URL": "http://minio:9000", "AWS_S3_REGION_NAME": "eu-central-1", "AWS_S3_SIGNATURE_VERSION": "s3v4", "AWS_SECRET_ACCESS_KEY": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "AWS_STORAGE_BUCKET_NAME": "pulp3", "DEFAULT_FILE_STORAGE": "storages.backends.s3boto3.S3Boto3Storage", "MEDIA_ROOT": "", "domain_enabled": true, "hide_guarded_distributions": true}\ +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true, "hide_guarded_distributions": true}\ pulp_scenario_env: {}\ ' vars/main.yaml export PULP_API_ROOT="/rerouted/djnd/" @@ -119,7 +119,7 @@ if [ "$TEST" = "azure" ]; then - ./azurite:/etc/pulp\ command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"' vars/main.yaml sed -i -e '$a azure_test: true\ -pulp_scenario_settings: {"AZURE_ACCOUNT_KEY": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "AZURE_ACCOUNT_NAME": "devstoreaccount1", "AZURE_CONNECTION_STRING": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "AZURE_CONTAINER": "pulp-test", "AZURE_LOCATION": "pulp3", "AZURE_OVERWRITE_FILES": true, "AZURE_URL_EXPIRATION_SECS": 120, "DEFAULT_FILE_STORAGE": "storages.backends.azure_storage.AzureStorage", "MEDIA_ROOT": "", "domain_enabled": true}\ +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true}\ pulp_scenario_env: {}\ ' vars/main.yaml fi diff --git a/CHANGES/5324.feature b/CHANGES/5324.feature new file mode 100644 index 00000000000..d2d6cc140ce --- /dev/null +++ b/CHANGES/5324.feature @@ -0,0 +1 @@ +Adapted PulpImport/Export to allow update django-import-export==4.x. \ No newline at end of file diff --git a/CHANGES/6988.feature b/CHANGES/6988.feature new file mode 100644 index 00000000000..db855921cf8 --- /dev/null +++ b/CHANGES/6988.feature @@ -0,0 +1 @@ +Allow use of Django5 as well as Django4. \ No newline at end of file diff --git a/lint_requirements.txt b/lint_requirements.txt index 859b6ac9ad8..6642a1a5340 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -6,6 +6,9 @@ # For more info visit https://github.com/pulp/plugin_template black==24.3.0 +# Click is pinned because of: +# https://github.com/pallets/click/issues/3065 +click<8.3 bump-my-version check-manifest flake8 diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index 3862deaab58..485c01e93c6 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -5,7 +5,6 @@ from importlib import import_module from django import apps -from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.db import connection, transaction from django.db.models.signals import post_migrate @@ -66,6 +65,12 @@ class PulpPluginAppConfig(apps.AppConfig): def __init__(self, app_name, app_module): super().__init__(app_name, app_module) + # begin compatibility layer for DEFAULT_FILE_STORAGE deprecation + # Workaround for getting the up-to-date settings instance + from django.conf import settings + + self.settings = settings + # end try: self.version @@ -307,6 +312,7 @@ def _populate_system_id(sender, apps, verbosity, **kwargs): def _ensure_default_domain(sender, **kwargs): + settings = sender.settings table_names = connection.introspection.table_names() if "core_domain" in table_names: from pulpcore.app.util import get_default_domain @@ -316,11 +322,11 @@ def _ensure_default_domain(sender, **kwargs): if ( settings.HIDE_GUARDED_DISTRIBUTIONS != default.hide_guarded_distributions or settings.REDIRECT_TO_OBJECT_STORAGE != default.redirect_to_object_storage - or settings.DEFAULT_FILE_STORAGE != default.storage_class + or settings.STORAGES["default"]["BACKEND"] != default.storage_class ): default.hide_guarded_distributions = settings.HIDE_GUARDED_DISTRIBUTIONS default.redirect_to_object_storage = settings.REDIRECT_TO_OBJECT_STORAGE - default.storage_class = settings.DEFAULT_FILE_STORAGE + default.storage_class = settings.STORAGES["default"]["BACKEND"] default.save(skip_hooks=True) @@ -386,8 +392,9 @@ def _get_permission(perm): def _populate_artifact_serving_distribution(sender, apps, verbosity, **kwargs): + settings = sender.settings if ( - settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem" + settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem" or not settings.REDIRECT_TO_OBJECT_STORAGE ): try: diff --git a/pulpcore/app/checks.py b/pulpcore/app/checks.py index 0cd4a0478b0..ec7c405ef80 100644 --- a/pulpcore/app/checks.py +++ b/pulpcore/app/checks.py @@ -23,7 +23,7 @@ def content_origin_check(app_configs, **kwargs): def storage_paths(app_configs, **kwargs): warnings = [] - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem": try: media_root_dev = Path(settings.MEDIA_ROOT).stat().st_dev except OSError: diff --git a/pulpcore/app/importexport.py b/pulpcore/app/importexport.py index 4e3480b3004..f3f475eaa60 100644 --- a/pulpcore/app/importexport.py +++ b/pulpcore/app/importexport.py @@ -47,6 +47,10 @@ def _write_export(the_tarfile, resource, dest_dir=None): # the data in batches to memory and concatenate the json lists via string manipulation. with tempfile.NamedTemporaryFile(dir=".", mode="w", encoding="utf8") as temp_file: if isinstance(resource.queryset, QuerySet): + # If we don't have any of "these" - skip writing + if resource.queryset.count() == 0: + return + temp_file.write("[") def process_batch(batch): @@ -117,7 +121,7 @@ def export_artifacts(export, artifacts): with ProgressReport(**data) as pb: pb.BATCH_INTERVAL = 5000 - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem": with tempfile.TemporaryDirectory(dir=".") as temp_dir: for artifact in pb.iter(artifacts.only("file").iterator()): with tempfile.NamedTemporaryFile(dir=temp_dir) as temp_file: diff --git a/pulpcore/app/modelresource.py b/pulpcore/app/modelresource.py index 54d3522fa98..632e982f06a 100644 --- a/pulpcore/app/modelresource.py +++ b/pulpcore/app/modelresource.py @@ -1,3 +1,4 @@ +from django.conf import settings from import_export import fields from import_export.widgets import ForeignKeyWidget from logging import getLogger @@ -36,8 +37,8 @@ def before_import_row(self, row, **kwargs): # the export converts None to blank strings but sha384 and sha512 have unique constraints # that get triggered if they are blank. convert checksums back into None if they are blank. for checksum in ALL_KNOWN_CONTENT_CHECKSUMS: - if row[checksum] == "": - row[checksum] = None + if row[checksum] == "" or checksum not in settings.ALLOWED_CONTENT_CHECKSUMS: + del row[checksum] class Meta: model = Artifact diff --git a/pulpcore/app/models/domain.py b/pulpcore/app/models/domain.py index 31630898646..a4ef35069d1 100644 --- a/pulpcore/app/models/domain.py +++ b/pulpcore/app/models/domain.py @@ -1,4 +1,5 @@ -from django.core.files.storage import get_storage_class, default_storage +from django.core.files.storage import default_storage +from django.utils.module_loading import import_string from django.db import models from django_lifecycle import hook, BEFORE_DELETE, BEFORE_UPDATE @@ -41,7 +42,7 @@ def get_storage(self): """Returns this domain's instantiated storage class.""" if self.name == "default": return default_storage - storage_class = get_storage_class(self.storage_class) + storage_class = import_string(self.storage_class) return storage_class(**self.storage_settings) @hook(BEFORE_DELETE, when="name", is_now="default") diff --git a/pulpcore/app/serializers/domain.py b/pulpcore/app/serializers/domain.py index 478e976bdc6..435588df717 100644 --- a/pulpcore/app/serializers/domain.py +++ b/pulpcore/app/serializers/domain.py @@ -1,7 +1,7 @@ from gettext import gettext as _ from django.conf import settings -from django.core.files.storage import import_string +from django.utils.module_loading import import_string from django.core.exceptions import ImproperlyConfigured from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema_field @@ -40,7 +40,10 @@ def to_representation(self, instance): # Should I convert back the saved settings to their Setting names for to_representation? if getattr(self.context.get("domain", None), "name", None) == "default": for setting_name, field in self.SETTING_MAPPING.items(): - if value := getattr(settings, setting_name.upper(), None): + value = getattr(settings, setting_name, None) or settings.STORAGES["default"].get( + "OPTIONS", {} + ).get(field) + if value: instance[field] = value return super().to_representation(instance) diff --git a/pulpcore/app/settings.py b/pulpcore/app/settings.py index 4e2197f040f..0a12edbb0df 100644 --- a/pulpcore/app/settings.py +++ b/pulpcore/app/settings.py @@ -17,6 +17,8 @@ from pkg_resources import iter_entry_points from cryptography.fernet import Fernet +from django.conf import global_settings +from django.core.files.storage import storages # noqa: F401 from django.core.exceptions import ImproperlyConfigured from django.db import connection @@ -42,7 +44,23 @@ STATIC_URL = "/assets/" STATIC_ROOT = DEPLOY_ROOT / STATIC_URL.strip("/") -DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.FileSystem" +# begin compatibility layer for DEFAULT_FILE_STORAGE +# Django does not allow STORAGES and DEFAULT_FILE_STORAGE in the same settings module +# (they are mutually exclusive). We set both on global_settings as defaults so that +# users can override either one via dynaconf, and Django picks the right one for its version. +_DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.FileSystem" +_STORAGES = { + "default": { + "BACKEND": "pulpcore.app.models.storage.FileSystem", + }, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + }, +} +setattr(global_settings, "DEFAULT_FILE_STORAGE", _DEFAULT_FILE_STORAGE) +setattr(global_settings, "STORAGES", _STORAGES) +# end DEFAULT_FILE_STORAGE compatibility layer + REDIRECT_TO_OBJECT_STORAGE = True WORKING_DIRECTORY = DEPLOY_ROOT / "tmp" @@ -317,16 +335,18 @@ from dynaconf import DjangoDynaconf, Validator # noqa # Validators +storage_keys = ("STORAGES.default.BACKEND", "DEFAULT_FILE_STORAGE") storage_validator = ( Validator("REDIRECT_TO_OBJECT_STORAGE", eq=False) - | Validator("DEFAULT_FILE_STORAGE", eq="pulpcore.app.models.storage.FileSystem") - | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.azure_storage.AzureStorage") - | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.s3boto3.S3Boto3Storage") - | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.gcloud.GoogleCloudStorage") + | Validator(*storage_keys, eq="pulpcore.app.models.storage.FileSystem") + | Validator(*storage_keys, eq="storages.backends.azure_storage.AzureStorage") + | Validator(*storage_keys, eq="storages.backends.s3boto3.S3Boto3Storage") + | Validator(*storage_keys, eq="storages.backends.gcloud.GoogleCloudStorage") ) storage_validator.messages["combined"] = ( - "'REDIRECT_TO_OBJECT_STORAGE=True' is only supported with the local file, S3, GCP or Azure" - "storage backend configured in DEFAULT_FILE_STORAGE." + "'REDIRECT_TO_OBJECT_STORAGE=True' is only supported with the local file, S3, GCP or Azure " + "storage backend configured in STORAGES['default']['BACKEND'] " + "(deprecated DEFAULT_FILE_STORAGE)." ) cache_enabled_validator = Validator("CACHE_ENABLED", eq=True) @@ -473,6 +493,10 @@ finally: connection.close() +# Ensures the cached property storage.backends uses the right value after dynaconf init +storages._backends = settings.STORAGES.copy() +storages.backends + settings.set("V3_API_ROOT", settings.API_ROOT + "api/v3/") # Not user configurable settings.set("V3_DOMAIN_API_ROOT", settings.API_ROOT + "/api/v3/") settings.set("V3_API_ROOT_NO_FRONT_SLASH", settings.V3_API_ROOT.lstrip("/")) diff --git a/pulpcore/app/tasks/export.py b/pulpcore/app/tasks/export.py index e6800456bb1..bc871f15b30 100644 --- a/pulpcore/app/tasks/export.py +++ b/pulpcore/app/tasks/export.py @@ -71,7 +71,7 @@ def _export_to_file_system(path, relative_paths_to_artifacts, method=FS_EXPORT_M ValidationError: When path is not in the ALLOWED_EXPORT_PATHS setting """ using_filesystem_storage = ( - settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem" + settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem" ) if method != FS_EXPORT_METHODS.WRITE and not using_filesystem_storage: diff --git a/pulpcore/app/tasks/importer.py b/pulpcore/app/tasks/importer.py index f091ce33210..f74b3c23b97 100644 --- a/pulpcore/app/tasks/importer.py +++ b/pulpcore/app/tasks/importer.py @@ -101,6 +101,9 @@ def _import_file(fpath, resource_class, retry=False): """ try: log.info(f"Importing file {fpath}.") + if not os.path.isfile(fpath): + log.info("...empty - skipping.") + return [] with open(fpath, "r") as json_file: resource = resource_class() log.info(f"...Importing resource {resource.__class__.__name__}.") @@ -109,6 +112,8 @@ def _import_file(fpath, resource_class, retry=False): # overlapping content. for batch_str in _impfile_iterator(json_file): data = Dataset().load(StringIO(batch_str)) + if not data: + return [] if retry: curr_attempt = 1 @@ -140,6 +145,7 @@ def _import_file(fpath, resource_class, retry=False): try: a_result = resource.import_data(data, raise_errors=True) except Exception as e: # noqa log on ANY exception and then re-raise + log.error(e) log.error(f"FATAL import-failure importing {fpath}") raise else: diff --git a/pulpcore/app/util.py b/pulpcore/app/util.py index 9ff52fe2594..468360f7b9f 100644 --- a/pulpcore/app/util.py +++ b/pulpcore/app/util.py @@ -387,7 +387,7 @@ def get_artifact_url(artifact, headers=None, http_method=None): if settings.DOMAIN_ENABLED: loc = f"domain {artifact_domain.name}.storage_class" else: - loc = "settings.DEFAULT_FILE_STORAGE" + loc = "settings.STORAGES['default']['BACKEND']" raise NotImplementedError( f"The value {loc}={artifact_domain.storage_class} does not allow redirecting." @@ -433,7 +433,9 @@ def get_default_domain(): try: default_domain = Domain.objects.get(name="default") except Domain.DoesNotExist: - default_domain = Domain(name="default", storage_class=settings.DEFAULT_FILE_STORAGE) + default_domain = Domain( + name="default", storage_class=settings.STORAGES["default"]["BACKEND"] + ) default_domain.save(skip_hooks=True) return default_domain diff --git a/pulpcore/app/views/status.py b/pulpcore/app/views/status.py index b11ae1c356b..d81839766b7 100644 --- a/pulpcore/app/views/status.py +++ b/pulpcore/app/views/status.py @@ -18,7 +18,7 @@ def _disk_usage(): - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem": try: return shutil.disk_usage(default_storage.location) except Exception: diff --git a/pulpcore/plugin/importexport.py b/pulpcore/plugin/importexport.py index 747a7c0cffa..7ca48e175bc 100644 --- a/pulpcore/plugin/importexport.py +++ b/pulpcore/plugin/importexport.py @@ -41,6 +41,9 @@ def set_up_queryset(self): def dehydrate_pulp_domain(self, content): return str(content.pulp_domain_id) + def render(self, value, obj=None, **kwargs): + return super().render(value, obj, coerce_to_string=False, **kwargs) + def __init__(self, repo_version=None): self.repo_version = repo_version if repo_version: diff --git a/pulpcore/tests/functional/__init__.py b/pulpcore/tests/functional/__init__.py index 882bb885d14..e2fff33c714 100644 --- a/pulpcore/tests/functional/__init__.py +++ b/pulpcore/tests/functional/__init__.py @@ -1076,28 +1076,37 @@ def _domain_factory(): keys = dict() keys["pulpcore.app.models.storage.FileSystem"] = ["MEDIA_ROOT"] keys["storages.backends.s3boto3.S3Boto3Storage"] = [ - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "AWS_S3_ENDPOINT_URL", - "AWS_S3_ADDRESSING_STYLE", - "AWS_S3_SIGNATURE_VERSION", - "AWS_S3_REGION_NAME", - "AWS_STORAGE_BUCKET_NAME", + "access_key", + "secret_key", + "endpoint_url", + "addressing_style", + "signature_version", + "region_name", + "bucket_name", ] keys["storages.backends.azure_storage.AzureStorage"] = [ - "AZURE_ACCOUNT_NAME", - "AZURE_CONTAINER", - "AZURE_ACCOUNT_KEY", - "AZURE_URL_EXPIRATION_SECS", - "AZURE_OVERWRITE_FILES", - "AZURE_LOCATION", + "account_name", + "azure_container", + "account_key", + "expiration_secs", + "overwrite_files", + "location", ] settings = dict() - for key in keys[pulp_settings.DEFAULT_FILE_STORAGE]: - settings[key] = getattr(pulp_settings, key, None) + backend = pulp_settings.STORAGES["default"]["BACKEND"] + not_defined_settings = (k for k in keys[backend] if k not in settings) + # Read storage settings from STORAGES.default.OPTIONS + storages_dict = getattr(pulp_settings, "STORAGES", {}) + storage_options = storages_dict.get("default", {}).get("OPTIONS", {}) + if storage_options: + for key in not_defined_settings: + settings[key] = storage_options.get(key) + else: + for key in not_defined_settings: + settings[key] = getattr(pulp_settings, key, None) body = { "name": str(uuid.uuid4()), - "storage_class": pulp_settings.DEFAULT_FILE_STORAGE, + "storage_class": backend, "storage_settings": settings, } return gen_object_with_cleanup(domains_api_client, body) diff --git a/pulpcore/tests/functional/api/test_artifact_distribution.py b/pulpcore/tests/functional/api/test_artifact_distribution.py index d6735347c22..bdcaaf9cf32 100644 --- a/pulpcore/tests/functional/api/test_artifact_distribution.py +++ b/pulpcore/tests/functional/api/test_artifact_distribution.py @@ -2,9 +2,6 @@ import subprocess from hashlib import sha256 -from django.conf import settings - - OBJECT_STORAGES = ( "storages.backends.s3boto3.S3Boto3Storage", "storages.backends.azure_storage.AzureStorage", @@ -12,7 +9,7 @@ ) -def test_artifact_distribution(random_artifact): +def test_artifact_distribution(random_artifact, pulp_settings): artifact_uuid = random_artifact.pulp_href.split("/")[-2] commands = ( @@ -29,7 +26,7 @@ def test_artifact_distribution(random_artifact): hasher = sha256() hasher.update(response.content) assert hasher.hexdigest() == random_artifact.sha256 - if settings.DEFAULT_FILE_STORAGE in OBJECT_STORAGES: + if pulp_settings.STORAGES["default"]["BACKEND"] in OBJECT_STORAGES: content_disposition = response.headers.get("Content-Disposition") assert content_disposition is not None filename = artifact_uuid diff --git a/pulpcore/tests/functional/api/test_crd_artifacts.py b/pulpcore/tests/functional/api/test_crd_artifacts.py index 8b099979fcb..ce08ec4e03f 100644 --- a/pulpcore/tests/functional/api/test_crd_artifacts.py +++ b/pulpcore/tests/functional/api/test_crd_artifacts.py @@ -6,7 +6,6 @@ import uuid import pytest -from django.conf import settings from pulpcore.client.pulpcore import ApiException @@ -144,11 +143,11 @@ def test_upload_mixed_attrs(artifacts_api_client, pulpcore_random_file): @pytest.mark.parallel -def test_delete_artifact(artifacts_api_client, pulpcore_random_file): +def test_delete_artifact(artifacts_api_client, pulpcore_random_file, pulp_settings): """Delete an artifact, it is removed from the filesystem.""" - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if pulp_settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem": pytest.skip("this test only works for filesystem storage") - media_root = settings.MEDIA_ROOT + media_root = pulp_settings.MEDIA_ROOT artifact = artifacts_api_client.create(pulpcore_random_file["name"]) path_to_file = os.path.join(media_root, artifact.file) diff --git a/pulpcore/tests/functional/api/test_crud_domains.py b/pulpcore/tests/functional/api/test_crud_domains.py index 781fea2d90c..3d89b5887f5 100644 --- a/pulpcore/tests/functional/api/test_crud_domains.py +++ b/pulpcore/tests/functional/api/test_crud_domains.py @@ -4,7 +4,6 @@ import random import json from pulpcore.client.pulpcore import ApiException -from pulpcore.app import settings from pulpcore.tests.functional.utils import PulpTaskError @@ -50,7 +49,7 @@ def test_crud_domains(domains_api_client, monitor_task): @pytest.mark.parallel -def test_default_domain(domains_api_client): +def test_default_domain(domains_api_client, pulp_settings): """Test properties around the default domain.""" domains = domains_api_client.list(name="default") assert domains.count == 1 @@ -58,9 +57,9 @@ def test_default_domain(domains_api_client): # Read the default domain, ensure storage is set to default default_domain = domains.results[0] assert default_domain.name == "default" - assert default_domain.storage_class == settings.DEFAULT_FILE_STORAGE - assert default_domain.redirect_to_object_storage == settings.REDIRECT_TO_OBJECT_STORAGE - assert default_domain.hide_guarded_distributions == settings.HIDE_GUARDED_DISTRIBUTIONS + assert default_domain.storage_class == pulp_settings.STORAGES["default"]["BACKEND"] + assert default_domain.redirect_to_object_storage == pulp_settings.REDIRECT_TO_OBJECT_STORAGE + assert default_domain.hide_guarded_distributions == pulp_settings.HIDE_GUARDED_DISTRIBUTIONS # Try to create another default domain body = { @@ -91,9 +90,11 @@ def test_default_domain(domains_api_client): @pytest.mark.parallel -def test_active_domain_deletion(domains_api_client, rbac_contentguard_api_client, monitor_task): +def test_active_domain_deletion( + domains_api_client, rbac_contentguard_api_client, monitor_task, pulp_settings +): """Test trying to delete a domain that is in use, has objects in it.""" - if not settings.DOMAIN_ENABLED: + if not pulp_settings.DOMAIN_ENABLED: pytest.skip("Domains not enabled") name = str(uuid.uuid4()) body = { @@ -133,9 +134,10 @@ def test_orphan_domain_deletion( gen_object_with_cleanup, monitor_task, tmp_path, + pulp_settings, ): """Test trying to delete a domain that is in use, has objects in it.""" - if not settings.DOMAIN_ENABLED: + if not pulp_settings.DOMAIN_ENABLED: pytest.skip("Domains not enabled") body = { "name": str(uuid.uuid4()), @@ -177,9 +179,9 @@ def test_orphan_domain_deletion( @pytest.mark.parallel -def test_special_domain_creation(domains_api_client, gen_object_with_cleanup): +def test_special_domain_creation(domains_api_client, gen_object_with_cleanup, pulp_settings): """Test many possible domain creation scenarios.""" - if not settings.DOMAIN_ENABLED: + if not pulp_settings.DOMAIN_ENABLED: pytest.skip("Domains not enabled") # This test needs to account for which environment it is running in storage_types = { diff --git a/pulpcore/tests/functional/api/test_status.py b/pulpcore/tests/functional/api/test_status.py index f3711054a3e..dfae098f96c 100644 --- a/pulpcore/tests/functional/api/test_status.py +++ b/pulpcore/tests/functional/api/test_status.py @@ -2,7 +2,6 @@ import pytest -from django.conf import settings from jsonschema import validate from pulpcore.client.pulpcore import ApiException @@ -59,9 +58,9 @@ @pytest.fixture(scope="module") -def expected_pulp_status_schema(): +def expected_pulp_status_schema(pulp_settings): """Returns the expected status response.""" - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if pulp_settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem": STATUS["properties"]["storage"].pop("properties") STATUS["properties"]["storage"]["type"] = "null" diff --git a/pulpcore/tests/unit/models/test_content.py b/pulpcore/tests/unit/models/test_content.py index 319180d64c2..04eb424b8d3 100644 --- a/pulpcore/tests/unit/models/test_content.py +++ b/pulpcore/tests/unit/models/test_content.py @@ -43,7 +43,7 @@ def test_create_read_delete_content(tmp_path): @pytest.mark.django_db def test_storage_location(tmp_path, settings): - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem": pytest.skip("Skipping test for nonlocal storage.") tf = tmp_path / "ab" diff --git a/pulpcore/tests/unit/viewsets/test_viewset_base.py b/pulpcore/tests/unit/viewsets/test_viewset_base.py index 7da60311528..da1a0d21a8a 100644 --- a/pulpcore/tests/unit/viewsets/test_viewset_base.py +++ b/pulpcore/tests/unit/viewsets/test_viewset_base.py @@ -1,5 +1,5 @@ import pytest -from pytest_django.asserts import assertQuerysetEqual +from pytest_django.asserts import assertQuerySetEqual import unittest from django.http import Http404, QueryDict @@ -23,7 +23,7 @@ def test_adds_filters(): queryset = viewset.get_queryset() expected = models.RepositoryVersion.objects.filter(repository__pk=repo.pk) - assertQuerysetEqual(queryset, expected) + assertQuerySetEqual(queryset, expected) @pytest.mark.django_db @@ -38,7 +38,7 @@ def test_does_not_add_filters(): queryset = viewset.get_queryset() expected = models.Repository.objects.all() - assertQuerysetEqual(queryset, expected) + assertQuerySetEqual(queryset, expected) def test_must_define_serializer_class(): diff --git a/requirements.txt b/requirements.txt index 5b319cd92f0..d55b6556d2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,15 @@ aiodns>=3.0,<=3.2.0 aiofiles>=22.1,<23.3.0 -aiohttp>=3.8.1,<3.10.12 +aiohttp>=3.9.0,<3.10.12 asyncio-throttle>=1.0,<=1.0.2 async-timeout>=4.0.3,<4.0.4;python_version<"3.11" backoff>=2.1.2,<2.2.2 click>=8.1.0,<=8.1.7 cryptography>=38.0.1,<42.0.9 -Django~=4.2.0 # LTS version, switch only if we have a compelling reason to +Django>=4.2.24,<5.3, !=5.0, !=5.1 # LTS version, switch only if we have a compelling reason to django-filter>=23.1,<=23.5 django-guid>=3.3,<=3.4.0 -django-import-export>=2.9,<3.4.0 +django-import-export>=2.9,<5.0 django-lifecycle>=1.0,<=1.1.2 djangorestframework>=3.14.0,<=3.15.2 djangorestframework-queryfields>=1.0,<=1.1.0 @@ -35,8 +35,8 @@ python-gnupg>=0.5,<=0.5.2 PyYAML>=5.1.1,<=6.0.1 redis>=4.3,<5.0.3 setuptools>=39.2,<69.1.0 -tablib<3.6.0 +tablib>=3.5.0,<4.0, !=3.6 url-normalize>=1.4.3,<=1.4.3 uuid6>=2023.5.2,<=2024.1.12 whitenoise>=5.0,<6.7.0 -yarl>=1.8,<1.15.3 +yarl>=1.9.0,<1.15.3 diff --git a/template_config.yml b/template_config.yml index fd20cba4580..af2e4dbb974 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,15 +1,10 @@ -# This config represents the latest values used when running the plugin-template. Any settings that -# were not present before running plugin-template have been added with their default values. - -# generated with plugin_template - api_root: /pulp/ black: true check_commit_message: true check_gettext: true check_manifest: true check_stray_pulpcore_imports: false -ci_base_image: ghcr.io/pulp/pulp-ci-centos +ci_base_image: ghcr.io/pulp/pulp-ci-centos9 ci_env: {} ci_trigger: '{pull_request: {branches: [''*'']}}' cli_package: pulp-cli @@ -51,30 +46,40 @@ pulp_settings: - /tmp orphan_protection_time: 0 pulp_settings_azure: - AZURE_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== - AZURE_ACCOUNT_NAME: devstoreaccount1 - AZURE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1; - AZURE_CONTAINER: pulp-test - AZURE_LOCATION: pulp3 - AZURE_OVERWRITE_FILES: true - AZURE_URL_EXPIRATION_SECS: 120 - DEFAULT_FILE_STORAGE: storages.backends.azure_storage.AzureStorage MEDIA_ROOT: '' + STORAGES: + default: + BACKEND: storages.backends.azure_storage.AzureStorage + OPTIONS: + account_key: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + account_name: devstoreaccount1 + connection_string: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1; + azure_container: pulp-test + location: pulp3 + overwrite_files: true + expiration_secs: 120 + staticfiles: + BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage domain_enabled: true pulp_settings_gcp: null pulp_settings_s3: - AWS_ACCESS_KEY_ID: AKIAIT2Z5TDYPX3ARJBA - AWS_DEFAULT_ACL: '@none None' - AWS_S3_ADDRESSING_STYLE: path - AWS_S3_ENDPOINT_URL: http://minio:9000 - AWS_S3_REGION_NAME: eu-central-1 - AWS_S3_SIGNATURE_VERSION: s3v4 - AWS_SECRET_ACCESS_KEY: fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS - AWS_STORAGE_BUCKET_NAME: pulp3 - DEFAULT_FILE_STORAGE: storages.backends.s3boto3.S3Boto3Storage MEDIA_ROOT: '' domain_enabled: true hide_guarded_distributions: true + STORAGES: + default: + BACKEND: storages.backends.s3boto3.S3Boto3Storage + OPTIONS: + access_key: AKIAIT2Z5TDYPX3ARJBA + secret_key: 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 + bucket_name: pulp3 + endpoint_url: http://minio:9000 + region_name: eu-central-1 + signature_version: s3v4 + addressing_style: path + default_acl: '@none' + staticfiles: + BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage pydocstyle: true release_email: pulp-infra@redhat.com release_user: pulpbot @@ -99,4 +104,3 @@ test_performance: false test_reroute: true test_s3: true use_issue_template: true -