diff --git a/CHANGES/5324.feature b/CHANGES/5324.feature new file mode 100644 index 00000000000..1f08a1cb0b4 --- /dev/null +++ b/CHANGES/5324.feature @@ -0,0 +1 @@ +Adapted PulpImport/Export to allow update django-import-export==4.x. diff --git a/CHANGES/6988.feature b/CHANGES/6988.feature new file mode 100644 index 00000000000..11200bdaca7 --- /dev/null +++ b/CHANGES/6988.feature @@ -0,0 +1,3 @@ +Allow use of Django 5 as well as Django 4. Note the following breaking changes if upgrading to +Django 5: storage configuration must use the new ``STORAGES`` format instead of +``DEFAULT_FILE_STORAGE``, Python >= 3.10 is required, and PostgreSQL >= 14 is required. diff --git a/pulpcore/app/importexport.py b/pulpcore/app/importexport.py index c8ade370778..dd30042b134 100644 --- a/pulpcore/app/importexport.py +++ b/pulpcore/app/importexport.py @@ -48,6 +48,10 @@ def _write_export(the_tarfile, resource, dest_dir=None): # the data in batches to memory and concatenate the json lists via string manipulation. with tempfile.NamedTemporaryFile(dir=".", mode="w", encoding="utf8") as temp_file: if isinstance(resource.queryset, QuerySet): + # If we don't have any of "these" - skip writing + if resource.queryset.count() == 0: + return + temp_file.write("[") def process_batch(batch): diff --git a/pulpcore/app/modelresource.py b/pulpcore/app/modelresource.py index 54d3522fa98..632e982f06a 100644 --- a/pulpcore/app/modelresource.py +++ b/pulpcore/app/modelresource.py @@ -1,3 +1,4 @@ +from django.conf import settings from import_export import fields from import_export.widgets import ForeignKeyWidget from logging import getLogger @@ -36,8 +37,8 @@ def before_import_row(self, row, **kwargs): # the export converts None to blank strings but sha384 and sha512 have unique constraints # that get triggered if they are blank. convert checksums back into None if they are blank. for checksum in ALL_KNOWN_CONTENT_CHECKSUMS: - if row[checksum] == "": - row[checksum] = None + if row[checksum] == "" or checksum not in settings.ALLOWED_CONTENT_CHECKSUMS: + del row[checksum] class Meta: model = Artifact diff --git a/pulpcore/app/tasks/importer.py b/pulpcore/app/tasks/importer.py index a8217411d7f..97781fe816d 100644 --- a/pulpcore/app/tasks/importer.py +++ b/pulpcore/app/tasks/importer.py @@ -228,6 +228,9 @@ def _import_file(fpath, resource_class, retry=False): """ try: log.info(f"Importing file {fpath}.") + if not os.path.isfile(fpath): + log.info("...empty - skipping.") + return [] with open(fpath, "r") as json_file: resource = resource_class() log.info(f"...Importing resource {resource.__class__.__name__}.") @@ -236,6 +239,8 @@ def _import_file(fpath, resource_class, retry=False): # overlapping content. for batch_str in _impfile_iterator(json_file): data = Dataset().load(StringIO(batch_str)) + if not data: + return [] if retry: curr_attempt = 1 @@ -267,6 +272,7 @@ def _import_file(fpath, resource_class, retry=False): try: a_result = resource.import_data(data, raise_errors=True) except Exception as e: # noqa log on ANY exception and then re-raise + log.error(e) log.error(f"FATAL import-failure importing {fpath}") raise else: diff --git a/pulpcore/content/handler.py b/pulpcore/content/handler.py index 89ec8db415f..1f9a843c41b 100644 --- a/pulpcore/content/handler.py +++ b/pulpcore/content/handler.py @@ -7,6 +7,7 @@ import struct from gettext import gettext as _ from datetime import datetime, timedelta +from datetime import timezone as dt_timezone from aiohttp.client_exceptions import ClientResponseError, ClientConnectionError from aiohttp.web import FileResponse, StreamResponse, HTTPOk @@ -439,9 +440,9 @@ def _parse_checkpoint_path(path): else: raise PathNotResolved(path) - request_timestamp = request_timestamp.replace(tzinfo=timezone.utc) + request_timestamp = request_timestamp.replace(tzinfo=dt_timezone.utc) # Future timestamps are not allowed for checkpoints - if request_timestamp > datetime.now(tz=timezone.utc): + if request_timestamp > datetime.now(tz=dt_timezone.utc): raise PathNotResolved(path) # The timestamp is truncated to seconds, so we need to cover the whole second request_timestamp = request_timestamp.replace(microsecond=999999) diff --git a/pulpcore/plugin/importexport.py b/pulpcore/plugin/importexport.py index 2ceb25182ac..af2cbfb0f3c 100644 --- a/pulpcore/plugin/importexport.py +++ b/pulpcore/plugin/importexport.py @@ -41,6 +41,9 @@ def set_up_queryset(self): def dehydrate_pulp_domain(self, content): return str(content.pulp_domain_id) + def render(self, value, obj=None, **kwargs): + return super().render(value, obj, coerce_to_string=False, **kwargs) + def __init__(self, repo_version=None): self.repo_version = repo_version if repo_version: diff --git a/pulpcore/tests/functional/api/using_plugin/test_crud_repos.py b/pulpcore/tests/functional/api/using_plugin/test_crud_repos.py index eb0523d669f..91f6f494b5b 100644 --- a/pulpcore/tests/functional/api/using_plugin/test_crud_repos.py +++ b/pulpcore/tests/functional/api/using_plugin/test_crud_repos.py @@ -299,7 +299,7 @@ def raise_for_invalid_request(remote_attrs): """Check if Pulp returns HTTP 400 after issuing an invalid request.""" with pytest.raises(ApiException) as ae: file_bindings.RemotesFileApi.create(remote_attrs) - assert ae.value.status == 400 + assert ae.value.status == 400 # Test the validation of an invalid absolute pathname. remote_attrs = { diff --git a/pulpcore/tests/unit/viewsets/test_viewset_base.py b/pulpcore/tests/unit/viewsets/test_viewset_base.py index 7da60311528..da1a0d21a8a 100644 --- a/pulpcore/tests/unit/viewsets/test_viewset_base.py +++ b/pulpcore/tests/unit/viewsets/test_viewset_base.py @@ -1,5 +1,5 @@ import pytest -from pytest_django.asserts import assertQuerysetEqual +from pytest_django.asserts import assertQuerySetEqual import unittest from django.http import Http404, QueryDict @@ -23,7 +23,7 @@ def test_adds_filters(): queryset = viewset.get_queryset() expected = models.RepositoryVersion.objects.filter(repository__pk=repo.pk) - assertQuerysetEqual(queryset, expected) + assertQuerySetEqual(queryset, expected) @pytest.mark.django_db @@ -38,7 +38,7 @@ def test_does_not_add_filters(): queryset = viewset.get_queryset() expected = models.Repository.objects.all() - assertQuerysetEqual(queryset, expected) + assertQuerySetEqual(queryset, expected) def test_must_define_serializer_class(): diff --git a/pyproject.toml b/pyproject.toml index 265733b0545..6d2d991c752 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,10 +32,10 @@ dependencies = [ "backoff>=2.1.2,<2.3", # Looks like only bugfixes in z-Stream. "click>=8.1.0,<8.3", # Uses milestone.feature.fix https://palletsprojects.com/versions . "cryptography>=44.0.3,<46.0", # SemVer compatible https://cryptography.io/en/latest/api-stability/#versioning . - "Django~=4.2.0", # LTS version, switch only if we have a compelling reason to". + "Django>=4.2.24,<5.3, !=5.0, !=5.1", # LTS version, switch only if we have a compelling reason to". "django-filter>=23.1,<=25.1", # Looks like CalVer. "django-guid>=3.3.0,<3.6", # Looks like only bugfixes in z-Stream. - "django-import-export>=2.9,<3.4.0", + "django-import-export>=2.9,<5.0", "django-lifecycle>=1.0,<=1.2.4", "djangorestframework>=3.14.0,<=3.16.1", "djangorestframework-queryfields>=1.0,<=1.1.0", @@ -59,7 +59,7 @@ dependencies = [ "python-gnupg>=0.5,<=0.5.4", "PyYAML>=5.1.1,<6.1", # Looks like only bugfixes in z-Stream. "redis>=4.3.0,<6.5", # Looks like only bugfixes in z-Stream. - "tablib>=3.5.0,<3.6", # 3.6.0 breaks with import export. Not sure about semver. + "tablib>=3.5.0,<4.0, !=3.6", # 3.6.0 breaks with import export. Not sure about semver. "url-normalize>=1.4.3,<2.3", # SemVer. https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md#changelog "uuid6>=2023.5.2,<=2025.0.1", "whitenoise>=5.0,<6.10.0",