From 7c62ea024c7a54ae874cb79b78b983a931bbdc0e Mon Sep 17 00:00:00 2001 From: Goulwen Date: Mon, 28 Sep 2015 15:03:16 +0200 Subject: [PATCH 1/7] Python 3 compatibility --- src/django_fields/fields.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/django_fields/fields.py b/src/django_fields/fields.py index 0075a6f..552ec61 100644 --- a/src/django_fields/fields.py +++ b/src/django_fields/fields.py @@ -28,6 +28,12 @@ except: import pickle +if sys.version_info[0] == 3: + PYTHON3 = True +else: + PYTHON3 = False + + class BaseEncryptedField(models.Field): '''This code is based on the djangosnippet #1095 You can find the original at http://www.djangosnippets.org/snippets/1095/''' @@ -268,7 +274,11 @@ def get_db_prep_value(self, value, connection=None, prepared=False): class EncryptedIntField(BaseEncryptedNumberField): __metaclass__ = models.SubfieldBase - max_raw_length = len(str(-sys.maxint - 1)) + + if PYTHON3 is True: + max_raw_length = len(str(-sys.maxsize - 1)) + else: + max_raw_length = len(str(-sys.maxint - 1)) number_type = int format_string = "%d" @@ -276,7 +286,10 @@ class EncryptedIntField(BaseEncryptedNumberField): class EncryptedLongField(BaseEncryptedNumberField): __metaclass__ = models.SubfieldBase max_raw_length = None # no limit - number_type = long + if PYTHON3 is True: + number_type = int + else: + number_type = long format_string = "%d" def get_internal_type(self): @@ -324,7 +337,7 @@ def get_internal_type(self): def formfield(self, **kwargs): try: - from localflavor.us.forms import USPhoneNumberField + from localflavor.us.forms import USPhoneNumberField except ImportError: from django.contrib.localflavor.us.forms import USPhoneNumberField @@ -343,7 +356,7 @@ def formfield(self, **kwargs): try: from localflavor.us.forms import USSocialSecurityNumberField except ImportError: - from django.contrib.localflavor.us.forms import USSocialSecurityNumberField + from django.contrib.localflavor.us.forms import USSocialSecurityNumberField defaults = {'form_class': USSocialSecurityNumberField} defaults.update(kwargs) From 159bbb363eef0c154808fc01af0036dc15f4bb1e Mon Sep 17 00:00:00 2001 From: Goulwen Date: Mon, 28 Sep 2015 15:40:40 +0200 Subject: [PATCH 2/7] More compat with python3 --- src/django_fields/fields.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/django_fields/fields.py b/src/django_fields/fields.py index 552ec61..be9b1d9 100644 --- a/src/django_fields/fields.py +++ b/src/django_fields/fields.py @@ -82,7 +82,12 @@ def __init__(self, *args, **kwargs): super(BaseEncryptedField, self).__init__(*args, **kwargs) def _is_encrypted(self, value): - return isinstance(value, basestring) and value.startswith(self.prefix) + if PYTHON3 is True: + return isinstance(value, str) and value.startswith( + self.prefix) + else: + return isinstance(value, basestring) and value.startswith( + self.prefix) def _get_padding(self, value): # We always want at least 2 chars of padding (including zero byte), @@ -126,7 +131,12 @@ def get_db_prep_value(self, value, connection=None, prepared=False): self.iv) value = self.prefix + binascii.b2a_hex(self.iv + self.cipher.encrypt(value)) else: - value = self.prefix + binascii.b2a_hex(self.cipher.encrypt(value)) + if PYTHON3 is True: + value = self.prefix + binascii.b2a_hex( + self.cipher.encrypt(value)).decode('utf-8') + else: + value = self.prefix + binascii.b2a_hex( + self.cipher.encrypt(value)) return value def deconstruct(self): From 317643181bd9b33cbc8d71f589cdea84fc5e11e8 Mon Sep 17 00:00:00 2001 From: Goulwen Date: Wed, 30 Sep 2015 15:59:46 +0200 Subject: [PATCH 3/7] Python 3 compatibility --- src/django_fields/fields.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/django_fields/fields.py b/src/django_fields/fields.py index be9b1d9..0b96c02 100644 --- a/src/django_fields/fields.py +++ b/src/django_fields/fields.py @@ -83,8 +83,8 @@ def __init__(self, *args, **kwargs): def _is_encrypted(self, value): if PYTHON3 is True: - return isinstance(value, str) and value.startswith( - self.prefix) + is_enc = isinstance(value, str) and value.startswith(self.prefix) + return is_enc else: return isinstance(value, basestring) and value.startswith( self.prefix) @@ -95,7 +95,7 @@ def _get_padding(self, value): mod = (len(value) + 2) % self.cipher.block_size return self.cipher.block_size - mod + 2 - def to_python(self, value): + def from_db_value(self, value, expression, connection, context): if self._is_encrypted(value): if self.block_type: self.iv = binascii.a2b_hex(value[len(self.prefix):])[:len(self.iv)] @@ -107,7 +107,7 @@ def to_python(self, value): else: decrypt_value = binascii.a2b_hex(value[len(self.prefix):]) return force_unicode( - self.cipher.decrypt(decrypt_value).split('\0')[0] + self.cipher.decrypt(decrypt_value).split(b'\0')[0] ) return value From 6ba1cb00e9c1014b0841ff24fc9fce4fcdd65efd Mon Sep 17 00:00:00 2001 From: Goulwen Date: Wed, 30 Sep 2015 16:35:44 +0200 Subject: [PATCH 4/7] Port to django 1.8 and port tests to python 3. --- src/django_fields/fields.py | 36 +++++++++++++++++++++++----- src/django_fields/models.py | 12 +++++++++- src/django_fields/tests.py | 48 +++++++++++++++++++++++++++---------- 3 files changed, 76 insertions(+), 20 deletions(-) diff --git a/src/django_fields/fields.py b/src/django_fields/fields.py index 0b96c02..6b9828c 100644 --- a/src/django_fields/fields.py +++ b/src/django_fields/fields.py @@ -129,7 +129,12 @@ def get_db_prep_value(self, value, connection=None, prepared=False): self.secret_key, getattr(self.cipher_object, self.block_type), self.iv) - value = self.prefix + binascii.b2a_hex(self.iv + self.cipher.encrypt(value)) + if PYTHON3 is True: + value = self.prefix + binascii.b2a_hex( + self.iv + self.cipher.encrypt(value)).decode('utf-8') + else: + value = self.prefix + binascii.b2a_hex( + self.iv + self.cipher.encrypt(value)) else: if PYTHON3 is True: value = self.prefix + binascii.b2a_hex( @@ -207,6 +212,9 @@ def formfield(self, **kwargs): return super(BaseEncryptedDateField, self).formfield(**defaults) def to_python(self, value): + return self.from_db_value(value) + + def from_db_value(self, value, expression, connection, context): # value is either a date or a string in the format "YYYY:MM:DD" if value in fields.EMPTY_VALUES: @@ -215,7 +223,8 @@ def to_python(self, value): if isinstance(value, self.date_class): date_value = value else: - date_text = super(BaseEncryptedDateField, self).to_python(value) + date_text = super(BaseEncryptedDateField, self).from_db_value( + value, expression, connection, context) date_value = self.date_class(*map(int, date_text.split(':'))) return date_value @@ -264,11 +273,15 @@ def get_internal_type(self): return 'CharField' def to_python(self, value): + return self.from_db_value(value) + + def from_db_value(self, value, expression, connection, context): # value is either an int or a string of an integer if isinstance(value, self.number_type) or value == '': number = value else: - number_text = super(BaseEncryptedNumberField, self).to_python(value) + number_text = super(BaseEncryptedNumberField, self).from_db_value( + value, expression, connection, context) number = self.number_type(number_text) return number @@ -324,13 +337,24 @@ def get_db_prep_value(self, value, connection=None, prepared=False): return pickle.dumps(value) def to_python(self, value): - if not isinstance(value, basestring): - return value + return self.from_db_value(value) + + def from_db_value(self, value, expression, connection, context): + if PYTHON3 is True: + if not isinstance(value, str): + return value + else: + if not isinstance(value, basestring): + return value # Tries to convert unicode objects to string, cause loads pickle from # unicode excepts ugly ``KeyError: '\x00'``. try: - return pickle.loads(smart_str(value)) + if PYTHON3 is True: + val = value.encode('utf-8') + return pickle.loads(val) + else: + return pickle.loads(smart_str(value)) # If pickle could not loads from string it's means that it's Python # string saved to PickleField. except ValueError: diff --git a/src/django_fields/models.py b/src/django_fields/models.py index b1afa50..3b7e3ed 100644 --- a/src/django_fields/models.py +++ b/src/django_fields/models.py @@ -1,8 +1,14 @@ # -*- coding: utf-8 -*- import re +import sys from django.db import models +if sys.version_info[0] == 3: + PYTHON3 = True +else: + PYTHON3 = False + class PrivateFieldsMetaclass(models.base.ModelBase): """Metaclass to set right default db_column values @@ -64,7 +70,11 @@ def __init__(self, *args, **kwargs): field_names = set(f.name for f in self._meta.fields) - for key, value in kwargs.iteritems(): + if PYTHON3 is True: + items = kwargs.items() + else: + items = kwargs.iteritems() + for key, value in items: if prefix + key in field_names: key = prefix + key new_kwargs[key] = value diff --git a/src/django_fields/tests.py b/src/django_fields/tests.py index 971c994..33ce283 100644 --- a/src/django_fields/tests.py +++ b/src/django_fields/tests.py @@ -20,6 +20,11 @@ from .models import ModelWithPrivateFields +if sys.version_info[0] == 3: + PYTHON3 = True +else: + PYTHON3 = False + class EncObject(models.Model): max_password = 20 @@ -180,14 +185,14 @@ def test_none_value(self): def _get_encrypted_password(self, id): cursor = connection.cursor() cursor.execute("select password from django_fields_encobject where id = %s", [id,]) - passwords = map(lambda x: x[0], cursor.fetchall()) + passwords = list(map(lambda x: x[0], cursor.fetchall())) self.assertEqual(len(passwords), 1) # only one return passwords[0] def _get_encrypted_password_cipher(self, id): cursor = connection.cursor() cursor.execute("select password from django_fields_cipherencobject where id = %s", [id,]) - passwords = map(lambda x: x[0], cursor.fetchall()) + passwords = list(map(lambda x: x[0], cursor.fetchall())) self.assertEqual(len(passwords), 1) # only one return passwords[0] @@ -254,21 +259,21 @@ def test_date_encryption_w_cipher(self): def _get_encrypted_date(self, id): cursor = connection.cursor() cursor.execute("select important_date from django_fields_encdate where id = %s", [id,]) - important_dates = map(lambda x: x[0], cursor.fetchall()) + important_dates = list(map(lambda x: x[0], cursor.fetchall())) self.assertEqual(len(important_dates), 1) # only one return important_dates[0] def _get_encrypted_datetime(self, id): cursor = connection.cursor() cursor.execute("select important_datetime from django_fields_encdatetime where id = %s", [id,]) - important_datetimes = map(lambda x: x[0], cursor.fetchall()) + important_datetimes = list(map(lambda x: x[0], cursor.fetchall())) self.assertEqual(len(important_datetimes), 1) # only one return important_datetimes[0] def _get_encrypted_date_cipher(self, id): cursor = connection.cursor() cursor.execute("select important_date from django_fields_cipherencdate where id = %s", [id,]) - important_dates = map(lambda x: x[0], cursor.fetchall()) + important_dates = list(map(lambda x: x[0], cursor.fetchall())) self.assertEqual(len(important_dates), 1) # only one return important_dates[0] @@ -280,20 +285,37 @@ def setUp(self): EncFloat.objects.all().delete() def test_int_encryption(self): - self._test_number_encryption(EncInt, 'int', sys.maxint) + if PYTHON3 is True: + self._test_number_encryption(EncInt, 'int', sys.maxsize) + else: + self._test_number_encryption(EncInt, 'int', sys.maxint) def test_min_int_encryption(self): - self._test_number_encryption(EncInt, 'int', -sys.maxint - 1) + if PYTHON3 is True: + self._test_number_encryption(EncInt, 'int', -sys.maxsize - 1) + else: + self._test_number_encryption(EncInt, 'int', -sys.maxint - 1) def test_long_encryption(self): - self._test_number_encryption(EncLong, 'long', long(sys.maxint) * 100L) + if PYTHON3 is True: + self._test_number_encryption( + EncLong, 'long', int(sys.maxsize) * 100) + else: + self._test_number_encryption( + EncLong, 'long', long(sys.maxint) * long(100)) def test_float_encryption(self): - value = 123.456 + sys.maxint + if PYTHON3 is True: + value = 123.456 + sys.maxsize + else: + value = 123.456 + sys.maxint self._test_number_encryption(EncFloat, 'float', value) def test_one_third_float_encryption(self): - value = sys.maxint + (1.0 / 3.0) + if PYTHON3 is True: + value = sys.maxsize + (1.0 / 3.0) + else: + value = sys.maxint + (1.0 / 3.0) self._test_number_encryption(EncFloat, 'float', value) def _test_number_encryption(self, number_class, type_name, value): @@ -311,7 +333,7 @@ def _get_encrypted_number(self, type_name, id): cursor = connection.cursor() sql = "select important_number from django_fields_enc%s where id = %%s" % (type_name,) cursor.execute(sql, [id,]) - important_numbers = map(lambda x: x[0], cursor.fetchall()) + important_numbers = list(map(lambda x: x[0], cursor.fetchall())) self.assertEqual(len(important_numbers), 1) # only one return important_numbers[0] @@ -418,7 +440,7 @@ def test_minimum_padding(self): def _get_encrypted_email(self, id): cursor = connection.cursor() cursor.execute("select email from django_fields_emailobject where id = %s", [id,]) - emails = map(lambda x: x[0], cursor.fetchall()) + emails = list(map(lambda x: x[0], cursor.fetchall())) self.assertEqual(len(emails), 1) # only one return emails[0] @@ -490,7 +512,7 @@ def test_cipher_storage_length_versus_schema_length(self): def _get_raw_password_value(self, id): cursor = connection.cursor() cursor.execute("select password from django_fields_cipherencobject where id = %s", [id, ]) - passwords = map(lambda x: x[0], cursor.fetchall()) + passwords = list(map(lambda x: x[0], cursor.fetchall())) self.assertEqual(len(passwords), 1) # only one return passwords[0] From 82c8ade5f4d1412825938ad51e12da141bf06af0 Mon Sep 17 00:00:00 2001 From: Goulwen Date: Fri, 2 Oct 2015 14:57:47 +0200 Subject: [PATCH 5/7] Python 3 support --- src/django_fields/fields.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/django_fields/fields.py b/src/django_fields/fields.py index 6b9828c..16025b3 100644 --- a/src/django_fields/fields.py +++ b/src/django_fields/fields.py @@ -1,4 +1,5 @@ import binascii +import codecs import datetime import string import sys @@ -137,6 +138,7 @@ def get_db_prep_value(self, value, connection=None, prepared=False): self.iv + self.cipher.encrypt(value)) else: if PYTHON3 is True: + print('>>>', value, '-->', len(value)/16) value = self.prefix + binascii.b2a_hex( self.cipher.encrypt(value)).decode('utf-8') else: @@ -334,7 +336,13 @@ class PickleField(models.TextField): serialize = False def get_db_prep_value(self, value, connection=None, prepared=False): - return pickle.dumps(value) + if PYTHON3 is True: + # When PYTHON3, we convert data to base64 to prevent errors when + # unpickling. + val = codecs.encode(pickle.dumps(value), 'base64').decode() + return val + else: + return pickle.dumps(value) def to_python(self, value): return self.from_db_value(value) @@ -351,8 +359,10 @@ def from_db_value(self, value, expression, connection, context): # unicode excepts ugly ``KeyError: '\x00'``. try: if PYTHON3 is True: - val = value.encode('utf-8') - return pickle.loads(val) + # When PYTHON3, data are in base64 to prevent errors when + # unpickling. + val = pickle.loads(codecs.decode(value.encode(), "base64")) + return val else: return pickle.loads(smart_str(value)) # If pickle could not loads from string it's means that it's Python From 5a53b878a083926e51dcb7e6ca56dcffced931eb Mon Sep 17 00:00:00 2001 From: Goulwen Date: Fri, 16 Oct 2015 15:19:46 +0200 Subject: [PATCH 6/7] Compat --- src/django_fields/fields.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/django_fields/fields.py b/src/django_fields/fields.py index 16025b3..334a278 100644 --- a/src/django_fields/fields.py +++ b/src/django_fields/fields.py @@ -9,7 +9,6 @@ from django.forms import fields from django.db import models from django.conf import settings -from django.utils.encoding import smart_str, force_unicode from django.utils.translation import ugettext_lazy as _ from Crypto import Random from Crypto.Random import random @@ -31,8 +30,10 @@ if sys.version_info[0] == 3: PYTHON3 = True + from django.utils.encoding import smart_str, force_text as force_unicode else: PYTHON3 = False + from django.utils.encoding import smart_str, force_unicode class BaseEncryptedField(models.Field): From fe5f5f89fbe2f40f50b8f1e5c6ec015e4b6bba02 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 22 Jul 2016 12:26:32 +0300 Subject: [PATCH 7/7] Removed usages of the deprecated SubfieldBase metaclass. --- src/django_fields/fields.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/src/django_fields/fields.py b/src/django_fields/fields.py index 334a278..0cc0c77 100644 --- a/src/django_fields/fields.py +++ b/src/django_fields/fields.py @@ -160,8 +160,6 @@ def deconstruct(self): class EncryptedTextField(BaseEncryptedField): - __metaclass__ = models.SubfieldBase - def get_internal_type(self): return 'TextField' @@ -172,8 +170,6 @@ def formfield(self, **kwargs): class EncryptedCharField(BaseEncryptedField): - __metaclass__ = models.SubfieldBase - def get_internal_type(self): return "CharField" @@ -214,9 +210,6 @@ def formfield(self, **kwargs): defaults.update(kwargs) return super(BaseEncryptedDateField, self).formfield(**defaults) - def to_python(self, value): - return self.from_db_value(value) - def from_db_value(self, value, expression, connection, context): # value is either a date or a string in the format "YYYY:MM:DD" @@ -246,7 +239,6 @@ def get_db_prep_value(self, value, connection=None, prepared=False): class EncryptedDateField(BaseEncryptedDateField): - __metaclass__ = models.SubfieldBase form_widget = forms.DateInput form_field = forms.DateField save_format = "%Y:%m:%d" @@ -256,7 +248,6 @@ class EncryptedDateField(BaseEncryptedDateField): class EncryptedDateTimeField(BaseEncryptedDateField): # FIXME: This doesn't handle time zones, but Python doesn't really either. - __metaclass__ = models.SubfieldBase form_widget = forms.DateTimeInput form_field = forms.DateTimeField save_format = "%Y:%m:%d:%H:%M:%S:%f" @@ -275,9 +266,6 @@ def __init__(self, *args, **kwargs): def get_internal_type(self): return 'CharField' - def to_python(self, value): - return self.from_db_value(value) - def from_db_value(self, value, expression, connection, context): # value is either an int or a string of an integer if isinstance(value, self.number_type) or value == '': @@ -299,8 +287,6 @@ def get_db_prep_value(self, value, connection=None, prepared=False): class EncryptedIntField(BaseEncryptedNumberField): - __metaclass__ = models.SubfieldBase - if PYTHON3 is True: max_raw_length = len(str(-sys.maxsize - 1)) else: @@ -310,7 +296,6 @@ class EncryptedIntField(BaseEncryptedNumberField): class EncryptedLongField(BaseEncryptedNumberField): - __metaclass__ = models.SubfieldBase max_raw_length = None # no limit if PYTHON3 is True: number_type = int @@ -323,7 +308,6 @@ def get_internal_type(self): class EncryptedFloatField(BaseEncryptedNumberField): - __metaclass__ = models.SubfieldBase max_raw_length = 150 # arbitrary, but should be sufficient number_type = float # If this format is too long for some architectures, change it. @@ -331,7 +315,6 @@ class EncryptedFloatField(BaseEncryptedNumberField): class PickleField(models.TextField): - __metaclass__ = models.SubfieldBase editable = False serialize = False @@ -345,9 +328,6 @@ def get_db_prep_value(self, value, connection=None, prepared=False): else: return pickle.dumps(value) - def to_python(self, value): - return self.from_db_value(value) - def from_db_value(self, value, expression, connection, context): if PYTHON3 is True: if not isinstance(value, str): @@ -375,8 +355,6 @@ def from_db_value(self, value, expression, connection, context): class EncryptedUSPhoneNumberField(BaseEncryptedField): - __metaclass__ = models.SubfieldBase - def get_internal_type(self): return "CharField" @@ -392,8 +370,6 @@ def formfield(self, **kwargs): class EncryptedUSSocialSecurityNumberField(BaseEncryptedField): - __metaclass__ = models.SubfieldBase - def get_internal_type(self): return "CharField" @@ -408,7 +384,6 @@ def formfield(self, **kwargs): return super(EncryptedUSSocialSecurityNumberField, self).formfield(**defaults) class EncryptedEmailField(BaseEncryptedField): - __metaclass__ = models.SubfieldBase description = _("E-mail address") def get_internal_type(self):