From 5c03162f3f56ee8cd28b47c6498f8ff75cfc9feb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ho=C5=99ej=C5=A1ek?= Date: Thu, 19 Oct 2017 13:59:51 +0200 Subject: [PATCH 001/201] Update README.markdown --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 46b367b..3193721 100644 --- a/README.markdown +++ b/README.markdown @@ -8,4 +8,4 @@ Support for Python 3.3 and higher. ## Documentation -Documentation: http://opensource.seznam.cz/python-fastjsonschema/ +Documentation: https://seznam.github.io/python-fastjsonschema/ From 04f140bf58317d423bb253b9e73a08190e5c4aed Mon Sep 17 00:00:00 2001 From: anentropic Date: Tue, 31 Oct 2017 11:14:01 +0000 Subject: [PATCH 002/201] fix pip install documentation --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index c47e933..884c322 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,7 +6,7 @@ Installation .. code-block:: bash - pip install python-fastjsonschema + pip install fastjsonschema Documentation ************* From af6d7d730353f81f74c6136303a1a506d9fad0d6 Mon Sep 17 00:00:00 2001 From: Kris Date: Mon, 12 Mar 2018 10:19:04 -0400 Subject: [PATCH 003/201] Add test suite submodule --- .gitmodules | 3 +++ JSON-Schema-Test-Suite | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 JSON-Schema-Test-Suite diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..017ca86 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "JSON-Schema-Test-Suite"] + path = JSON-Schema-Test-Suite + url = https://github.com/json-schema-org/JSON-Schema-Test-Suite.git diff --git a/JSON-Schema-Test-Suite b/JSON-Schema-Test-Suite new file mode 160000 index 0000000..71843cb --- /dev/null +++ b/JSON-Schema-Test-Suite @@ -0,0 +1 @@ +Subproject commit 71843cbbd4be3194ee83253b402656550c8c0522 From 04b26b54e627b38d3ad4005a6585314c247d0ffa Mon Sep 17 00:00:00 2001 From: Kris Date: Mon, 12 Mar 2018 14:58:07 -0400 Subject: [PATCH 004/201] Add test runner script --- json-schema-test-suite-draft-4.conf | 16 +++ json_schema_test_suite.py | 180 ++++++++++++++++++++++++++++ requirements-to-freeze.txt | 1 + requirements.txt | 1 + 4 files changed, 198 insertions(+) create mode 100644 json-schema-test-suite-draft-4.conf create mode 100755 json_schema_test_suite.py create mode 100644 requirements-to-freeze.txt create mode 100644 requirements.txt diff --git a/json-schema-test-suite-draft-4.conf b/json-schema-test-suite-draft-4.conf new file mode 100644 index 0000000..6fd9568 --- /dev/null +++ b/json-schema-test-suite-draft-4.conf @@ -0,0 +1,16 @@ +[suite] +dir = JSON-Schema-Test-Suite/tests/draft4 + +[ignore] +# File paths relative to suite dir +# Tests in these files will be run but the results will be ignored +files = + definitions.json + dependencies.json + optional/bignum.json + optional/ecmascript-regex.json + optional/format.json + optional/zeroTerminatedFloats.json + ref.json + refRemote.json + uniqueItems.json diff --git a/json_schema_test_suite.py b/json_schema_test_suite.py new file mode 100755 index 0000000..7dfef5a --- /dev/null +++ b/json_schema_test_suite.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python -u +# coding: utf-8 + +"""Run a JSON Schema test suite and print results.""" + +from collections import Counter, defaultdict, namedtuple +from enum import Enum, auto +from pathlib import Path +from textwrap import dedent +import argparse +import configparser +import json +import sys + +from colorama import Fore +import fastjsonschema + +class TestResult(Enum): + FALSE_POSITIVE = auto() + TRUE_POSITIVE = auto() + FALSE_NEGATIVE = auto() + TRUE_NEGATIVE = auto() + UNDEFINED = auto() + IGNORED = auto() + +Test = namedtuple("Test", "description exception result ignore") + + +def _get_parser(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + p.add_argument("--strict", help="Do not ignore test files, even if configured to do so", action="store_false") + p.add_argument("--verbose", help="Print all test results", action="store_true") + p.add_argument("path", help="Path to either a configuration file or a single JSON test file", type=Path) + return p + + +def _main(): + ignore_file_paths = set() + + config = configparser.ConfigParser() + try: + config.read(str(args.path)) + suite_dir_path = Path(config.get("suite", "dir")).resolve() + test_file_paths = sorted(set(suite_dir_path.glob("**/*.json"))) + if args.strict: + for l in config.get("ignore", "files").splitlines(): + p = Path(suite_dir_path / l).resolve() + if p.is_file(): + ignore_file_paths.add(p) + except configparser.MissingSectionHeaderError: + test_file_paths = {args.path.resolve()} + + tests = defaultdict(dict) + test_results = Counter() + + schema_exceptions = {} + for test_file_path in test_file_paths: + with test_file_path.open() as f: + tests[test_file_path.name] = defaultdict(dict) + ignore = True if test_file_path in ignore_file_paths else False + test_data = json.load(f) + for test_case in test_data: + test_case_description = test_case["description"] + schema = test_case["schema"] + tests[test_file_path.name][test_case_description] = [] + try: + validate = fastjsonschema.compile(schema) + except Exception as e: + schema_exceptions[test_file_path] = e + for test in test_case["tests"]: + description = test["description"] + data = test["data"] + result = exception = None + try: + if test["valid"]: + try: + validate(data) + result = TestResult.TRUE_POSITIVE + except fastjsonschema.exceptions.JsonSchemaException as e: + result = TestResult.FALSE_NEGATIVE + exception = e + else: + try: + validate(data) + result = TestResult.FALSE_POSITIVE + except fastjsonschema.exceptions.JsonSchemaException as e: + result = TestResult.TRUE_NEGATIVE + exception = e + except Exception as e: + result = TestResult.UNDEFINED + exception = e + tests[test_file_path.name][test_case_description].append(Test(description, exception, result, ignore)) + test_results.update({TestResult.IGNORED if ignore else result: 1}) + + for file_name, test_cases in sorted(tests.items()): + for test_case in test_cases.values(): + if any(t for t in test_case if t.ignore): + print(Fore.MAGENTA + "⛔" + Fore.RESET, file_name) + break + else: + if any(t for t in test_case if t.result in (TestResult.FALSE_POSITIVE, TestResult.FALSE_NEGATIVE)): + print(Fore.RED + "✘" + Fore.RESET, file_name) + break + elif any(t for t in test_case if t.result == TestResult.UNDEFINED): + print(Fore.YELLOW + "⚠" + Fore.RESET, file_name) + break + else: + print(Fore.GREEN + "✔" + Fore.RESET, file_name) + break + for test_case_description, test_case in test_cases.items(): + if not any(t for t in test_case if t.ignore): + if any(t for t in test_case if t.result in (TestResult.FALSE_POSITIVE, TestResult.FALSE_NEGATIVE)): + print(" " + Fore.RED + "✘" + Fore.RESET, test_case_description) + elif any(t for t in test_case if t.result == TestResult.UNDEFINED): + print(" " + Fore.YELLOW + "⚠" + Fore.RESET, test_case_description) + elif args.verbose: + print(" " + Fore.GREEN + "✔" + Fore.RESET, test_case_description) + for test in test_case: + if test.result in (TestResult.FALSE_POSITIVE, TestResult.FALSE_NEGATIVE): + print(" " + Fore.RED + "✘" + Fore.RESET, + Fore.CYAN + test.result.name + Fore.RESET, + Fore.RED + type(test.exception).__name__ + Fore.RESET, + "{}: {}".format(test.description, test.exception)) + elif test.result == TestResult.UNDEFINED: + print(" " + Fore.YELLOW + "⚠" + Fore.RESET, + Fore.CYAN + test.result.name + Fore.RESET, + Fore.YELLOW + type(test.exception).__name__ + Fore.RESET, + "{}: {}".format(test.description, test.exception)) + elif args.verbose: + print(" " + Fore.GREEN + "✔" + Fore.RESET, + Fore.CYAN + test.result.name + Fore.RESET, + test.description) + + if schema_exceptions: + print("\nSchema exceptions:\n") + for file_path, exception in sorted(schema_exceptions.items()): + if file_path in ignore_file_paths: + print(Fore.MAGENTA + "⛔" + Fore.RESET, end=" ") + else: + print(Fore.RED + "✘" + Fore.RESET, end=" ") + try: + print("{}: {}: '{}'".format(file_path.name, exception, exception.text.strip())) + except AttributeError: + print("{}: {}".format(file_path.name, exception)) + + total = sum(test_results.values()) + sub_total = total - test_results[TestResult.IGNORED] + total_failures = total_passes = 0 + print("\nSummary of {} tests:\n".format(total)) + + print("Failures:\n") + for result in (TestResult.FALSE_POSITIVE, TestResult.FALSE_NEGATIVE, TestResult.UNDEFINED): + total_failures += test_results[result] + if result == TestResult.UNDEFINED: + print(Fore.YELLOW + "⚠", end=" ") + else: + print(Fore.RED + "✘", end=" ") + print(Fore.CYAN + "{:<14}".format(result.name) + Fore.RESET, + "{:>4} {:>6.1%}".format(test_results[result], test_results[result] / sub_total)) + print(" {:>4} {:>6.1%}".format(total_failures, total_failures / sub_total)) + + print("\nPasses:\n") + for result in (TestResult.TRUE_POSITIVE, TestResult.TRUE_NEGATIVE): + total_passes += test_results[result] + print(Fore.GREEN + "✔", + Fore.CYAN + "{:<14}".format(result.name) + Fore.RESET, + "{:>4} {:>6.1%}".format(test_results[result], test_results[result] / sub_total)) + print(" {:>4} {:6.1%}".format(total_passes, total_passes / sub_total)) + + print("\n" + Fore.MAGENTA + "⛔" + Fore.RESET, + "Ignored: {:>10}".format(test_results[TestResult.IGNORED])) + print("Coverage: {:>7}/{} {:>6.1%}".format(total_failures + total_passes, total, + (total_failures + total_passes) / total)) + + return total_failures > 0 + + +if __name__ == "__main__": + args = _get_parser().parse_args() + sys.exit(_main()) diff --git a/requirements-to-freeze.txt b/requirements-to-freeze.txt new file mode 100644 index 0000000..3fcfb51 --- /dev/null +++ b/requirements-to-freeze.txt @@ -0,0 +1 @@ +colorama diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..fdfb25f --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +colorama==0.3.9 From 0ad15895e0959691d2b2aba3770ce085f55af370 Mon Sep 17 00:00:00 2001 From: Kris Date: Tue, 13 Mar 2018 15:44:10 -0400 Subject: [PATCH 005/201] Update test requirements, install target --- Makefile | 2 +- requirements-to-freeze.txt | 1 - requirements.txt | 1 - setup.py | 10 ++++++++++ 4 files changed, 11 insertions(+), 3 deletions(-) delete mode 100644 requirements-to-freeze.txt delete mode 100644 requirements.txt diff --git a/Makefile b/Makefile index f65acca..26841eb 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ upload: python3 setup.py register sdist upload install: - python3 setup.py install + pip install --editable .[test] test: python3 -m pytest tests diff --git a/requirements-to-freeze.txt b/requirements-to-freeze.txt deleted file mode 100644 index 3fcfb51..0000000 --- a/requirements-to-freeze.txt +++ /dev/null @@ -1 +0,0 @@ -colorama diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index fdfb25f..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -colorama==0.3.9 diff --git a/setup.py b/setup.py index d24a7b6..95cea7b 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,16 @@ version='1.1', packages=['fastjsonschema'], + extras_require={ + "test": [ + "colorama", + "jsonschema", + "json-spec", + "pytest", + "validictory", + ], + }, + url='https://github.com/seznam/python-fastjsonschema', author='Michal Horejsek', author_email='horejsekmichal@gmail.com', From 28e96f454e833166ff651b13330f49a3f606aa32 Mon Sep 17 00:00:00 2001 From: Kris Date: Wed, 14 Mar 2018 16:58:34 -0400 Subject: [PATCH 006/201] Fix performance script --- performance.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/performance.py b/performance.py index c72e3b4..beaed05 100644 --- a/performance.py +++ b/performance.py @@ -1,4 +1,4 @@ - +from textwrap import dedent import timeit # apt-get install jsonschema json-spec validictory @@ -93,18 +93,18 @@ def t(func, valid_values=True): """ if valid_values: - code = """ + code = dedent(""" for value in VALUES_OK: {}(value, JSON_SCHEMA) - """.format(func) + """.format(func)) else: - code = """ + code = dedent(""" try: for value in VALUES_BAD: {}(value, JSON_SCHEMA) except: pass - """.format(func) + """.format(func)) res = timeit.timeit(code, setup, number=NUMBER) print('{:<20} {:<10} ==> {}'.format(module, 'valid' if valid_values else 'invalid', res)) From cb650e28115baeafcd2cb0ce963b1175bd34c6e2 Mon Sep 17 00:00:00 2001 From: Kris Date: Tue, 13 Mar 2018 16:00:12 -0400 Subject: [PATCH 007/201] Fix typo --- fastjsonschema/generator.py | 5 +++-- tests/test_array.py | 4 ++-- tests/test_object.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 585e685..2be99c6 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -358,7 +358,7 @@ def generate_items(self): if 'additionalItems' in self._definition: if self._definition['additionalItems'] is False: - self.l('if {variable}_len > {}: raise JsonSchemaException("{name} must contain only spcified items")', len(self._definition['items'])) + self.l('if {variable}_len > {}: raise JsonSchemaException("{name} must contain only specified items")', len(self._definition['items'])) else: with self.l('for {variable}_x, {variable}_item in enumerate({variable}[{0}:], {0}):', len(self._definition['items'])): self.generate_func_code_block( @@ -404,8 +404,9 @@ def generate_properties(self): self.l('else: {variable}["{}"] = {}', key, repr(prop_definition['default'])) if 'additionalProperties' in self._definition: + # import pdb; pdb.set_trace() if self._definition['additionalProperties'] is False: - self.l('if {variable}_keys: raise JsonSchemaException("{name} must contain only spcified properties")') + self.l('if {variable}_keys: raise JsonSchemaException("{name} must contain only specified properties")') else: with self.l('for {variable}_key in {variable}_keys:'): self.l('{variable}_value = {variable}.get({variable}_key)') diff --git a/tests/test_array.py b/tests/test_array.py index 9448576..b4934ac 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -114,8 +114,8 @@ def test_different_items_with_additional_items(asserter, value, expected): ([1], [1]), ([1, 'a'], [1, 'a']), ([1, 2], JsonSchemaException('data[1] must be string')), - ([1, 'a', 2], JsonSchemaException('data must contain only spcified items')), - ([1, 'a', 'b'], JsonSchemaException('data must contain only spcified items')), + ([1, 'a', 2], JsonSchemaException('data must contain only specified items')), + ([1, 'a', 'b'], JsonSchemaException('data must contain only specified items')), ]) def test_different_items_without_additional_items(asserter, value, expected): asserter({ diff --git a/tests/test_object.py b/tests/test_object.py index 493abc5..0a686a9 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -97,7 +97,7 @@ def test_properties_with_additional_properties(asserter, value, expected): ({'a': 1}, {'a': 1}), ({'a': 1, 'b': ''}, {'a': 1, 'b': ''}), ({'a': 1, 'b': 2}, JsonSchemaException('data.b must be string')), - ({'a': 1, 'b': '', 'any': True}, JsonSchemaException('data must contain only spcified properties')), + ({'a': 1, 'b': '', 'any': True}, JsonSchemaException('data must contain only specified properties')), ]) def test_properties_without_additional_properties(asserter, value, expected): asserter({ From a8ba7c4ae6c0809292f65a0c197452e5c8152ee7 Mon Sep 17 00:00:00 2001 From: Kris Date: Tue, 13 Mar 2018 15:35:17 -0400 Subject: [PATCH 008/201] Validate minimum and maximum for non-numbers --- fastjsonschema/generator.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 2be99c6..9c571f0 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -294,6 +294,10 @@ def generate_pattern(self): self.l('raise JsonSchemaException("{name} must match pattern {pattern}")') def generate_minimum(self): + with self.l('try:'): + self.l('{variable} = float({variable})') + with self.l('except ValueError:'): + self.l('return {variable}') if self._definition.get('exclusiveMinimum', False): with self.l('if {variable} <= {minimum}:'): self.l('raise JsonSchemaException("{name} must be bigger than {minimum}")') @@ -302,6 +306,10 @@ def generate_minimum(self): self.l('raise JsonSchemaException("{name} must be bigger than or equal to {minimum}")') def generate_maximum(self): + with self.l('try:'): + self.l('{variable} = float({variable})') + with self.l('except ValueError:'): + self.l('return {variable}') if self._definition.get('exclusiveMaximum', False): with self.l('if {variable} >= {maximum}:'): self.l('raise JsonSchemaException("{name} must be smaller than {maximum}")') From 93f23a3aab81e416b0e281bd7eeb33cabc5fc783 Mon Sep 17 00:00:00 2001 From: Kris Date: Wed, 14 Mar 2018 07:07:47 -0400 Subject: [PATCH 009/201] Validate additional properties --- fastjsonschema/generator.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 9c571f0..bcec603 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -74,6 +74,7 @@ def __init__(self, definition): ('maxProperties', self.generate_max_properties), ('required', self.generate_required), ('properties', self.generate_properties), + ('additionalProperties', self.generate_additional_properties), )) self.generate_func_code(definition) @@ -398,6 +399,10 @@ def generate_required(self): self.l('raise JsonSchemaException("{name} must contain {required} properties")') def generate_properties(self): + with self.l('try:'): + self.l('{variable}.keys()') + with self.l('except AttributeError:'): + self.l('return {variable}') self.l('{variable}_keys = set({variable}.keys())') for key, prop_definition in self._definition['properties'].items(): with self.l('if "{}" in {variable}_keys:', key): @@ -412,7 +417,6 @@ def generate_properties(self): self.l('else: {variable}["{}"] = {}', key, repr(prop_definition['default'])) if 'additionalProperties' in self._definition: - # import pdb; pdb.set_trace() if self._definition['additionalProperties'] is False: self.l('if {variable}_keys: raise JsonSchemaException("{name} must contain only specified properties")') else: @@ -423,3 +427,26 @@ def generate_properties(self): '{}_value'.format(self._variable), '{}.{{{}_key}}'.format(self._variable_name, self._variable), ) + + def generate_additional_properties(self): + with self.l('try:'): + self.l('{variable}.keys()') + with self.l('except AttributeError:'): + self.l('return {variable}') + self.l('{variable}_keys = set({variable}.keys())') + add_prop_definition = self._definition["additionalProperties"] + if add_prop_definition is False: + with self.l('for key in {variable}_keys:'): + with self.l('if key not in "{}":', self._definition['properties']): + self.l('raise JsonSchemaException("{name} may not contain additional properties")') + else: + with self.l('for {variable}_key in {variable}_keys:'): + with self.l('if {variable}_key not in "{}":', self._definition.get('properties', [])): + self.l('{variable}_value = {variable}.get({variable}_key)') + self.generate_func_code_block( + add_prop_definition, + '{}_value'.format(self._variable), + '{}.{{{}_key}}'.format(self._variable_name, self._variable), + ) + if 'default' in add_prop_definition: + self.l('else: {variable}["{}"] = {}', key, repr(add_prop_definition['default'])) From 494b6ce8f95bfefc48e34d7bbf2684c9df66332c Mon Sep 17 00:00:00 2001 From: Kris Date: Wed, 14 Mar 2018 09:41:57 -0400 Subject: [PATCH 010/201] Validate additional items --- fastjsonschema/generator.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index bcec603..3269715 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -376,12 +376,13 @@ def generate_items(self): '{}[{{{}_x}}]'.format(self._variable_name, self._variable), ) else: - with self.l('for {variable}_x, {variable}_item in enumerate({variable}):'): - self.generate_func_code_block( - self._definition['items'], - '{}_item'.format(self._variable), - '{}[{{{}_x}}]'.format(self._variable_name, self._variable), - ) + if self._definition['items']: + with self.l('for {variable}_x, {variable}_item in enumerate({variable}):'): + self.generate_func_code_block( + self._definition['items'], + '{}_item'.format(self._variable), + '{}[{{{}_x}}]'.format(self._variable_name, self._variable), + ) def generate_min_properties(self): self.create_variable_with_length() From fe82f6e1807579b736a6d3cb8ef7fe8166dcbe6e Mon Sep 17 00:00:00 2001 From: Kris Date: Wed, 14 Mar 2018 12:52:23 -0400 Subject: [PATCH 011/201] Validate not --- fastjsonschema/generator.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 3269715..7e2bf5f 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -52,6 +52,7 @@ def __init__(self, definition): self._variable = None self._variable_name = None self._definition = None + self._context = {} self._json_keywords_to_function = OrderedDict(( ('type', self.generate_type), @@ -156,18 +157,18 @@ def generate_func_code(self, definition): self.generate_func_code_block(definition, 'data', 'data') self.l('return data') - def generate_func_code_block(self, definition, variable, variable_name): + def generate_func_code_block(self, definition, variable, variable_name, **kwargs): """ Creates validation rules for current definition. """ - backup = self._definition, self._variable, self._variable_name - self._definition, self._variable, self._variable_name = definition, variable, variable_name + backup = self._definition, self._variable, self._variable_name, self._context + self._definition, self._variable, self._variable_name, self._context = definition, variable, variable_name, kwargs.get('context', {}) for key, func in self._json_keywords_to_function.items(): if key in definition: func() - self._definition, self._variable, self._variable_name = backup + self._definition, self._variable, self._variable_name, self._context = backup def generate_type(self): """ @@ -272,12 +273,16 @@ def generate_not(self): {'not': {'type': 'null'}} - Valid values for this definitions are 'hello', 42, ... but not None. + Valid values for this definitions are 'hello', 42, {} ... but not None. """ - with self.l('try:'): - self.generate_func_code_block(self._definition['not'], self._variable, self._variable_name) - self.l('except JsonSchemaException: pass') - self.l('else: raise JsonSchemaException("{name} must not be valid by not definition")') + if self._definition['not'] is not None and not self._definition['not']: # {} + with self.l('if "{}" in {}.keys():', self._context['key'], self._context['definition'].get('properties', {})): + self.l('raise JsonSchemaException("{name} must not be valid by not definition")') + else: + with self.l('try:'): + self.generate_func_code_block(self._definition['not'], self._variable, self._variable_name) + self.l('except JsonSchemaException: pass') + self.l('else: raise JsonSchemaException("{name} must not be valid by not definition")') def generate_min_length(self): self.create_variable_with_length() @@ -413,6 +418,10 @@ def generate_properties(self): prop_definition, '{}_{}'.format(self._variable, key), '{}.{}'.format(self._variable_name, key), + context={ + "definition": self._definition, + "key": key + }, ) if 'default' in prop_definition: self.l('else: {variable}["{}"] = {}', key, repr(prop_definition['default'])) From d3648dfc93b0d78adaa275c3a3acd19ff56bff8a Mon Sep 17 00:00:00 2001 From: Kris Date: Wed, 14 Mar 2018 13:38:03 -0400 Subject: [PATCH 012/201] Validate items --- fastjsonschema/generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 7e2bf5f..0656c3c 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -357,6 +357,8 @@ def generate_unique_items(self): self.l('raise JsonSchemaException("{name} must contain unique items")') def generate_items(self): + with self.l('if isinstance({variable}, dict):'): + self.l('return {variable}') self.create_variable_with_length() if isinstance(self._definition['items'], list): for x, item_definition in enumerate(self._definition['items']): From f1e353c7227dcd59558d475e7fac10b76343d8fe Mon Sep 17 00:00:00 2001 From: Kris Date: Wed, 14 Mar 2018 13:41:54 -0400 Subject: [PATCH 013/201] Validate max items --- fastjsonschema/generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 0656c3c..c41c82f 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -333,6 +333,8 @@ def generate_min_items(self): self.l('raise JsonSchemaException("{name} must contain at least {minItems} items")') def generate_max_items(self): + with self.l('if not isinstance({variable}, list):'): + self.l('return {variable}') self.create_variable_with_length() with self.l('if {variable}_len > {maxItems}:'): self.l('raise JsonSchemaException("{name} must contain less than or equal to {maxItems} items")') From 3419f30504b242fdc04674f837268681c71727b1 Mon Sep 17 00:00:00 2001 From: Kris Date: Wed, 14 Mar 2018 13:43:10 -0400 Subject: [PATCH 014/201] Validate max length --- fastjsonschema/generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index c41c82f..dadc972 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -290,6 +290,8 @@ def generate_min_length(self): self.l('raise JsonSchemaException("{name} must be longer than or equal to {minLength} characters")') def generate_max_length(self): + with self.l('if not isinstance({variable}, str):'): + self.l('return {variable}') self.create_variable_with_length() with self.l('if {variable}_len > {maxLength}:'): self.l('raise JsonSchemaException("{name} must be shorter than or equal to {maxLength} characters")') From 306914b57430bae9456bb1e0309b5fca5d9c744f Mon Sep 17 00:00:00 2001 From: Kris Date: Wed, 14 Mar 2018 13:56:44 -0400 Subject: [PATCH 015/201] Validate max properties --- fastjsonschema/generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index dadc972..e5c855b 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -401,6 +401,8 @@ def generate_min_properties(self): self.l('raise JsonSchemaException("{name} must contain at least {minProperties} properties")') def generate_max_properties(self): + with self.l('if not isinstance({variable}, dict):'): + self.l('return {variable}') self.create_variable_with_length() with self.l('if {variable}_len > {maxProperties}:'): self.l('raise JsonSchemaException("{name} must contain less than or equal to {maxProperties} properties")') From 11f72ce8270106ad25faf800554edeef81c01989 Mon Sep 17 00:00:00 2001 From: Kris Date: Wed, 14 Mar 2018 13:57:40 -0400 Subject: [PATCH 016/201] Validate min items --- fastjsonschema/generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index e5c855b..8027130 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -330,6 +330,8 @@ def generate_multiple_of(self): self.l('raise JsonSchemaException("{name} must be multiple of {multipleOf}")') def generate_min_items(self): + with self.l('if not isinstance({variable}, list):'): + self.l('return {variable}') self.create_variable_with_length() with self.l('if {variable}_len < {minItems}:'): self.l('raise JsonSchemaException("{name} must contain at least {minItems} items")') From 3493b38ee6d9cebac4c502192aed9d9e753289b9 Mon Sep 17 00:00:00 2001 From: Kris Date: Wed, 14 Mar 2018 14:09:47 -0400 Subject: [PATCH 017/201] Validate min length --- fastjsonschema/generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 8027130..2885087 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -285,6 +285,8 @@ def generate_not(self): self.l('else: raise JsonSchemaException("{name} must not be valid by not definition")') def generate_min_length(self): + with self.l('if not isinstance({variable}, str):'): + self.l('return {variable}') self.create_variable_with_length() with self.l('if {variable}_len < {minLength}:'): self.l('raise JsonSchemaException("{name} must be longer than or equal to {minLength} characters")') From a4d68a8d88c7ea5d51e0cfb0b09496605a9dfcde Mon Sep 17 00:00:00 2001 From: Kris Date: Wed, 14 Mar 2018 14:27:41 -0400 Subject: [PATCH 018/201] Validate min properties --- fastjsonschema/generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 2885087..11832cf 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -400,6 +400,8 @@ def generate_items(self): ) def generate_min_properties(self): + with self.l('if not isinstance({variable}, dict):'): + self.l('return {variable}') self.create_variable_with_length() with self.l('if {variable}_len < {minProperties}:'): self.l('raise JsonSchemaException("{name} must contain at least {minProperties} properties")') From 70679469d6b2ed00f7b043262652335d5cb2a04d Mon Sep 17 00:00:00 2001 From: Kris Date: Wed, 14 Mar 2018 15:07:38 -0400 Subject: [PATCH 019/201] Validate multiple of --- fastjsonschema/generator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 11832cf..03a5b3b 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -328,7 +328,10 @@ def generate_maximum(self): self.l('raise JsonSchemaException("{name} must be smaller than or equal to {maximum}")') def generate_multiple_of(self): - with self.l('if {variable} % {multipleOf} != 0:'): + with self.l('if not isinstance({variable}, (int, float)):'): + self.l('return {variable}') + self.l('quotient = {variable} / {multipleOf}') + with self.l('if int(quotient) != quotient:'): self.l('raise JsonSchemaException("{name} must be multiple of {multipleOf}")') def generate_min_items(self): From 81eb5756dc8f4d1ef8498a17a0cd5488e5a145ec Mon Sep 17 00:00:00 2001 From: Kris Date: Wed, 14 Mar 2018 15:28:54 -0400 Subject: [PATCH 020/201] Validate required --- fastjsonschema/generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 03a5b3b..85f9e61 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -417,6 +417,8 @@ def generate_max_properties(self): self.l('raise JsonSchemaException("{name} must contain less than or equal to {maxProperties} properties")') def generate_required(self): + with self.l('if not isinstance({variable}, dict):'): + self.l('return {variable}') self.create_variable_with_length() with self.l('if not all(prop in {variable} for prop in {required}):'): self.l('raise JsonSchemaException("{name} must contain {required} properties")') From 668f8244d5e2ce1be12b405ffb81a4ee52dbb9d3 Mon Sep 17 00:00:00 2001 From: Kris Date: Wed, 14 Mar 2018 16:07:22 -0400 Subject: [PATCH 021/201] Validate pattern --- fastjsonschema/generator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 85f9e61..f49bf56 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -299,8 +299,10 @@ def generate_max_length(self): self.l('raise JsonSchemaException("{name} must be shorter than or equal to {maxLength} characters")') def generate_pattern(self): + with self.l('if not isinstance({variable}, str):'): + self.l('return {variable}') self._compile_regexps['{}_re'.format(self._variable)] = re.compile(self._definition['pattern']) - with self.l('if not {variable}_re.match({variable}):'): + with self.l('if not {variable}_re.search({variable}):'): self.l('raise JsonSchemaException("{name} must match pattern {pattern}")') def generate_minimum(self): From ae98690cd9f8ab54d836bd955888ca80f2986d21 Mon Sep 17 00:00:00 2001 From: Kris Date: Thu, 15 Mar 2018 11:14:39 -0400 Subject: [PATCH 022/201] Validate pattern properties --- fastjsonschema/generator.py | 46 +++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index f49bf56..4160306 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -75,6 +75,7 @@ def __init__(self, definition): ('maxProperties', self.generate_max_properties), ('required', self.generate_required), ('properties', self.generate_properties), + ('patternProperties', self.generate_pattern_properties), ('additionalProperties', self.generate_additional_properties), )) @@ -447,6 +448,19 @@ def generate_properties(self): if 'default' in prop_definition: self.l('else: {variable}["{}"] = {}', key, repr(prop_definition['default'])) + if 'patternProperties' in self._definition: + self.generate_func_code_block( + {'patternProperties': self._definition['patternProperties']}, + self._variable, + '{}.{}'.format(self._variable_name, 'patternProperties'), + ) + self.l('pattern_keys = set()') + with self.l('for key in {variable}_keys:'): + for pattern in self._definition['patternProperties'].keys(): + with self.l('if globals()["{}_re"].search(key):', pattern): + self.l('pattern_keys.add(key)') + self.l('{variable}_keys -= pattern_keys') + if 'additionalProperties' in self._definition: if self._definition['additionalProperties'] is False: self.l('if {variable}_keys: raise JsonSchemaException("{name} must contain only specified properties")') @@ -465,12 +479,17 @@ def generate_additional_properties(self): with self.l('except AttributeError:'): self.l('return {variable}') self.l('{variable}_keys = set({variable}.keys())') - add_prop_definition = self._definition["additionalProperties"] - if add_prop_definition is False: + + if 'patternProperties' in self._definition: + self.l('pattern_keys = set()') with self.l('for key in {variable}_keys:'): - with self.l('if key not in "{}":', self._definition['properties']): - self.l('raise JsonSchemaException("{name} may not contain additional properties")') - else: + for pattern in self._definition['patternProperties'].keys(): + with self.l('if globals()["{}_re"].search(key):', pattern): + self.l('pattern_keys.add(key)') + self.l('{variable}_keys -= pattern_keys') + + add_prop_definition = self._definition["additionalProperties"] + if add_prop_definition: with self.l('for {variable}_key in {variable}_keys:'): with self.l('if {variable}_key not in "{}":', self._definition.get('properties', [])): self.l('{variable}_value = {variable}.get({variable}_key)') @@ -481,3 +500,20 @@ def generate_additional_properties(self): ) if 'default' in add_prop_definition: self.l('else: {variable}["{}"] = {}', key, repr(add_prop_definition['default'])) + + def generate_pattern_properties(self): + with self.l('if not isinstance({variable}, dict):'): + self.l('return {variable}') + for pattern, definition in self._definition['patternProperties'].items(): + self._compile_regexps['{}_re'.format(pattern)] = re.compile(pattern) + with self.l('for key, val in {variable}.items():'): + for pattern, definition in self._definition['patternProperties'].items(): + if not definition: + self.l('pass') + else: + with self.l('if globals()["{}_re"].search(key):', pattern): + self.generate_func_code_block( + definition, + 'val', + '{}.{{key}}'.format(self._variable_name), + ) From 0e7132386e2d90ebe4f67db0ccf68388ea65cef9 Mon Sep 17 00:00:00 2001 From: Kris Date: Sat, 17 Mar 2018 07:24:02 -0400 Subject: [PATCH 023/201] Test instance type not attribute --- fastjsonschema/generator.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 4160306..e5b8caa 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -427,9 +427,7 @@ def generate_required(self): self.l('raise JsonSchemaException("{name} must contain {required} properties")') def generate_properties(self): - with self.l('try:'): - self.l('{variable}.keys()') - with self.l('except AttributeError:'): + with self.l('if not isinstance({variable}, dict):'): self.l('return {variable}') self.l('{variable}_keys = set({variable}.keys())') for key, prop_definition in self._definition['properties'].items(): @@ -474,9 +472,7 @@ def generate_properties(self): ) def generate_additional_properties(self): - with self.l('try:'): - self.l('{variable}.keys()') - with self.l('except AttributeError:'): + with self.l('if not isinstance({variable}, dict):'): self.l('return {variable}') self.l('{variable}_keys = set({variable}.keys())') From 83af56a754807378ae0464f5922dc1c82e487048 Mon Sep 17 00:00:00 2001 From: Kris Date: Tue, 20 Mar 2018 09:38:37 -0400 Subject: [PATCH 024/201] Simplify additional properties logic --- fastjsonschema/generator.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index e5b8caa..33916e5 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -477,12 +477,7 @@ def generate_additional_properties(self): self.l('{variable}_keys = set({variable}.keys())') if 'patternProperties' in self._definition: - self.l('pattern_keys = set()') - with self.l('for key in {variable}_keys:'): - for pattern in self._definition['patternProperties'].keys(): - with self.l('if globals()["{}_re"].search(key):', pattern): - self.l('pattern_keys.add(key)') - self.l('{variable}_keys -= pattern_keys') + self.l('return {variable}') add_prop_definition = self._definition["additionalProperties"] if add_prop_definition: From 8015d6ac4f7392673805d481dce6770504641f35 Mon Sep 17 00:00:00 2001 From: Kris Date: Tue, 20 Mar 2018 10:12:20 -0400 Subject: [PATCH 025/201] Fix properties keys check --- fastjsonschema/generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 33916e5..f0ef124 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -481,8 +481,9 @@ def generate_additional_properties(self): add_prop_definition = self._definition["additionalProperties"] if add_prop_definition: + properties_keys = self._definition.get("properties", {}).keys() with self.l('for {variable}_key in {variable}_keys:'): - with self.l('if {variable}_key not in "{}":', self._definition.get('properties', [])): + with self.l('if {variable}_key not in "{}":', properties_keys): self.l('{variable}_value = {variable}.get({variable}_key)') self.generate_func_code_block( add_prop_definition, From a0ae2672ae137ae37c23ad5043fb9fcf64c85a3f Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 27 Mar 2018 14:42:05 +0200 Subject: [PATCH 026/201] Pytest JSON schema test suits --- json-schema-test-suite-draft-4.conf | 16 --- json_schema_test_suite.py | 180 --------------------------- tests/test_json_schema_test_suits.py | 61 +++++++++ 3 files changed, 61 insertions(+), 196 deletions(-) delete mode 100644 json-schema-test-suite-draft-4.conf delete mode 100755 json_schema_test_suite.py create mode 100644 tests/test_json_schema_test_suits.py diff --git a/json-schema-test-suite-draft-4.conf b/json-schema-test-suite-draft-4.conf deleted file mode 100644 index 6fd9568..0000000 --- a/json-schema-test-suite-draft-4.conf +++ /dev/null @@ -1,16 +0,0 @@ -[suite] -dir = JSON-Schema-Test-Suite/tests/draft4 - -[ignore] -# File paths relative to suite dir -# Tests in these files will be run but the results will be ignored -files = - definitions.json - dependencies.json - optional/bignum.json - optional/ecmascript-regex.json - optional/format.json - optional/zeroTerminatedFloats.json - ref.json - refRemote.json - uniqueItems.json diff --git a/json_schema_test_suite.py b/json_schema_test_suite.py deleted file mode 100755 index 7dfef5a..0000000 --- a/json_schema_test_suite.py +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env python -u -# coding: utf-8 - -"""Run a JSON Schema test suite and print results.""" - -from collections import Counter, defaultdict, namedtuple -from enum import Enum, auto -from pathlib import Path -from textwrap import dedent -import argparse -import configparser -import json -import sys - -from colorama import Fore -import fastjsonschema - -class TestResult(Enum): - FALSE_POSITIVE = auto() - TRUE_POSITIVE = auto() - FALSE_NEGATIVE = auto() - TRUE_NEGATIVE = auto() - UNDEFINED = auto() - IGNORED = auto() - -Test = namedtuple("Test", "description exception result ignore") - - -def _get_parser(): - p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) - p.add_argument("--strict", help="Do not ignore test files, even if configured to do so", action="store_false") - p.add_argument("--verbose", help="Print all test results", action="store_true") - p.add_argument("path", help="Path to either a configuration file or a single JSON test file", type=Path) - return p - - -def _main(): - ignore_file_paths = set() - - config = configparser.ConfigParser() - try: - config.read(str(args.path)) - suite_dir_path = Path(config.get("suite", "dir")).resolve() - test_file_paths = sorted(set(suite_dir_path.glob("**/*.json"))) - if args.strict: - for l in config.get("ignore", "files").splitlines(): - p = Path(suite_dir_path / l).resolve() - if p.is_file(): - ignore_file_paths.add(p) - except configparser.MissingSectionHeaderError: - test_file_paths = {args.path.resolve()} - - tests = defaultdict(dict) - test_results = Counter() - - schema_exceptions = {} - for test_file_path in test_file_paths: - with test_file_path.open() as f: - tests[test_file_path.name] = defaultdict(dict) - ignore = True if test_file_path in ignore_file_paths else False - test_data = json.load(f) - for test_case in test_data: - test_case_description = test_case["description"] - schema = test_case["schema"] - tests[test_file_path.name][test_case_description] = [] - try: - validate = fastjsonschema.compile(schema) - except Exception as e: - schema_exceptions[test_file_path] = e - for test in test_case["tests"]: - description = test["description"] - data = test["data"] - result = exception = None - try: - if test["valid"]: - try: - validate(data) - result = TestResult.TRUE_POSITIVE - except fastjsonschema.exceptions.JsonSchemaException as e: - result = TestResult.FALSE_NEGATIVE - exception = e - else: - try: - validate(data) - result = TestResult.FALSE_POSITIVE - except fastjsonschema.exceptions.JsonSchemaException as e: - result = TestResult.TRUE_NEGATIVE - exception = e - except Exception as e: - result = TestResult.UNDEFINED - exception = e - tests[test_file_path.name][test_case_description].append(Test(description, exception, result, ignore)) - test_results.update({TestResult.IGNORED if ignore else result: 1}) - - for file_name, test_cases in sorted(tests.items()): - for test_case in test_cases.values(): - if any(t for t in test_case if t.ignore): - print(Fore.MAGENTA + "⛔" + Fore.RESET, file_name) - break - else: - if any(t for t in test_case if t.result in (TestResult.FALSE_POSITIVE, TestResult.FALSE_NEGATIVE)): - print(Fore.RED + "✘" + Fore.RESET, file_name) - break - elif any(t for t in test_case if t.result == TestResult.UNDEFINED): - print(Fore.YELLOW + "⚠" + Fore.RESET, file_name) - break - else: - print(Fore.GREEN + "✔" + Fore.RESET, file_name) - break - for test_case_description, test_case in test_cases.items(): - if not any(t for t in test_case if t.ignore): - if any(t for t in test_case if t.result in (TestResult.FALSE_POSITIVE, TestResult.FALSE_NEGATIVE)): - print(" " + Fore.RED + "✘" + Fore.RESET, test_case_description) - elif any(t for t in test_case if t.result == TestResult.UNDEFINED): - print(" " + Fore.YELLOW + "⚠" + Fore.RESET, test_case_description) - elif args.verbose: - print(" " + Fore.GREEN + "✔" + Fore.RESET, test_case_description) - for test in test_case: - if test.result in (TestResult.FALSE_POSITIVE, TestResult.FALSE_NEGATIVE): - print(" " + Fore.RED + "✘" + Fore.RESET, - Fore.CYAN + test.result.name + Fore.RESET, - Fore.RED + type(test.exception).__name__ + Fore.RESET, - "{}: {}".format(test.description, test.exception)) - elif test.result == TestResult.UNDEFINED: - print(" " + Fore.YELLOW + "⚠" + Fore.RESET, - Fore.CYAN + test.result.name + Fore.RESET, - Fore.YELLOW + type(test.exception).__name__ + Fore.RESET, - "{}: {}".format(test.description, test.exception)) - elif args.verbose: - print(" " + Fore.GREEN + "✔" + Fore.RESET, - Fore.CYAN + test.result.name + Fore.RESET, - test.description) - - if schema_exceptions: - print("\nSchema exceptions:\n") - for file_path, exception in sorted(schema_exceptions.items()): - if file_path in ignore_file_paths: - print(Fore.MAGENTA + "⛔" + Fore.RESET, end=" ") - else: - print(Fore.RED + "✘" + Fore.RESET, end=" ") - try: - print("{}: {}: '{}'".format(file_path.name, exception, exception.text.strip())) - except AttributeError: - print("{}: {}".format(file_path.name, exception)) - - total = sum(test_results.values()) - sub_total = total - test_results[TestResult.IGNORED] - total_failures = total_passes = 0 - print("\nSummary of {} tests:\n".format(total)) - - print("Failures:\n") - for result in (TestResult.FALSE_POSITIVE, TestResult.FALSE_NEGATIVE, TestResult.UNDEFINED): - total_failures += test_results[result] - if result == TestResult.UNDEFINED: - print(Fore.YELLOW + "⚠", end=" ") - else: - print(Fore.RED + "✘", end=" ") - print(Fore.CYAN + "{:<14}".format(result.name) + Fore.RESET, - "{:>4} {:>6.1%}".format(test_results[result], test_results[result] / sub_total)) - print(" {:>4} {:>6.1%}".format(total_failures, total_failures / sub_total)) - - print("\nPasses:\n") - for result in (TestResult.TRUE_POSITIVE, TestResult.TRUE_NEGATIVE): - total_passes += test_results[result] - print(Fore.GREEN + "✔", - Fore.CYAN + "{:<14}".format(result.name) + Fore.RESET, - "{:>4} {:>6.1%}".format(test_results[result], test_results[result] / sub_total)) - print(" {:>4} {:6.1%}".format(total_passes, total_passes / sub_total)) - - print("\n" + Fore.MAGENTA + "⛔" + Fore.RESET, - "Ignored: {:>10}".format(test_results[TestResult.IGNORED])) - print("Coverage: {:>7}/{} {:>6.1%}".format(total_failures + total_passes, total, - (total_failures + total_passes) / total)) - - return total_failures > 0 - - -if __name__ == "__main__": - args = _get_parser().parse_args() - sys.exit(_main()) diff --git a/tests/test_json_schema_test_suits.py b/tests/test_json_schema_test_suits.py new file mode 100644 index 0000000..b516c55 --- /dev/null +++ b/tests/test_json_schema_test_suits.py @@ -0,0 +1,61 @@ +import json +from pathlib import Path + +import pytest + +from fastjsonschema import CodeGenerator, JsonSchemaException, compile + + +def pytest_generate_tests(metafunc): + suite_dir = 'JSON-Schema-Test-Suite/tests/draft4' + ignored_suite_files = [ + 'definitions.json', + 'dependencies.json', + 'bignum.json', + 'ecmascript-regex.json', + 'format.json', + 'zeroTerminatedFloats.json', + 'ref.json', + 'refRemote.json', + 'uniqueItems.json', + ] + + suite_dir_path = Path(suite_dir).resolve() + test_file_paths = sorted(set(suite_dir_path.glob("**/*.json"))) + + param_values = [] + param_ids = [] + + for test_file_path in test_file_paths: + with test_file_path.open() as test_file: + test_cases = json.load(test_file) + for test_case in test_cases: + for test_data in test_case['tests']: + param_values.append(pytest.param( + test_case['schema'], + test_data['data'], + test_data['valid'], + marks=pytest.mark.xfail if test_file_path.name in ignored_suite_files else pytest.mark.none, + )) + param_ids.append('{} / {} / {}'.format( + test_file_path.name, + test_case['description'], + test_data['description'], + )) + + metafunc.parametrize(['schema', 'data', 'is_valid'], param_values, ids=param_ids) + + +def test(schema, data, is_valid): + # For debug purposes. When test fails, it will print stdout. + print(CodeGenerator(schema).func_code) + + validate = compile(schema) + try: + validate(data) + except JsonSchemaException: + if is_valid: + raise + else: + if not is_valid: + pytest.fail('Test should not pass') From 0c6652cec5b71180991df6ac70ddaa8e2818beab Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 27 Mar 2018 14:51:09 +0200 Subject: [PATCH 027/201] Fixed check of minimum and maximum --- fastjsonschema/generator.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index f0ef124..8243dc8 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -307,27 +307,29 @@ def generate_pattern(self): self.l('raise JsonSchemaException("{name} must match pattern {pattern}")') def generate_minimum(self): + # We cannot retype original variable. Check for integer could fail then. with self.l('try:'): - self.l('{variable} = float({variable})') + self.l('{variable}_max = float({variable})') with self.l('except ValueError:'): self.l('return {variable}') if self._definition.get('exclusiveMinimum', False): - with self.l('if {variable} <= {minimum}:'): + with self.l('if {variable}_max <= {minimum}:'): self.l('raise JsonSchemaException("{name} must be bigger than {minimum}")') else: - with self.l('if {variable} < {minimum}:'): + with self.l('if {variable}_max < {minimum}:'): self.l('raise JsonSchemaException("{name} must be bigger than or equal to {minimum}")') def generate_maximum(self): + # We cannot retype original variable. Check for integer could fail then. with self.l('try:'): - self.l('{variable} = float({variable})') + self.l('{variable}_max = float({variable})') with self.l('except ValueError:'): self.l('return {variable}') if self._definition.get('exclusiveMaximum', False): - with self.l('if {variable} >= {maximum}:'): + with self.l('if {variable}_max >= {maximum}:'): self.l('raise JsonSchemaException("{name} must be smaller than {maximum}")') else: - with self.l('if {variable} > {maximum}:'): + with self.l('if {variable}_max > {maximum}:'): self.l('raise JsonSchemaException("{name} must be smaller than or equal to {maximum}")') def generate_multiple_of(self): From da1dbc07feb29098fff045dce83975f922cd93df Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 27 Mar 2018 17:28:00 +0200 Subject: [PATCH 028/201] Removed unused code with undeclared variable --- fastjsonschema/generator.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 8243dc8..85d6b93 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -492,8 +492,6 @@ def generate_additional_properties(self): '{}_value'.format(self._variable), '{}.{{{}_key}}'.format(self._variable_name, self._variable), ) - if 'default' in add_prop_definition: - self.l('else: {variable}["{}"] = {}', key, repr(add_prop_definition['default'])) def generate_pattern_properties(self): with self.l('if not isinstance({variable}, dict):'): From c8dad87b3cb350f2a3f56ea325a784a5a14662a2 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 28 Mar 2018 10:27:23 +0200 Subject: [PATCH 029/201] Pattern and additional properties and soon-return fixes --- fastjsonschema/generator.py | 316 ++++++++++++--------------- tests/test_json_schema_test_suits.py | 3 +- 2 files changed, 142 insertions(+), 177 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 85d6b93..662489e 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -52,7 +52,6 @@ def __init__(self, definition): self._variable = None self._variable_name = None self._definition = None - self._context = {} self._json_keywords_to_function = OrderedDict(( ('type', self.generate_type), @@ -136,7 +135,7 @@ def l(self, line, *args, **kwds): def create_variable_with_length(self): """ - In code append code for creating variable with length of that variable + Append code for creating variable with length of that variable (for example length of list or dictionary) with name ``{variable}_len``. It can be called several times and always it's done only when that variable still does not exists. @@ -147,6 +146,17 @@ def create_variable_with_length(self): self._variables.add(variable_name) self.l('{variable}_len = len({variable})') + def create_variable_keys(self): + """ + Append code for creating variable with keys of that variable (dictionary) + with name ``{variable}_len``. It can be called several times and always + it's done only when that variable still does not exists. + """ + variable_name = '{}_keys'.format(self._variable) + if variable_name in self._variables: + return + self._variables.add(variable_name) + self.l('{variable}_keys = set({variable}.keys())') def generate_func_code(self, definition): """ @@ -158,18 +168,23 @@ def generate_func_code(self, definition): self.generate_func_code_block(definition, 'data', 'data') self.l('return data') - def generate_func_code_block(self, definition, variable, variable_name, **kwargs): + def generate_func_code_block(self, definition, variable, variable_name, clear_variables=False): """ Creates validation rules for current definition. """ - backup = self._definition, self._variable, self._variable_name, self._context - self._definition, self._variable, self._variable_name, self._context = definition, variable, variable_name, kwargs.get('context', {}) + backup = self._definition, self._variable, self._variable_name + self._definition, self._variable, self._variable_name = definition, variable, variable_name + if clear_variables: + backup_variables = self._variables + self._variables = set() for key, func in self._json_keywords_to_function.items(): if key in definition: func() - self._definition, self._variable, self._variable_name, self._context = backup + self._definition, self._variable, self._variable_name = backup + if clear_variables: + self._variables = backup_variables def generate_type(self): """ @@ -211,7 +226,7 @@ def generate_all_of(self): Valid values for this definition are 5, 6, 7, ... but not 4 or 'abc' for example. """ for definition_item in self._definition['allOf']: - self.generate_func_code_block(definition_item, self._variable, self._variable_name) + self.generate_func_code_block(definition_item, self._variable, self._variable_name, clear_variables=True) def generate_any_of(self): """ @@ -233,7 +248,7 @@ def generate_any_of(self): for definition_item in self._definition['anyOf']: with self.l('if not {variable}_any_of_count:'): with self.l('try:'): - self.generate_func_code_block(definition_item, self._variable, self._variable_name) + self.generate_func_code_block(definition_item, self._variable, self._variable_name, clear_variables=True) self.l('{variable}_any_of_count += 1') self.l('except JsonSchemaException: pass') @@ -259,7 +274,7 @@ def generate_one_of(self): self.l('{variable}_one_of_count = 0') for definition_item in self._definition['oneOf']: with self.l('try:'): - self.generate_func_code_block(definition_item, self._variable, self._variable_name) + self.generate_func_code_block(definition_item, self._variable, self._variable_name, clear_variables=True) self.l('{variable}_one_of_count += 1') self.l('except JsonSchemaException: pass') @@ -276,8 +291,8 @@ def generate_not(self): Valid values for this definitions are 'hello', 42, {} ... but not None. """ - if self._definition['not'] is not None and not self._definition['not']: # {} - with self.l('if "{}" in {}.keys():', self._context['key'], self._context['definition'].get('properties', {})): + if not self._definition['not']: + with self.l('if {}:', self._variable): self.l('raise JsonSchemaException("{name} must not be valid by not definition")') else: with self.l('try:'): @@ -286,72 +301,58 @@ def generate_not(self): self.l('else: raise JsonSchemaException("{name} must not be valid by not definition")') def generate_min_length(self): - with self.l('if not isinstance({variable}, str):'): - self.l('return {variable}') - self.create_variable_with_length() - with self.l('if {variable}_len < {minLength}:'): - self.l('raise JsonSchemaException("{name} must be longer than or equal to {minLength} characters")') + with self.l('if isinstance({variable}, str):'): + self.create_variable_with_length() + with self.l('if {variable}_len < {minLength}:'): + self.l('raise JsonSchemaException("{name} must be longer than or equal to {minLength} characters")') def generate_max_length(self): - with self.l('if not isinstance({variable}, str):'): - self.l('return {variable}') - self.create_variable_with_length() - with self.l('if {variable}_len > {maxLength}:'): - self.l('raise JsonSchemaException("{name} must be shorter than or equal to {maxLength} characters")') + with self.l('if isinstance({variable}, str):'): + self.create_variable_with_length() + with self.l('if {variable}_len > {maxLength}:'): + self.l('raise JsonSchemaException("{name} must be shorter than or equal to {maxLength} characters")') def generate_pattern(self): - with self.l('if not isinstance({variable}, str):'): - self.l('return {variable}') - self._compile_regexps['{}_re'.format(self._variable)] = re.compile(self._definition['pattern']) - with self.l('if not {variable}_re.search({variable}):'): - self.l('raise JsonSchemaException("{name} must match pattern {pattern}")') + with self.l('if isinstance({variable}, str):'): + self._compile_regexps['{}_re'.format(self._variable)] = re.compile(self._definition['pattern']) + with self.l('if not {variable}_re.search({variable}):'): + self.l('raise JsonSchemaException("{name} must match pattern {pattern}")') def generate_minimum(self): - # We cannot retype original variable. Check for integer could fail then. - with self.l('try:'): - self.l('{variable}_max = float({variable})') - with self.l('except ValueError:'): - self.l('return {variable}') - if self._definition.get('exclusiveMinimum', False): - with self.l('if {variable}_max <= {minimum}:'): - self.l('raise JsonSchemaException("{name} must be bigger than {minimum}")') - else: - with self.l('if {variable}_max < {minimum}:'): - self.l('raise JsonSchemaException("{name} must be bigger than or equal to {minimum}")') + with self.l('if isinstance({variable}, (int, float)):'): + if self._definition.get('exclusiveMinimum', False): + with self.l('if {variable} <= {minimum}:'): + self.l('raise JsonSchemaException("{name} must be bigger than {minimum}")') + else: + with self.l('if {variable} < {minimum}:'): + self.l('raise JsonSchemaException("{name} must be bigger than or equal to {minimum}")') def generate_maximum(self): - # We cannot retype original variable. Check for integer could fail then. - with self.l('try:'): - self.l('{variable}_max = float({variable})') - with self.l('except ValueError:'): - self.l('return {variable}') - if self._definition.get('exclusiveMaximum', False): - with self.l('if {variable}_max >= {maximum}:'): - self.l('raise JsonSchemaException("{name} must be smaller than {maximum}")') - else: - with self.l('if {variable}_max > {maximum}:'): - self.l('raise JsonSchemaException("{name} must be smaller than or equal to {maximum}")') + with self.l('if isinstance({variable}, (int, float)):'): + if self._definition.get('exclusiveMaximum', False): + with self.l('if {variable} >= {maximum}:'): + self.l('raise JsonSchemaException("{name} must be smaller than {maximum}")') + else: + with self.l('if {variable} > {maximum}:'): + self.l('raise JsonSchemaException("{name} must be smaller than or equal to {maximum}")') def generate_multiple_of(self): - with self.l('if not isinstance({variable}, (int, float)):'): - self.l('return {variable}') - self.l('quotient = {variable} / {multipleOf}') - with self.l('if int(quotient) != quotient:'): - self.l('raise JsonSchemaException("{name} must be multiple of {multipleOf}")') + with self.l('if isinstance({variable}, (int, float)):'): + self.l('quotient = {variable} / {multipleOf}') + with self.l('if int(quotient) != quotient:'): + self.l('raise JsonSchemaException("{name} must be multiple of {multipleOf}")') def generate_min_items(self): - with self.l('if not isinstance({variable}, list):'): - self.l('return {variable}') - self.create_variable_with_length() - with self.l('if {variable}_len < {minItems}:'): - self.l('raise JsonSchemaException("{name} must contain at least {minItems} items")') + with self.l('if isinstance({variable}, list):'): + self.create_variable_with_length() + with self.l('if {variable}_len < {minItems}:'): + self.l('raise JsonSchemaException("{name} must contain at least {minItems} items")') def generate_max_items(self): - with self.l('if not isinstance({variable}, list):'): - self.l('return {variable}') - self.create_variable_with_length() - with self.l('if {variable}_len > {maxItems}:'): - self.l('raise JsonSchemaException("{name} must contain less than or equal to {maxItems} items")') + with self.l('if isinstance({variable}, list):'): + self.create_variable_with_length() + with self.l('if {variable}_len > {maxItems}:'): + self.l('raise JsonSchemaException("{name} must contain less than or equal to {maxItems} items")') def generate_unique_items(self): """ @@ -373,139 +374,102 @@ def generate_unique_items(self): self.l('raise JsonSchemaException("{name} must contain unique items")') def generate_items(self): - with self.l('if isinstance({variable}, dict):'): - self.l('return {variable}') - self.create_variable_with_length() - if isinstance(self._definition['items'], list): - for x, item_definition in enumerate(self._definition['items']): - with self.l('if {variable}_len > {}:', x): - self.l('{variable}_{0} = {variable}[{0}]', x) - self.generate_func_code_block( - item_definition, - '{}_{}'.format(self._variable, x), - '{}[{}]'.format(self._variable_name, x), - ) - if 'default' in item_definition: - self.l('else: {variable}.append({})', repr(item_definition['default'])) - - if 'additionalItems' in self._definition: - if self._definition['additionalItems'] is False: - self.l('if {variable}_len > {}: raise JsonSchemaException("{name} must contain only specified items")', len(self._definition['items'])) - else: - with self.l('for {variable}_x, {variable}_item in enumerate({variable}[{0}:], {0}):', len(self._definition['items'])): + with self.l('if isinstance({variable}, list):'): + self.create_variable_with_length() + if isinstance(self._definition['items'], list): + for x, item_definition in enumerate(self._definition['items']): + with self.l('if {variable}_len > {}:', x): + self.l('{variable}_{0} = {variable}[{0}]', x) self.generate_func_code_block( - self._definition['additionalItems'], + item_definition, + '{}_{}'.format(self._variable, x), + '{}[{}]'.format(self._variable_name, x), + ) + if 'default' in item_definition: + self.l('else: {variable}.append({})', repr(item_definition['default'])) + + if 'additionalItems' in self._definition: + if self._definition['additionalItems'] is False: + self.l('if {variable}_len > {}: raise JsonSchemaException("{name} must contain only specified items")', len(self._definition['items'])) + else: + with self.l('for {variable}_x, {variable}_item in enumerate({variable}[{0}:], {0}):', len(self._definition['items'])): + self.generate_func_code_block( + self._definition['additionalItems'], + '{}_item'.format(self._variable), + '{}[{{{}_x}}]'.format(self._variable_name, self._variable), + ) + else: + if self._definition['items']: + with self.l('for {variable}_x, {variable}_item in enumerate({variable}):'): + self.generate_func_code_block( + self._definition['items'], '{}_item'.format(self._variable), '{}[{{{}_x}}]'.format(self._variable_name, self._variable), ) - else: - if self._definition['items']: - with self.l('for {variable}_x, {variable}_item in enumerate({variable}):'): - self.generate_func_code_block( - self._definition['items'], - '{}_item'.format(self._variable), - '{}[{{{}_x}}]'.format(self._variable_name, self._variable), - ) def generate_min_properties(self): - with self.l('if not isinstance({variable}, dict):'): - self.l('return {variable}') - self.create_variable_with_length() - with self.l('if {variable}_len < {minProperties}:'): - self.l('raise JsonSchemaException("{name} must contain at least {minProperties} properties")') + with self.l('if isinstance({variable}, dict):'): + self.create_variable_with_length() + with self.l('if {variable}_len < {minProperties}:'): + self.l('raise JsonSchemaException("{name} must contain at least {minProperties} properties")') def generate_max_properties(self): - with self.l('if not isinstance({variable}, dict):'): - self.l('return {variable}') - self.create_variable_with_length() - with self.l('if {variable}_len > {maxProperties}:'): - self.l('raise JsonSchemaException("{name} must contain less than or equal to {maxProperties} properties")') + with self.l('if isinstance({variable}, dict):'): + self.create_variable_with_length() + with self.l('if {variable}_len > {maxProperties}:'): + self.l('raise JsonSchemaException("{name} must contain less than or equal to {maxProperties} properties")') def generate_required(self): - with self.l('if not isinstance({variable}, dict):'): - self.l('return {variable}') - self.create_variable_with_length() - with self.l('if not all(prop in {variable} for prop in {required}):'): - self.l('raise JsonSchemaException("{name} must contain {required} properties")') + with self.l('if isinstance({variable}, dict):'): + self.create_variable_with_length() + with self.l('if not all(prop in {variable} for prop in {required}):'): + self.l('raise JsonSchemaException("{name} must contain {required} properties")') def generate_properties(self): - with self.l('if not isinstance({variable}, dict):'): - self.l('return {variable}') - self.l('{variable}_keys = set({variable}.keys())') - for key, prop_definition in self._definition['properties'].items(): - with self.l('if "{}" in {variable}_keys:', key): - self.l('{variable}_keys.remove("{}")', key) - self.l('{variable}_{0} = {variable}["{0}"]', key) - self.generate_func_code_block( - prop_definition, - '{}_{}'.format(self._variable, key), - '{}.{}'.format(self._variable_name, key), - context={ - "definition": self._definition, - "key": key - }, - ) - if 'default' in prop_definition: - self.l('else: {variable}["{}"] = {}', key, repr(prop_definition['default'])) - - if 'patternProperties' in self._definition: - self.generate_func_code_block( - {'patternProperties': self._definition['patternProperties']}, - self._variable, - '{}.{}'.format(self._variable_name, 'patternProperties'), - ) - self.l('pattern_keys = set()') - with self.l('for key in {variable}_keys:'): - for pattern in self._definition['patternProperties'].keys(): - with self.l('if globals()["{}_re"].search(key):', pattern): - self.l('pattern_keys.add(key)') - self.l('{variable}_keys -= pattern_keys') - - if 'additionalProperties' in self._definition: - if self._definition['additionalProperties'] is False: - self.l('if {variable}_keys: raise JsonSchemaException("{name} must contain only specified properties")') - else: - with self.l('for {variable}_key in {variable}_keys:'): - self.l('{variable}_value = {variable}.get({variable}_key)') - self.generate_func_code_block( - self._definition['additionalProperties'], - '{}_value'.format(self._variable), - '{}.{{{}_key}}'.format(self._variable_name, self._variable), - ) - - def generate_additional_properties(self): - with self.l('if not isinstance({variable}, dict):'): - self.l('return {variable}') - self.l('{variable}_keys = set({variable}.keys())') - - if 'patternProperties' in self._definition: - self.l('return {variable}') - - add_prop_definition = self._definition["additionalProperties"] - if add_prop_definition: - properties_keys = self._definition.get("properties", {}).keys() - with self.l('for {variable}_key in {variable}_keys:'): - with self.l('if {variable}_key not in "{}":', properties_keys): - self.l('{variable}_value = {variable}.get({variable}_key)') + with self.l('if isinstance({variable}, dict):'): + self.create_variable_keys() + for key, prop_definition in self._definition['properties'].items(): + with self.l('if "{}" in {variable}_keys:', key): + self.l('{variable}_keys.remove("{}")', key) + self.l('{variable}_{0} = {variable}["{0}"]', key) self.generate_func_code_block( - add_prop_definition, - '{}_value'.format(self._variable), - '{}.{{{}_key}}'.format(self._variable_name, self._variable), + prop_definition, + '{}_{}'.format(self._variable, key), + '{}.{}'.format(self._variable_name, key), ) + if 'default' in prop_definition: + self.l('else: {variable}["{}"] = {}', key, repr(prop_definition['default'])) def generate_pattern_properties(self): - with self.l('if not isinstance({variable}, dict):'): - self.l('return {variable}') - for pattern, definition in self._definition['patternProperties'].items(): - self._compile_regexps['{}_re'.format(pattern)] = re.compile(pattern) - with self.l('for key, val in {variable}.items():'): + with self.l('if isinstance({variable}, dict):'): + self.create_variable_keys() for pattern, definition in self._definition['patternProperties'].items(): - if not definition: - self.l('pass') - else: + self._compile_regexps['{}_re'.format(pattern)] = re.compile(pattern) + with self.l('for key, val in {variable}.items():'): + for pattern, definition in self._definition['patternProperties'].items(): with self.l('if globals()["{}_re"].search(key):', pattern): + with self.l('if key in {variable}_keys:'): + self.l('{variable}_keys.remove(key)') self.generate_func_code_block( definition, 'val', '{}.{{key}}'.format(self._variable_name), ) + + def generate_additional_properties(self): + with self.l('if isinstance({variable}, dict):'): + self.create_variable_keys() + add_prop_definition = self._definition["additionalProperties"] + if add_prop_definition: + properties_keys = self._definition.get("properties", {}).keys() + with self.l('for {variable}_key in {variable}_keys:'): + with self.l('if {variable}_key not in "{}":', properties_keys): + self.l('{variable}_value = {variable}.get({variable}_key)') + self.generate_func_code_block( + add_prop_definition, + '{}_value'.format(self._variable), + '{}.{{{}_key}}'.format(self._variable_name, self._variable), + ) + else: + with self.l('if {variable}_keys:'): + self.l('raise JsonSchemaException("{name} must contain only specified properties")') diff --git a/tests/test_json_schema_test_suits.py b/tests/test_json_schema_test_suits.py index b516c55..a79e079 100644 --- a/tests/test_json_schema_test_suits.py +++ b/tests/test_json_schema_test_suits.py @@ -52,7 +52,8 @@ def test(schema, data, is_valid): validate = compile(schema) try: - validate(data) + result = validate(data) + print('Validate result:', result) except JsonSchemaException: if is_valid: raise From 366c84bb2062673fd6f330ce5acc307c6a0ddca3 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 28 Mar 2018 11:39:09 +0200 Subject: [PATCH 030/201] Base support of references --- fastjsonschema/__init__.py | 7 +++-- fastjsonschema/generator.py | 42 ++++++++++++++++++++++++++-- setup.py | 3 ++ tests/test_json_schema_test_suits.py | 2 -- tests/test_object.py | 1 - 5 files changed, 47 insertions(+), 8 deletions(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 1e2f45c..a33561a 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -83,6 +83,7 @@ def compile(definition): Exception :any:`JsonSchemaException` is thrown when validation fails. """ code_generator = CodeGenerator(definition) - local_state = {} - exec(code_generator.func_code, code_generator.global_state, local_state) - return local_state['func'] + # Do not pass local state so it can recursively call itself. + global_state = code_generator.global_state + exec(code_generator.func_code, global_state) + return global_state['func'] diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 662489e..983feef 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -8,6 +8,8 @@ from collections import OrderedDict import re +import requests + from .exceptions import JsonSchemaException from .indent import indent @@ -51,9 +53,11 @@ def __init__(self, definition): self._indent = 0 self._variable = None self._variable_name = None + self._root_definition = definition self._definition = None self._json_keywords_to_function = OrderedDict(( + ('$ref', self.generate_ref), ('type', self.generate_type), ('enum', self.generate_enum), ('allOf', self.generate_all_of), @@ -186,6 +190,30 @@ def generate_func_code_block(self, definition, variable, variable_name, clear_va if clear_variables: self._variables = backup_variables + def generate_ref(self): + """ + Ref can be link to remote or local definition. + + .. code-block:: python + + {'$ref': 'http://json-schema.org/draft-04/schema#'} + { + 'properties': { + 'foo': {'type': 'integer'}, + 'bar': {'$ref': '#/properties/foo'} + } + } + """ + if self._definition['$ref'].startswith('http'): + res = requests.get(self._definition['$ref']) + definition = res.json() + self.generate_func_code_block(definition, self._variable, self._variable_name) + elif self._definition['$ref'] == '#': + self.l('func({variable})') + else: + #TODO: Create more functions for any ref and call it here. + self.l('pass') + def generate_type(self): """ Validation of type. Can be one type or list of types. @@ -206,6 +234,15 @@ def generate_type(self): self.l('raise JsonSchemaException("{name} must be {}")', ' or '.join(types)) def generate_enum(self): + """ + Means that only value specified in the enum is valid. + + .. code-block:: python + + { + 'enum': ['a', 'b'], + } + """ with self.l('if {variable} not in {enum}:'): self.l('raise JsonSchemaException("{name} must be one of {enum}")') @@ -429,12 +466,13 @@ def generate_properties(self): with self.l('if isinstance({variable}, dict):'): self.create_variable_keys() for key, prop_definition in self._definition['properties'].items(): + key_name = re.sub(r'($[^a-zA-Z]|[^a-zA-Z0-9])', '', key) with self.l('if "{}" in {variable}_keys:', key): self.l('{variable}_keys.remove("{}")', key) - self.l('{variable}_{0} = {variable}["{0}"]', key) + self.l('{variable}_{0} = {variable}["{0}"]', key_name) self.generate_func_code_block( prop_definition, - '{}_{}'.format(self._variable, key), + '{}_{}'.format(self._variable, key_name), '{}.{}'.format(self._variable_name, key), ) if 'default' in prop_definition: diff --git a/setup.py b/setup.py index 95cea7b..5116425 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,9 @@ version='1.1', packages=['fastjsonschema'], + install_requires=[ + 'requests', + ], extras_require={ "test": [ "colorama", diff --git a/tests/test_json_schema_test_suits.py b/tests/test_json_schema_test_suits.py index a79e079..96e07a1 100644 --- a/tests/test_json_schema_test_suits.py +++ b/tests/test_json_schema_test_suits.py @@ -11,10 +11,8 @@ def pytest_generate_tests(metafunc): ignored_suite_files = [ 'definitions.json', 'dependencies.json', - 'bignum.json', 'ecmascript-regex.json', 'format.json', - 'zeroTerminatedFloats.json', 'ref.json', 'refRemote.json', 'uniqueItems.json', diff --git a/tests/test_object.py b/tests/test_object.py index 0a686a9..d355230 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -110,7 +110,6 @@ def test_properties_without_additional_properties(asserter, value, expected): }, value, expected) -@pytest.mark.xfail @pytest.mark.parametrize('value, expected', [ ({}, {}), ({'a': 1}, {'a': 1}), From 9cfe5dab973498fa417a49c2341de3c70192c56b Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 28 Mar 2018 12:10:58 +0200 Subject: [PATCH 031/201] Support of formats --- fastjsonschema/generator.py | 16 ++++++++++++++++ tests/test_json_schema_test_suits.py | 1 - 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 983feef..2fb8453 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -67,6 +67,7 @@ def __init__(self, definition): ('minLength', self.generate_min_length), ('maxLength', self.generate_max_length), ('pattern', self.generate_pattern), + ('format', self.generate_format), ('minimum', self.generate_minimum), ('maximum', self.generate_maximum), ('multipleOf', self.generate_multiple_of), @@ -355,6 +356,21 @@ def generate_pattern(self): with self.l('if not {variable}_re.search({variable}):'): self.l('raise JsonSchemaException("{name} must match pattern {pattern}")') + def generate_format(self): + with self.l('if isinstance({variable}, str):'): + self._generate_format('date-time', 'date_time_re_pattern', r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{6})?Z?$') + self._generate_format('uri', 'uri_re_pattern', r'^\w+:(\/?\/?)[^\s]+$') + self._generate_format('email', 'email_re_pattern', r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$') + self._generate_format('ipv4', 'ipv4_re_pattern', r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$') + self._generate_format('ipv6', 'ipv6_re_pattern', r'^(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|::(?:[0-9A-Fa-f]{1,4}:){5}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){4}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){3}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,2}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){2}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,3}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}:(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,4}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,5}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}|(?:(?:[0-9A-Fa-f]{1,4}:){,6}[0-9A-Fa-f]{1,4})?::)$') + self._generate_format('hostname', 'hostname_re_pattern', r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{1,62}[A-Za-z0-9])$') + + def _generate_format(self, format_name, regexp_name, regexp): + if self._definition['format'] == format_name: + self._compile_regexps[regexp_name] = re.compile(regexp) + with self.l('if not {}.match({variable}):', regexp_name): + self.l('raise JsonSchemaException("{name} must be {}")', format_name) + def generate_minimum(self): with self.l('if isinstance({variable}, (int, float)):'): if self._definition.get('exclusiveMinimum', False): diff --git a/tests/test_json_schema_test_suits.py b/tests/test_json_schema_test_suits.py index 96e07a1..42e8189 100644 --- a/tests/test_json_schema_test_suits.py +++ b/tests/test_json_schema_test_suits.py @@ -12,7 +12,6 @@ def pytest_generate_tests(metafunc): 'definitions.json', 'dependencies.json', 'ecmascript-regex.json', - 'format.json', 'ref.json', 'refRemote.json', 'uniqueItems.json', From fae4509447bfab73e50fc382e5c5f820feb72371 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 24 Apr 2018 14:12:57 +0200 Subject: [PATCH 032/201] #5 Fix underscores in property names --- fastjsonschema/generator.py | 2 +- tests/test_object.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 2fb8453..4e6859b 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -485,7 +485,7 @@ def generate_properties(self): key_name = re.sub(r'($[^a-zA-Z]|[^a-zA-Z0-9])', '', key) with self.l('if "{}" in {variable}_keys:', key): self.l('{variable}_keys.remove("{}")', key) - self.l('{variable}_{0} = {variable}["{0}"]', key_name) + self.l('{variable}_{0} = {variable}["{1}"]', key_name, key) self.generate_func_code_block( prop_definition, '{}_{}'.format(self._variable, key_name), diff --git a/tests/test_object.py b/tests/test_object.py index d355230..8d425b8 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -98,6 +98,8 @@ def test_properties_with_additional_properties(asserter, value, expected): ({'a': 1, 'b': ''}, {'a': 1, 'b': ''}), ({'a': 1, 'b': 2}, JsonSchemaException('data.b must be string')), ({'a': 1, 'b': '', 'any': True}, JsonSchemaException('data must contain only specified properties')), + ({'cd': True}, JsonSchemaException('data must contain only specified properties')), + ({'c_d': True}, {'c_d': True}), ]) def test_properties_without_additional_properties(asserter, value, expected): asserter({ @@ -105,6 +107,7 @@ def test_properties_without_additional_properties(asserter, value, expected): 'properties': { 'a': {'type': 'number'}, 'b': {'type': 'string'}, + 'c_d': {'type': 'boolean'}, }, 'additionalProperties': False, }, value, expected) From a14b53f6123805ee780295503c759c2bfe1d900f Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 24 Apr 2018 14:33:06 +0200 Subject: [PATCH 033/201] Documentation --- fastjsonschema/__init__.py | 52 ++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index a33561a..7cf34d1 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -1,46 +1,44 @@ """ -This project is there because commonly used JSON schema libraries in Python -are really slow which was problem at out project. Just let's see some numbers first: +This project was made to come up with fast JSON validations. Just let's see some numbers first: - * Probalby most popular ``jsonschema`` can take in tests up to 11 seconds for valid inputs - and 2.5 seconds for invalid inputs. - * Secondly most popular ``json-spec`` is even worse with up to 12 and 3 seconds. - * Lastly ``validictory`` is much better with 800 or 50 miliseconds, but it does not + * Probalby most popular ``jsonschema`` can take in tests up to 7 seconds for valid inputs + and 1.6 seconds for invalid inputs. + * Secondly most popular ``json-spec`` is even worse with up to 11 and 2.6 seconds. + * Lastly ``validictory`` is much better with 640 or 30 miliseconds, but it does not follow all standards and it can be still slow for some purposes. -That's why there is this project which compiles definition into Python most stupid code +That's why this project exists. It compiles definition into Python most stupid code which people would had hard time to write by themselfs because of not-written-rule DRY -(don't repeat yourself). When you compile definition, then times are 60 miliseconds for -valid inputs and 3 miliseconds for invalid inputs. Pretty amazing, right? :-) +(don't repeat yourself). When you compile definition, then times are 90 miliseconds for +valid inputs and 5 miliseconds for invalid inputs. Pretty amazing, right? :-) You can try it for yourself with included script: .. code-block:: bash $ make performance - fast_compiled valid ==> 0.06058199889957905 - fast_compiled invalid ==> 0.0028909146785736084 - fast_not_compiled valid ==> 7.054106639698148 - fast_not_compiled invalid ==> 1.6773221027106047 - jsonschema valid ==> 11.189393147826195 - jsonschema invalid ==> 2.642645660787821 - jsonspec valid ==> 11.942349303513765 - jsonspec invalid ==> 2.9887414034456015 - validictory valid ==> 0.7500483158230782 - validictory invalid ==> 0.03606216423213482 - -This library follows and implements `JSON schema `_. Sometimes + fast_compiled valid ==> 0.09240092901140451 + fast_compiled invalid ==> 0.004246290685236454 + fast_not_compiled valid ==> 6.710726021323353 + fast_not_compiled invalid ==> 1.5449269418604672 + jsonschema valid ==> 6.963333621155471 + jsonschema invalid ==> 1.6309524956159294 + jsonspec valid ==> 10.576010060030967 + jsonspec invalid ==> 2.6199211929924786 + validictory valid ==> 0.6349993739277124 + validictory invalid ==> 0.03125431900843978 + +This library follows and implements `JSON schema v4 `_. Sometimes it's not perfectly clear so I recommend also check out this `understaning json schema `_. Note that there are some differences compared to JSON schema standard: - * ``dependency`` for objects are not implemented yet. Future implementation will not change speed. - * ``patternProperty`` for objects are not implemented yet. Future implementation can little bit - slow down validation of object properties. Of course only for those who uses ``properties``. - * ``definitions`` for sharing JSON schema are not implemented yet. Future implementation will - not change speed. - * Regular expressions are full what Python provides, not only what JSON schema allows. It's easier + * ``dependency`` for objects are not implemented yet. Future implementation will not change the speed. + * ``definitions`` and ``ref`` for sharing JSON schema are not implemented yet. Future implementation will + not change the speed. + * ``uniqueItems`` does not work with Python objects yet. Future implementation may change the speed. + * Regular expressions are full Python ones, not only what JSON schema allows. It's easier to allow everything and also it's faster to compile without limits. So keep in mind that when you will use more advanced regular expression, it may not work with other library. * JSON schema says you can use keyword ``default`` for providing default values. This implementation From e7fbac06596cda77c80eea869fe85ac6593a9a6f Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 24 Apr 2018 14:37:23 +0200 Subject: [PATCH 034/201] Support uniqueItems of objects --- fastjsonschema/__init__.py | 1 - fastjsonschema/generator.py | 2 +- tests/test_json_schema_test_suits.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 7cf34d1..63da03d 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -37,7 +37,6 @@ * ``dependency`` for objects are not implemented yet. Future implementation will not change the speed. * ``definitions`` and ``ref`` for sharing JSON schema are not implemented yet. Future implementation will not change the speed. - * ``uniqueItems`` does not work with Python objects yet. Future implementation may change the speed. * Regular expressions are full Python ones, not only what JSON schema allows. It's easier to allow everything and also it's faster to compile without limits. So keep in mind that when you will use more advanced regular expression, it may not work with other library. diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 4e6859b..6b40b71 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -423,7 +423,7 @@ def generate_unique_items(self): 2.1439831256866455 """ self.create_variable_with_length() - with self.l('if {variable}_len > len(set({variable})):'): + with self.l('if {variable}_len > len(set(str(x) for x in {variable})):'): self.l('raise JsonSchemaException("{name} must contain unique items")') def generate_items(self): diff --git a/tests/test_json_schema_test_suits.py b/tests/test_json_schema_test_suits.py index 42e8189..c764d78 100644 --- a/tests/test_json_schema_test_suits.py +++ b/tests/test_json_schema_test_suits.py @@ -14,7 +14,6 @@ def pytest_generate_tests(metafunc): 'ecmascript-regex.json', 'ref.json', 'refRemote.json', - 'uniqueItems.json', ] suite_dir_path = Path(suite_dir).resolve() From ffe222b8af19cf1ab7741bb07bc8f1687c178a50 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 24 Apr 2018 14:39:36 +0200 Subject: [PATCH 035/201] Version 1.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5116425..6deaed8 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='fastjsonschema', - version='1.1', + version='1.2', packages=['fastjsonschema'], install_requires=[ From 43b4899393303ef9a2c13ed17e62320623cab87a Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 25 Apr 2018 13:59:29 +0200 Subject: [PATCH 036/201] #6 Fix pattern inside of anyOf --- Makefile | 3 +++ fastjsonschema/generator.py | 4 ++-- tests/test_integration.py | 24 ++++++++++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 26841eb..640bd11 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,9 @@ install: test: python3 -m pytest tests +test-lf: + python3 -m pytest --last-fail tests + performance: python3 performance.py diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 6b40b71..17810d4 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -352,8 +352,8 @@ def generate_max_length(self): def generate_pattern(self): with self.l('if isinstance({variable}, str):'): - self._compile_regexps['{}_re'.format(self._variable)] = re.compile(self._definition['pattern']) - with self.l('if not {variable}_re.search({variable}):'): + self._compile_regexps['{}_re'.format(self._definition['pattern'])] = re.compile(self._definition['pattern']) + with self.l('if not globals()["{}_re"].search({variable}):', self._definition['pattern']): self.l('raise JsonSchemaException("{name} must match pattern {pattern}")') def generate_format(self): diff --git a/tests/test_integration.py b/tests/test_integration.py index 57af252..f0ac8e9 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -99,3 +99,27 @@ def test_integration(asserter, value, expected): ]}, ], }, value, expected) + + +def test_any_of_with_patterns(asserter): + asserter({ + 'type': 'object', + 'properties': { + 'hash': { + 'anyOf': [ + { + 'type': 'string', + 'pattern': '^AAA' + }, + { + 'type': 'string', + 'pattern': '^BBB' + } + ] + } + } + }, { + 'hash': 'AAAXXX', + }, { + 'hash': 'AAAXXX', + }) From 999506b02e23035d9b7e88f524b0c224100e5d6f Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 25 Apr 2018 14:34:40 +0200 Subject: [PATCH 037/201] Version 1.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6deaed8..097b93f 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='fastjsonschema', - version='1.2', + version='1.3', packages=['fastjsonschema'], install_requires=[ From 83a004e1238cc78630233f69929147e267ea7a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillaume=20Desv=C3=A9?= Date: Wed, 25 Apr 2018 17:04:46 +0200 Subject: [PATCH 038/201] More precise date-time regex and handling of '+hh:mm' --- fastjsonschema/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 17810d4..2eaf8e2 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -358,7 +358,7 @@ def generate_pattern(self): def generate_format(self): with self.l('if isinstance({variable}, str):'): - self._generate_format('date-time', 'date_time_re_pattern', r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{6})?Z?$') + self._generate_format('date-time', 'date_time_re_pattern', r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+(?:[+-][0-2]\d:[0-5]\d|Z)?$') self._generate_format('uri', 'uri_re_pattern', r'^\w+:(\/?\/?)[^\s]+$') self._generate_format('email', 'email_re_pattern', r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$') self._generate_format('ipv4', 'ipv4_re_pattern', r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$') From d44cf035a297f97a1fda434e9363eba11af8d2c1 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 30 Apr 2018 12:22:33 +0200 Subject: [PATCH 039/201] #8 Raise exception instead of silent pass --- fastjsonschema/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 2eaf8e2..2a07858 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -213,7 +213,7 @@ def generate_ref(self): self.l('func({variable})') else: #TODO: Create more functions for any ref and call it here. - self.l('pass') + raise Exception('Local ref is not supported yet') def generate_type(self): """ From 2b1bdd97392299db7cdcdf8003731325aef8554d Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 30 Apr 2018 12:36:25 +0200 Subject: [PATCH 040/201] #8 NotImplementedError instead of Exception --- fastjsonschema/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 2a07858..5b7b7a7 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -213,7 +213,7 @@ def generate_ref(self): self.l('func({variable})') else: #TODO: Create more functions for any ref and call it here. - raise Exception('Local ref is not supported yet') + raise NotImplementedError('Local ref is not supported yet') def generate_type(self): """ From 990aec58f61591b0155c04f4a342417a64429299 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Mon, 11 Jun 2018 05:21:51 +0300 Subject: [PATCH 041/201] implementation for dependencies --- fastjsonschema/__init__.py | 1 - fastjsonschema/generator.py | 13 +++++++++++++ tests/test_json_schema_test_suits.py | 1 - 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 63da03d..57bc399 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -34,7 +34,6 @@ Note that there are some differences compared to JSON schema standard: - * ``dependency`` for objects are not implemented yet. Future implementation will not change the speed. * ``definitions`` and ``ref`` for sharing JSON schema are not implemented yet. Future implementation will not change the speed. * Regular expressions are full Python ones, not only what JSON schema allows. It's easier diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 5b7b7a7..179d504 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -81,6 +81,7 @@ def __init__(self, definition): ('properties', self.generate_properties), ('patternProperties', self.generate_pattern_properties), ('additionalProperties', self.generate_additional_properties), + ('dependencies', self.generate_dependencies), )) self.generate_func_code(definition) @@ -527,3 +528,15 @@ def generate_additional_properties(self): else: with self.l('if {variable}_keys:'): self.l('raise JsonSchemaException("{name} must contain only specified properties")') + + def generate_dependencies(self): + with self.l('if isinstance({variable}, dict):'): + self.create_variable_keys() + for key, values in self._definition["dependencies"].items(): + with self.l('if "{}" in {variable}_keys:', key): + if isinstance(values, list): + for value in values: + with self.l('if "{}" not in {variable}_keys:', value): + self.l('raise JsonSchemaException("{name} missing dependency {} for {}")', value, key) + else: + self.generate_func_code_block(values, self._variable, self._variable_name, clear_variables=True) diff --git a/tests/test_json_schema_test_suits.py b/tests/test_json_schema_test_suits.py index c764d78..0ee5089 100644 --- a/tests/test_json_schema_test_suits.py +++ b/tests/test_json_schema_test_suits.py @@ -10,7 +10,6 @@ def pytest_generate_tests(metafunc): suite_dir = 'JSON-Schema-Test-Suite/tests/draft4' ignored_suite_files = [ 'definitions.json', - 'dependencies.json', 'ecmascript-regex.json', 'ref.json', 'refRemote.json', From 65e029a3b2302db1d3efd3eadc1e927fbd0d27ab Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 11 Jun 2018 10:33:05 +0200 Subject: [PATCH 042/201] Version 1.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 097b93f..29a83af 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='fastjsonschema', - version='1.3', + version='1.4', packages=['fastjsonschema'], install_requires=[ From d873e125bc71babefe49cf21f4779be2cde7cc3b Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Tue, 12 Jun 2018 21:28:24 +0300 Subject: [PATCH 043/201] partial ref implementation: 15 new cases into xpassed --- fastjsonschema/generator.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 179d504..023e297 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -7,6 +7,7 @@ from collections import OrderedDict import re +from urllib.parse import unquote import requests @@ -20,6 +21,23 @@ def enforce_list(variable): return [variable] +def resolve_path(schema, path): + """ + Return definition from path. + + Path is unescaped according https://tools.ietf.org/html/rfc6901 + """ + parts = unquote(path).split('/') if path else [] + current = schema + for part in parts: + part = part.replace(u"~1", u"/").replace(u"~0", u"~") + if isinstance(current, list): + current = current[int(part)] + else: + current = current[part] + return current + + class CodeGenerator: """ This class is not supposed to be used directly. Anything @@ -184,9 +202,13 @@ def generate_func_code_block(self, definition, variable, variable_name, clear_va backup_variables = self._variables self._variables = set() - for key, func in self._json_keywords_to_function.items(): - if key in definition: - func() + if '$ref' in definition: + # needed because ref overrides any sibling keywords + self.generate_ref() + else: + for key, func in self._json_keywords_to_function.items(): + if key in definition: + func() self._definition, self._variable, self._variable_name = backup if clear_variables: @@ -212,8 +234,12 @@ def generate_ref(self): self.generate_func_code_block(definition, self._variable, self._variable_name) elif self._definition['$ref'] == '#': self.l('func({variable})') + elif self._definition['$ref'].startswith('#'): + path = self._definition['$ref'].lstrip('#/') + current = resolve_path(self._root_definition, path) + self.generate_func_code_block(current, self._variable, self._variable_name, clear_variables=True) else: - #TODO: Create more functions for any ref and call it here. + # TODO: Create more functions for any ref and call it here. raise NotImplementedError('Local ref is not supported yet') def generate_type(self): From 7be8551a9409805dcb3c13c54bf3e3842abff02a Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Wed, 13 Jun 2018 03:45:13 +0300 Subject: [PATCH 044/201] pass definitions test + 2 more xpasses for local ref tests --- fastjsonschema/generator.py | 28 ++++++++++++++++++++-------- tests/test_json_schema_test_suits.py | 1 - 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 023e297..a0c1f21 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -7,7 +7,7 @@ from collections import OrderedDict import re -from urllib.parse import unquote +from urllib.parse import unquote, urlparse import requests @@ -63,9 +63,10 @@ class CodeGenerator: 'object': 'dict', } - def __init__(self, definition): + def __init__(self, definition, name='func'): self._code = [] self._compile_regexps = {} + self._validation_functions = set() self._variables = set() self._indent = 0 @@ -102,7 +103,7 @@ def __init__(self, definition): ('dependencies', self.generate_dependencies), )) - self.generate_func_code(definition) + self.generate_func_code(definition, name) @property def func_code(self): @@ -182,12 +183,12 @@ def create_variable_keys(self): self._variables.add(variable_name) self.l('{variable}_keys = set({variable}.keys())') - def generate_func_code(self, definition): + def generate_func_code(self, definition, name): """ Creates base code of validation function and calls helper for creating code by definition. """ - with self.l('def func(data):'): + with self.l('def {}(data):', name): self.l('NoneType = type(None)') self.generate_func_code_block(definition, 'data', 'data') self.l('return data') @@ -229,9 +230,18 @@ def generate_ref(self): } """ if self._definition['$ref'].startswith('http'): - res = requests.get(self._definition['$ref']) - definition = res.json() - self.generate_func_code_block(definition, self._variable, self._variable_name) + name = 'validate_' + self._definition['$ref'] + name = re.sub('[:/#.-]', '_', name) + if not name in self._validation_functions: + res = requests.get(self._definition['$ref']) + definition = res.json() + current = resolve_path(definition, urlparse(self._definition['$ref']).fragment) + code_generator = CodeGenerator(current, name) + self._code.insert(0, code_generator.func_code + '\n') + self._validation_functions.add(name) + for key, value in code_generator._compile_regexps.items(): + self._compile_regexps[key] = value + self.l('{}({variable})', name) elif self._definition['$ref'] == '#': self.l('func({variable})') elif self._definition['$ref'].startswith('#'): @@ -391,6 +401,8 @@ def generate_format(self): self._generate_format('ipv4', 'ipv4_re_pattern', r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$') self._generate_format('ipv6', 'ipv6_re_pattern', r'^(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|::(?:[0-9A-Fa-f]{1,4}:){5}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){4}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){3}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,2}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){2}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,3}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}:(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,4}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,5}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}|(?:(?:[0-9A-Fa-f]{1,4}:){,6}[0-9A-Fa-f]{1,4})?::)$') self._generate_format('hostname', 'hostname_re_pattern', r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{1,62}[A-Za-z0-9])$') + # TODO real pattern for regex + self._generate_format('regex', 'regex_re_pattern', r'^.+$') def _generate_format(self, format_name, regexp_name, regexp): if self._definition['format'] == format_name: diff --git a/tests/test_json_schema_test_suits.py b/tests/test_json_schema_test_suits.py index 0ee5089..f645f12 100644 --- a/tests/test_json_schema_test_suits.py +++ b/tests/test_json_schema_test_suits.py @@ -9,7 +9,6 @@ def pytest_generate_tests(metafunc): suite_dir = 'JSON-Schema-Test-Suite/tests/draft4' ignored_suite_files = [ - 'definitions.json', 'ecmascript-regex.json', 'ref.json', 'refRemote.json', From d22bbfcde7c8a1807acd2f2fa9a6b276556b0990 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Wed, 13 Jun 2018 09:37:15 +0300 Subject: [PATCH 045/201] fix ref fragment parsing: 2 more xpassed --- fastjsonschema/generator.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index a0c1f21..54299b0 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -27,6 +27,7 @@ def resolve_path(schema, path): Path is unescaped according https://tools.ietf.org/html/rfc6901 """ + path = path.lstrip('#/') parts = unquote(path).split('/') if path else [] current = schema for part in parts: @@ -74,6 +75,7 @@ def __init__(self, definition, name='func'): self._variable_name = None self._root_definition = definition self._definition = None + self._name = name self._json_keywords_to_function = OrderedDict(( ('$ref', self.generate_ref), @@ -243,9 +245,9 @@ def generate_ref(self): self._compile_regexps[key] = value self.l('{}({variable})', name) elif self._definition['$ref'] == '#': - self.l('func({variable})') + self.l('{}({variable})', self._name) elif self._definition['$ref'].startswith('#'): - path = self._definition['$ref'].lstrip('#/') + path = self._definition['$ref'] current = resolve_path(self._root_definition, path) self.generate_func_code_block(current, self._variable, self._variable_name, clear_variables=True) else: From 95df60d36c990654d7fdf857e48b9fc3efe9f905 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Thu, 14 Jun 2018 04:39:16 +0300 Subject: [PATCH 046/201] RefResolver adapted from https://github.com/Julian/jsonschema. 4 cases still xfails: Recursive references between schemas and base URI change - change folder in subschema --- fastjsonschema/generator.py | 84 ++++++++---------- fastjsonschema/ref_resolver.py | 157 +++++++++++++++++++++++++++++++++ 2 files changed, 196 insertions(+), 45 deletions(-) create mode 100644 fastjsonschema/ref_resolver.py diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 54299b0..c68cbdf 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -7,12 +7,12 @@ from collections import OrderedDict import re -from urllib.parse import unquote, urlparse import requests from .exceptions import JsonSchemaException from .indent import indent +from .ref_resolver import RefResolver def enforce_list(variable): @@ -21,24 +21,6 @@ def enforce_list(variable): return [variable] -def resolve_path(schema, path): - """ - Return definition from path. - - Path is unescaped according https://tools.ietf.org/html/rfc6901 - """ - path = path.lstrip('#/') - parts = unquote(path).split('/') if path else [] - current = schema - for part in parts: - part = part.replace(u"~1", u"/").replace(u"~0", u"~") - if isinstance(current, list): - current = current[int(part)] - else: - current = current[part] - return current - - class CodeGenerator: """ This class is not supposed to be used directly. Anything @@ -64,10 +46,9 @@ class CodeGenerator: 'object': 'dict', } - def __init__(self, definition, name='func'): + def __init__(self, definition, name='func', resolver=None): self._code = [] self._compile_regexps = {} - self._validation_functions = set() self._variables = set() self._indent = 0 @@ -77,8 +58,14 @@ def __init__(self, definition, name='func'): self._definition = None self._name = name + self._validation_functions = {} + self._validation_functions_done = set() + if resolver == None: + resolver = RefResolver.from_schema(definition) + self._resolver = resolver + self._json_keywords_to_function = OrderedDict(( - ('$ref', self.generate_ref), + ('definitions', self.generate_defitions), ('type', self.generate_type), ('enum', self.generate_enum), ('allOf', self.generate_all_of), @@ -190,10 +177,20 @@ def generate_func_code(self, definition, name): Creates base code of validation function and calls helper for creating code by definition. """ + self._validation_functions_done.add(self._resolver.get_uri()) + self.l('NoneType = type(None)') + self.l('') with self.l('def {}(data):', name): - self.l('NoneType = type(None)') self.generate_func_code_block(definition, 'data', 'data') self.l('return data') + while len(self._validation_functions) > 0: + uri, name = self._validation_functions.popitem() + self._validation_functions_done.add(uri) + self.l('') + with self._resolver.resolving(uri) as definition: + with self.l('def {}(data):', name): + self.generate_func_code_block(definition, 'data', 'data') + self.l('return data') def generate_func_code_block(self, definition, variable, variable_name, clear_variables=False): """ @@ -208,7 +205,14 @@ def generate_func_code_block(self, definition, variable, variable_name, clear_va if '$ref' in definition: # needed because ref overrides any sibling keywords self.generate_ref() - else: + elif 'id' in definition: + id = definition['id'] + with self._resolver.in_scope(id): + self._resolver.store_id(self._definition) + for key, func in self._json_keywords_to_function.items(): + if key in definition: + func() + else: for key, func in self._json_keywords_to_function.items(): if key in definition: func() @@ -231,28 +235,18 @@ def generate_ref(self): } } """ - if self._definition['$ref'].startswith('http'): - name = 'validate_' + self._definition['$ref'] - name = re.sub('[:/#.-]', '_', name) - if not name in self._validation_functions: - res = requests.get(self._definition['$ref']) - definition = res.json() - current = resolve_path(definition, urlparse(self._definition['$ref']).fragment) - code_generator = CodeGenerator(current, name) - self._code.insert(0, code_generator.func_code + '\n') - self._validation_functions.add(name) - for key, value in code_generator._compile_regexps.items(): - self._compile_regexps[key] = value + ref = self._definition['$ref'] + with self._resolver.in_scope(ref): + name = self._resolver.get_scope_name() + if 'validate' == name: + name = self._name + uri = self._resolver.get_uri() + if uri not in self._validation_functions_done: + self._validation_functions[uri] = name self.l('{}({variable})', name) - elif self._definition['$ref'] == '#': - self.l('{}({variable})', self._name) - elif self._definition['$ref'].startswith('#'): - path = self._definition['$ref'] - current = resolve_path(self._root_definition, path) - self.generate_func_code_block(current, self._variable, self._variable_name, clear_variables=True) - else: - # TODO: Create more functions for any ref and call it here. - raise NotImplementedError('Local ref is not supported yet') + + def generate_defitions(self): + pass def generate_type(self): """ diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py new file mode 100644 index 0000000..d41ece1 --- /dev/null +++ b/fastjsonschema/ref_resolver.py @@ -0,0 +1,157 @@ +""" +Adapted from https://github.com/Julian/jsonschema +""" +import contextlib +import re +from urllib import parse as urlparse +from urllib.parse import unquote +from urllib.request import urlopen +import json + +import requests + +from .exceptions import JsonSchemaException + + +def resolve_path(schema, fragment): + """ + Return definition from path. + + Path is unescaped according https://tools.ietf.org/html/rfc6901 + + :argument schema: the referrant schema document + :argument str fragment: a URI fragment to resolve within it + :returns: the retrieved schema definition + + """ + fragment = fragment.lstrip('/') + parts = unquote(fragment).split('/') if fragment else [] + for part in parts: + part = part.replace('~1', '/').replace('~0', '~') + if isinstance(schema, list): + schema = schema[int(part)] + elif part in schema: + schema = schema[part] + else: + raise JsonSchemaException('Unresolvable ref: {}'.format(part)) + return schema + + +def normalize(uri): + return urlparse.urlsplit(uri).geturl() + + +def resolve_remote(uri, handlers): + """ + Resolve a remote ``uri``. + + .. note:: + + Requests_ library is used to fetch ``http`` or ``https`` + requests from the remote ``uri``, if handlers does not + define otherwise. + + For unknown schemes urlib is used with UTF-8 encoding. + + .. _Requests: http://pypi.python.org/pypi/requests/ + + :argument str uri: the URI to resolve + :argument dict handlers: the URI resolver functions for each scheme + :returns: the retrieved schema document + + """ + scheme = urlparse.urlsplit(uri).scheme + if scheme in handlers: + result = handlers[scheme](uri) + elif scheme in ['http', 'https']: + result = requests.get(uri).json() + else: + result = json.loads(urlopen(uri).read().decode('utf-8')) + return result + + +class RefResolver(object): + """ + Resolve JSON References. + + :argument str base_uri: URI of the referring document + :argument schema: the actual referring schema document + :argument dict store: a mapping from URIs to documents to cache + :argument bool cache: whether remote refs should be cached after + first resolution + :argument dict handlers: a mapping from URI schemes to functions that + should be used to retrieve them + + """ + + def __init__(self, base_uri, schema, store=(), cache=True, handlers=()): + self.base_uri = base_uri + self.resolution_scope = base_uri + self.schema = schema + self.store = dict(store) + self.cache = cache + self.handlers = dict(handlers) + + @classmethod + def from_schema(cls, schema, *args, **kwargs): + """ + Construct a resolver from a JSON schema object. + + :argument schema schema: the referring schema + :rtype: :class:`RefResolver` + + """ + return cls(schema.get('id', ''), schema, *args, **kwargs) + + @contextlib.contextmanager + def in_scope(self, scope): + old_scope = self.resolution_scope + self.resolution_scope = urlparse.urljoin(old_scope, scope) + try: + yield + finally: + self.resolution_scope = old_scope + + @contextlib.contextmanager + def resolving(self, ref): + """ + Context manager which resolves a JSON ``ref`` and enters the + resolution scope of this ref. + + :argument str ref: reference to resolve + + """ + new_uri = urlparse.urljoin(self.resolution_scope, ref) + uri, fragment = urlparse.urldefrag(new_uri) + print(new_uri, uri, fragment, sep='\n') + + if normalize(uri) in self.store: + schema = self.store[normalize(uri)] + elif not uri or uri == self.base_uri: + schema = self.schema + else: + schema = resolve_remote(uri, self.handlers) + if self.cache: + self.store[normalize(uri)] = schema + + old_base_uri, old_schema = self.base_uri, self.schema + self.base_uri, self.schema = uri, schema + try: + with self.in_scope(uri): + yield resolve_path(schema, fragment) + finally: + self.base_uri, self.schema = old_base_uri, old_schema + + def store_id(self, schema): + id = self.resolution_scope + if normalize(id) not in self.store: + self.store[normalize(id)] = schema + + def get_uri(self): + return normalize(self.resolution_scope) + + def get_scope_name(self): + name = 'validate_' + unquote(self.resolution_scope).replace('~1', '_').replace('~0', '_') + name = re.sub(r'[:/#.-%]', '_', name) + name = name.lower().rstrip('_') + return name From 8b8727ccd4f1428d1516107655555d34119faa82 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Thu, 14 Jun 2018 05:44:07 +0300 Subject: [PATCH 047/201] fix ref_resolver + small refactoring: extract run_generate_functions --- fastjsonschema/generator.py | 16 ++++++++-------- fastjsonschema/ref_resolver.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index c68cbdf..7b84a1b 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -208,19 +208,19 @@ def generate_func_code_block(self, definition, variable, variable_name, clear_va elif 'id' in definition: id = definition['id'] with self._resolver.in_scope(id): - self._resolver.store_id(self._definition) - for key, func in self._json_keywords_to_function.items(): - if key in definition: - func() - else: - for key, func in self._json_keywords_to_function.items(): - if key in definition: - func() + self.run_generate_functions(definition) + else: + self.run_generate_functions(definition) self._definition, self._variable, self._variable_name = backup if clear_variables: self._variables = backup_variables + def run_generate_functions(self, definition): + for key, func in self._json_keywords_to_function.items(): + if key in definition: + func() + def generate_ref(self): """ Ref can be link to remote or local definition. diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index d41ece1..48b0cfa 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -152,6 +152,6 @@ def get_uri(self): def get_scope_name(self): name = 'validate_' + unquote(self.resolution_scope).replace('~1', '_').replace('~0', '_') - name = re.sub(r'[:/#.-%]', '_', name) + name = re.sub('[:/#\.\-\%]', '_', name) name = name.lower().rstrip('_') return name From 4cf248c65fa3eebf174524ef7a7beb10119f3af5 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Thu, 14 Jun 2018 12:23:06 +0300 Subject: [PATCH 048/201] Recursive references between schemas works now --- fastjsonschema/generator.py | 28 ++++++++++++++++++++-------- fastjsonschema/ref_resolver.py | 1 - tests/test_json_schema_test_suits.py | 1 - 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 7b84a1b..df87ae5 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -63,6 +63,8 @@ def __init__(self, definition, name='func', resolver=None): if resolver == None: resolver = RefResolver.from_schema(definition) self._resolver = resolver + if 'id' in definition: + self.generate_validation_function() self._json_keywords_to_function = OrderedDict(( ('definitions', self.generate_defitions), @@ -183,13 +185,14 @@ def generate_func_code(self, definition, name): with self.l('def {}(data):', name): self.generate_func_code_block(definition, 'data', 'data') self.l('return data') + # Generate parts that are referenced and not yet generated while len(self._validation_functions) > 0: uri, name = self._validation_functions.popitem() self._validation_functions_done.add(uri) self.l('') with self._resolver.resolving(uri) as definition: with self.l('def {}(data):', name): - self.generate_func_code_block(definition, 'data', 'data') + self.generate_func_code_block(definition, 'data', 'data', clear_variables=True) self.l('return data') def generate_func_code_block(self, definition, variable, variable_name, clear_variables=False): @@ -237,16 +240,24 @@ def generate_ref(self): """ ref = self._definition['$ref'] with self._resolver.in_scope(ref): - name = self._resolver.get_scope_name() - if 'validate' == name: - name = self._name - uri = self._resolver.get_uri() - if uri not in self._validation_functions_done: - self._validation_functions[uri] = name + name = self.generate_validation_function() self.l('{}({variable})', name) + def generate_validation_function(self): + name = self._resolver.get_scope_name() + if 'validate' == name: + name = self._name + uri = self._resolver.get_uri() + if uri not in self._validation_functions_done: + self._validation_functions[uri] = name + return name + def generate_defitions(self): - pass + definitions = self._definition['definitions'] + for _, value in definitions.items(): + if 'id' in value: + id = value['id'] + self._resolver.store[id] = value def generate_type(self): """ @@ -267,6 +278,7 @@ def generate_type(self): with self.l('if not isinstance({variable}, ({})){}:', python_types, extra): self.l('raise JsonSchemaException("{name} must be {}")', ' or '.join(types)) + def generate_enum(self): """ Means that only value specified in the enum is valid. diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index 48b0cfa..9575ca9 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -123,7 +123,6 @@ def resolving(self, ref): """ new_uri = urlparse.urljoin(self.resolution_scope, ref) uri, fragment = urlparse.urldefrag(new_uri) - print(new_uri, uri, fragment, sep='\n') if normalize(uri) in self.store: schema = self.store[normalize(uri)] diff --git a/tests/test_json_schema_test_suits.py b/tests/test_json_schema_test_suits.py index f645f12..6a1b63f 100644 --- a/tests/test_json_schema_test_suits.py +++ b/tests/test_json_schema_test_suits.py @@ -10,7 +10,6 @@ def pytest_generate_tests(metafunc): suite_dir = 'JSON-Schema-Test-Suite/tests/draft4' ignored_suite_files = [ 'ecmascript-regex.json', - 'ref.json', 'refRemote.json', ] From 95631561836288fe58c111675a45582fbeee4813 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Thu, 14 Jun 2018 19:26:09 +0300 Subject: [PATCH 049/201] Test test_json_schema_test_suits.py fakes now needed adresses at localhost... separate Flast server from JSON-Schema-Test-Suite is not needed anymore... .vscode added in .gitignore --- .gitignore | 2 +- fastjsonschema/__init__.py | 6 ++++-- fastjsonschema/ref_resolver.py | 12 ++++++----- tests/test_json_schema_test_suits.py | 30 +++++++++++++++++++++++++--- 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index ebb5ea6..12459fc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ *.pyc .cache - +.vscode .testmondata build/ diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 57bc399..2d91984 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -47,11 +47,12 @@ from .exceptions import JsonSchemaException from .generator import CodeGenerator +from .ref_resolver import RefResolver __all__ = ('JsonSchemaException', 'compile') -def compile(definition): +def compile(definition, handlers={}): """ Generates validation function for validating JSON schema by ``definition``. Example: @@ -78,7 +79,8 @@ def compile(definition): Exception :any:`JsonSchemaException` is thrown when validation fails. """ - code_generator = CodeGenerator(definition) + resolver = RefResolver.from_schema(definition, handlers=handlers) + code_generator = CodeGenerator(definition, resolver=resolver) # Do not pass local state so it can recursively call itself. global_state = code_generator.global_state exec(code_generator.func_code, global_state) diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index 9575ca9..8cdefe5 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -1,5 +1,7 @@ """ -Adapted from https://github.com/Julian/jsonschema +JSON Schema URI resolution scopes and dereferencing + +Code adapted from https://github.com/Julian/jsonschema """ import contextlib import re @@ -84,16 +86,16 @@ class RefResolver(object): """ - def __init__(self, base_uri, schema, store=(), cache=True, handlers=()): + def __init__(self, base_uri, schema, store=(), cache=True, handlers={}): self.base_uri = base_uri self.resolution_scope = base_uri self.schema = schema self.store = dict(store) self.cache = cache - self.handlers = dict(handlers) + self.handlers = handlers @classmethod - def from_schema(cls, schema, *args, **kwargs): + def from_schema(cls, schema, handlers={}, **kwargs): """ Construct a resolver from a JSON schema object. @@ -101,7 +103,7 @@ def from_schema(cls, schema, *args, **kwargs): :rtype: :class:`RefResolver` """ - return cls(schema.get('id', ''), schema, *args, **kwargs) + return cls(schema.get('id', ''), schema, handlers=handlers, **kwargs) @contextlib.contextmanager def in_scope(self, scope): diff --git a/tests/test_json_schema_test_suits.py b/tests/test_json_schema_test_suits.py index 6a1b63f..802d484 100644 --- a/tests/test_json_schema_test_suits.py +++ b/tests/test_json_schema_test_suits.py @@ -2,15 +2,36 @@ from pathlib import Path import pytest +import requests from fastjsonschema import CodeGenerator, JsonSchemaException, compile +remotes = { + "http://localhost:1234/integer.json": {u"type": u"integer"}, + "http://localhost:1234/name.json": { + u"type": "string", + u"definitions": { + u"orNull": {u"anyOf": [{u"type": u"null"}, {u"$ref": u"#"}]}, + }, + }, + "http://localhost:1234/subSchemas.json": { + u"integer": {u"type": u"integer"}, + u"refToInteger": {u"$ref": u"#/integer"}, + }, + "http://localhost:1234/folder/folderInteger.json": {u"type": u"integer"} +} +def remotes_handler(uri): + if uri in remotes: + return remotes[uri] + return requests.get(uri).json() def pytest_generate_tests(metafunc): suite_dir = 'JSON-Schema-Test-Suite/tests/draft4' ignored_suite_files = [ 'ecmascript-regex.json', - 'refRemote.json', + ] + ignore_tests = [ + "base URI change - change folder in subschema", ] suite_dir_path = Path(suite_dir).resolve() @@ -28,7 +49,10 @@ def pytest_generate_tests(metafunc): test_case['schema'], test_data['data'], test_data['valid'], - marks=pytest.mark.xfail if test_file_path.name in ignored_suite_files else pytest.mark.none, + marks=pytest.mark.xfail + if test_file_path.name in ignored_suite_files + or test_case['description'] in ignore_tests + else pytest.mark.none, )) param_ids.append('{} / {} / {}'.format( test_file_path.name, @@ -43,7 +67,7 @@ def test(schema, data, is_valid): # For debug purposes. When test fails, it will print stdout. print(CodeGenerator(schema).func_code) - validate = compile(schema) + validate = compile(schema, handlers={'http': remotes_handler}) try: result = validate(data) print('Validate result:', result) From 3c1a7c2ff35110dd1fa43379263ddb526568273a Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Thu, 14 Jun 2018 22:06:57 +0300 Subject: [PATCH 050/201] Fix for tests - Printing results when tests fail make them actually fail :) --- tests/test_json_schema_test_suits.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_json_schema_test_suits.py b/tests/test_json_schema_test_suits.py index 802d484..68e0210 100644 --- a/tests/test_json_schema_test_suits.py +++ b/tests/test_json_schema_test_suits.py @@ -4,7 +4,7 @@ import pytest import requests -from fastjsonschema import CodeGenerator, JsonSchemaException, compile +from fastjsonschema import CodeGenerator, RefResolver, JsonSchemaException, compile remotes = { "http://localhost:1234/integer.json": {u"type": u"integer"}, @@ -21,6 +21,7 @@ "http://localhost:1234/folder/folderInteger.json": {u"type": u"integer"} } def remotes_handler(uri): + print(uri) if uri in remotes: return remotes[uri] return requests.get(uri).json() @@ -65,7 +66,8 @@ def pytest_generate_tests(metafunc): def test(schema, data, is_valid): # For debug purposes. When test fails, it will print stdout. - print(CodeGenerator(schema).func_code) + resolver = RefResolver.from_schema(schema, handlers={'http': remotes_handler}) + print(CodeGenerator(schema, resolver=resolver).func_code) validate = compile(schema, handlers={'http': remotes_handler}) try: From 8de0ad23dc1b3bbd9b2c5253615bd0cbb58e2379 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Tue, 12 Jun 2018 21:28:24 +0300 Subject: [PATCH 051/201] Implementation for ``definitions`` and ``ref`` RefResolver is adapted from https://github.com/Julian/jsonschema. There are still some issues on with URI resolution scopes and dereferencing http://json-schema.org/draft-04/json-schema-core.html#rfc.section.7 Tests against draft-04: =============== 619 passed, 3 xfailed in 9.33 seconds =============== tests marked with xfailed: - ecmascript-regex.json file with 1 test case - "base URI change - change folder in subschema" in refRemote.json with 2 test cases Other smaller changes included: - Test test_json_schema_test_suits.py make fake replies on localhost queries. Flask server from JSON-Schema-Test-Suite is not needed anymore :) - .vscode added in .gitignore --- .gitignore | 2 +- fastjsonschema/__init__.py | 6 +- fastjsonschema/generator.py | 82 +++++++++++--- fastjsonschema/ref_resolver.py | 158 +++++++++++++++++++++++++++ tests/test_json_schema_test_suits.py | 38 +++++-- 5 files changed, 258 insertions(+), 28 deletions(-) create mode 100644 fastjsonschema/ref_resolver.py diff --git a/.gitignore b/.gitignore index ebb5ea6..12459fc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ *.pyc .cache - +.vscode .testmondata build/ diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 57bc399..2d91984 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -47,11 +47,12 @@ from .exceptions import JsonSchemaException from .generator import CodeGenerator +from .ref_resolver import RefResolver __all__ = ('JsonSchemaException', 'compile') -def compile(definition): +def compile(definition, handlers={}): """ Generates validation function for validating JSON schema by ``definition``. Example: @@ -78,7 +79,8 @@ def compile(definition): Exception :any:`JsonSchemaException` is thrown when validation fails. """ - code_generator = CodeGenerator(definition) + resolver = RefResolver.from_schema(definition, handlers=handlers) + code_generator = CodeGenerator(definition, resolver=resolver) # Do not pass local state so it can recursively call itself. global_state = code_generator.global_state exec(code_generator.func_code, global_state) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 179d504..df87ae5 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -12,6 +12,7 @@ from .exceptions import JsonSchemaException from .indent import indent +from .ref_resolver import RefResolver def enforce_list(variable): @@ -45,7 +46,7 @@ class CodeGenerator: 'object': 'dict', } - def __init__(self, definition): + def __init__(self, definition, name='func', resolver=None): self._code = [] self._compile_regexps = {} @@ -55,9 +56,18 @@ def __init__(self, definition): self._variable_name = None self._root_definition = definition self._definition = None + self._name = name + + self._validation_functions = {} + self._validation_functions_done = set() + if resolver == None: + resolver = RefResolver.from_schema(definition) + self._resolver = resolver + if 'id' in definition: + self.generate_validation_function() self._json_keywords_to_function = OrderedDict(( - ('$ref', self.generate_ref), + ('definitions', self.generate_defitions), ('type', self.generate_type), ('enum', self.generate_enum), ('allOf', self.generate_all_of), @@ -84,7 +94,7 @@ def __init__(self, definition): ('dependencies', self.generate_dependencies), )) - self.generate_func_code(definition) + self.generate_func_code(definition, name) @property def func_code(self): @@ -164,15 +174,26 @@ def create_variable_keys(self): self._variables.add(variable_name) self.l('{variable}_keys = set({variable}.keys())') - def generate_func_code(self, definition): + def generate_func_code(self, definition, name): """ Creates base code of validation function and calls helper for creating code by definition. """ - with self.l('def func(data):'): - self.l('NoneType = type(None)') + self._validation_functions_done.add(self._resolver.get_uri()) + self.l('NoneType = type(None)') + self.l('') + with self.l('def {}(data):', name): self.generate_func_code_block(definition, 'data', 'data') self.l('return data') + # Generate parts that are referenced and not yet generated + while len(self._validation_functions) > 0: + uri, name = self._validation_functions.popitem() + self._validation_functions_done.add(uri) + self.l('') + with self._resolver.resolving(uri) as definition: + with self.l('def {}(data):', name): + self.generate_func_code_block(definition, 'data', 'data', clear_variables=True) + self.l('return data') def generate_func_code_block(self, definition, variable, variable_name, clear_variables=False): """ @@ -184,14 +205,25 @@ def generate_func_code_block(self, definition, variable, variable_name, clear_va backup_variables = self._variables self._variables = set() - for key, func in self._json_keywords_to_function.items(): - if key in definition: - func() + if '$ref' in definition: + # needed because ref overrides any sibling keywords + self.generate_ref() + elif 'id' in definition: + id = definition['id'] + with self._resolver.in_scope(id): + self.run_generate_functions(definition) + else: + self.run_generate_functions(definition) self._definition, self._variable, self._variable_name = backup if clear_variables: self._variables = backup_variables + def run_generate_functions(self, definition): + for key, func in self._json_keywords_to_function.items(): + if key in definition: + func() + def generate_ref(self): """ Ref can be link to remote or local definition. @@ -206,15 +238,26 @@ def generate_ref(self): } } """ - if self._definition['$ref'].startswith('http'): - res = requests.get(self._definition['$ref']) - definition = res.json() - self.generate_func_code_block(definition, self._variable, self._variable_name) - elif self._definition['$ref'] == '#': - self.l('func({variable})') - else: - #TODO: Create more functions for any ref and call it here. - raise NotImplementedError('Local ref is not supported yet') + ref = self._definition['$ref'] + with self._resolver.in_scope(ref): + name = self.generate_validation_function() + self.l('{}({variable})', name) + + def generate_validation_function(self): + name = self._resolver.get_scope_name() + if 'validate' == name: + name = self._name + uri = self._resolver.get_uri() + if uri not in self._validation_functions_done: + self._validation_functions[uri] = name + return name + + def generate_defitions(self): + definitions = self._definition['definitions'] + for _, value in definitions.items(): + if 'id' in value: + id = value['id'] + self._resolver.store[id] = value def generate_type(self): """ @@ -235,6 +278,7 @@ def generate_type(self): with self.l('if not isinstance({variable}, ({})){}:', python_types, extra): self.l('raise JsonSchemaException("{name} must be {}")', ' or '.join(types)) + def generate_enum(self): """ Means that only value specified in the enum is valid. @@ -365,6 +409,8 @@ def generate_format(self): self._generate_format('ipv4', 'ipv4_re_pattern', r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$') self._generate_format('ipv6', 'ipv6_re_pattern', r'^(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|::(?:[0-9A-Fa-f]{1,4}:){5}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){4}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){3}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,2}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){2}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,3}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}:(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,4}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,5}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}|(?:(?:[0-9A-Fa-f]{1,4}:){,6}[0-9A-Fa-f]{1,4})?::)$') self._generate_format('hostname', 'hostname_re_pattern', r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{1,62}[A-Za-z0-9])$') + # TODO real pattern for regex + self._generate_format('regex', 'regex_re_pattern', r'^.+$') def _generate_format(self, format_name, regexp_name, regexp): if self._definition['format'] == format_name: diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py new file mode 100644 index 0000000..8cdefe5 --- /dev/null +++ b/fastjsonschema/ref_resolver.py @@ -0,0 +1,158 @@ +""" +JSON Schema URI resolution scopes and dereferencing + +Code adapted from https://github.com/Julian/jsonschema +""" +import contextlib +import re +from urllib import parse as urlparse +from urllib.parse import unquote +from urllib.request import urlopen +import json + +import requests + +from .exceptions import JsonSchemaException + + +def resolve_path(schema, fragment): + """ + Return definition from path. + + Path is unescaped according https://tools.ietf.org/html/rfc6901 + + :argument schema: the referrant schema document + :argument str fragment: a URI fragment to resolve within it + :returns: the retrieved schema definition + + """ + fragment = fragment.lstrip('/') + parts = unquote(fragment).split('/') if fragment else [] + for part in parts: + part = part.replace('~1', '/').replace('~0', '~') + if isinstance(schema, list): + schema = schema[int(part)] + elif part in schema: + schema = schema[part] + else: + raise JsonSchemaException('Unresolvable ref: {}'.format(part)) + return schema + + +def normalize(uri): + return urlparse.urlsplit(uri).geturl() + + +def resolve_remote(uri, handlers): + """ + Resolve a remote ``uri``. + + .. note:: + + Requests_ library is used to fetch ``http`` or ``https`` + requests from the remote ``uri``, if handlers does not + define otherwise. + + For unknown schemes urlib is used with UTF-8 encoding. + + .. _Requests: http://pypi.python.org/pypi/requests/ + + :argument str uri: the URI to resolve + :argument dict handlers: the URI resolver functions for each scheme + :returns: the retrieved schema document + + """ + scheme = urlparse.urlsplit(uri).scheme + if scheme in handlers: + result = handlers[scheme](uri) + elif scheme in ['http', 'https']: + result = requests.get(uri).json() + else: + result = json.loads(urlopen(uri).read().decode('utf-8')) + return result + + +class RefResolver(object): + """ + Resolve JSON References. + + :argument str base_uri: URI of the referring document + :argument schema: the actual referring schema document + :argument dict store: a mapping from URIs to documents to cache + :argument bool cache: whether remote refs should be cached after + first resolution + :argument dict handlers: a mapping from URI schemes to functions that + should be used to retrieve them + + """ + + def __init__(self, base_uri, schema, store=(), cache=True, handlers={}): + self.base_uri = base_uri + self.resolution_scope = base_uri + self.schema = schema + self.store = dict(store) + self.cache = cache + self.handlers = handlers + + @classmethod + def from_schema(cls, schema, handlers={}, **kwargs): + """ + Construct a resolver from a JSON schema object. + + :argument schema schema: the referring schema + :rtype: :class:`RefResolver` + + """ + return cls(schema.get('id', ''), schema, handlers=handlers, **kwargs) + + @contextlib.contextmanager + def in_scope(self, scope): + old_scope = self.resolution_scope + self.resolution_scope = urlparse.urljoin(old_scope, scope) + try: + yield + finally: + self.resolution_scope = old_scope + + @contextlib.contextmanager + def resolving(self, ref): + """ + Context manager which resolves a JSON ``ref`` and enters the + resolution scope of this ref. + + :argument str ref: reference to resolve + + """ + new_uri = urlparse.urljoin(self.resolution_scope, ref) + uri, fragment = urlparse.urldefrag(new_uri) + + if normalize(uri) in self.store: + schema = self.store[normalize(uri)] + elif not uri or uri == self.base_uri: + schema = self.schema + else: + schema = resolve_remote(uri, self.handlers) + if self.cache: + self.store[normalize(uri)] = schema + + old_base_uri, old_schema = self.base_uri, self.schema + self.base_uri, self.schema = uri, schema + try: + with self.in_scope(uri): + yield resolve_path(schema, fragment) + finally: + self.base_uri, self.schema = old_base_uri, old_schema + + def store_id(self, schema): + id = self.resolution_scope + if normalize(id) not in self.store: + self.store[normalize(id)] = schema + + def get_uri(self): + return normalize(self.resolution_scope) + + def get_scope_name(self): + name = 'validate_' + unquote(self.resolution_scope).replace('~1', '_').replace('~0', '_') + name = re.sub('[:/#\.\-\%]', '_', name) + name = name.lower().rstrip('_') + return name diff --git a/tests/test_json_schema_test_suits.py b/tests/test_json_schema_test_suits.py index 0ee5089..c80b894 100644 --- a/tests/test_json_schema_test_suits.py +++ b/tests/test_json_schema_test_suits.py @@ -2,17 +2,37 @@ from pathlib import Path import pytest +import requests -from fastjsonschema import CodeGenerator, JsonSchemaException, compile +from fastjsonschema import CodeGenerator, RefResolver, JsonSchemaException, compile +remotes = { + 'http://localhost:1234/integer.json': {'type': 'integer'}, + 'http://localhost:1234/name.json': { + 'type': 'string', + 'definitions': { + 'orNull': {'anyOf': [{'type': 'null'}, {'$ref': '#'}]}, + }, + }, + 'http://localhost:1234/subSchemas.json': { + 'integer': {'type': 'integer'}, + 'refToInteger': {'$ref': '#/integer'}, + }, + 'http://localhost:1234/folder/folderInteger.json': {'type': 'integer'} +} +def remotes_handler(uri): + print(uri) + if uri in remotes: + return remotes[uri] + return requests.get(uri).json() def pytest_generate_tests(metafunc): suite_dir = 'JSON-Schema-Test-Suite/tests/draft4' ignored_suite_files = [ - 'definitions.json', 'ecmascript-regex.json', - 'ref.json', - 'refRemote.json', + ] + ignore_tests = [ + "base URI change - change folder in subschema", ] suite_dir_path = Path(suite_dir).resolve() @@ -30,7 +50,10 @@ def pytest_generate_tests(metafunc): test_case['schema'], test_data['data'], test_data['valid'], - marks=pytest.mark.xfail if test_file_path.name in ignored_suite_files else pytest.mark.none, + marks=pytest.mark.xfail + if test_file_path.name in ignored_suite_files + or test_case['description'] in ignore_tests + else pytest.mark.none, )) param_ids.append('{} / {} / {}'.format( test_file_path.name, @@ -43,9 +66,10 @@ def pytest_generate_tests(metafunc): def test(schema, data, is_valid): # For debug purposes. When test fails, it will print stdout. - print(CodeGenerator(schema).func_code) + resolver = RefResolver.from_schema(schema, handlers={'http': remotes_handler}) + print(CodeGenerator(schema, resolver=resolver).func_code) - validate = compile(schema) + validate = compile(schema, handlers={'http': remotes_handler}) try: result = validate(data) print('Validate result:', result) From 3b964b039080aed4c9f3efc351b20fe8c84f7c4a Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Tue, 19 Jun 2018 14:42:53 +0300 Subject: [PATCH 052/201] Ref resolving moved on ref_resolver.walk function, all ref test cases pass now --- fastjsonschema/__init__.py | 2 +- fastjsonschema/generator.py | 70 +++++++++++++--------------- fastjsonschema/ref_resolver.py | 26 ++++++++++- tests/test_json_schema_test_suits.py | 1 - 4 files changed, 57 insertions(+), 42 deletions(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 2d91984..bedf065 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -84,4 +84,4 @@ def compile(definition, handlers={}): # Do not pass local state so it can recursively call itself. global_state = code_generator.global_state exec(code_generator.func_code, global_state) - return global_state['func'] + return global_state['validate'] diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index df87ae5..955287d 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -46,7 +46,7 @@ class CodeGenerator: 'object': 'dict', } - def __init__(self, definition, name='func', resolver=None): + def __init__(self, definition, resolver=None): self._code = [] self._compile_regexps = {} @@ -56,18 +56,20 @@ def __init__(self, definition, name='func', resolver=None): self._variable_name = None self._root_definition = definition self._definition = None - self._name = name - self._validation_functions = {} + # map schema URIs to validation function names for functions + # that are not yet generated, but need to be generated + self._needed_validation_functions = {} + # validation function names that are already done self._validation_functions_done = set() + if resolver == None: resolver = RefResolver.from_schema(definition) self._resolver = resolver if 'id' in definition: - self.generate_validation_function() + self._needed_validation_functions[self._resolver.get_uri()] = self._resolver.get_scope_name() self._json_keywords_to_function = OrderedDict(( - ('definitions', self.generate_defitions), ('type', self.generate_type), ('enum', self.generate_enum), ('allOf', self.generate_all_of), @@ -94,7 +96,7 @@ def __init__(self, definition, name='func', resolver=None): ('dependencies', self.generate_dependencies), )) - self.generate_func_code(definition, name) + self.generate_func_code(definition) @property def func_code(self): @@ -174,7 +176,7 @@ def create_variable_keys(self): self._variables.add(variable_name) self.l('{variable}_keys = set({variable}.keys())') - def generate_func_code(self, definition, name): + def generate_func_code(self, definition): """ Creates base code of validation function and calls helper for creating code by definition. @@ -182,18 +184,27 @@ def generate_func_code(self, definition, name): self._validation_functions_done.add(self._resolver.get_uri()) self.l('NoneType = type(None)') self.l('') - with self.l('def {}(data):', name): + with self.l('def validate(data):'): self.generate_func_code_block(definition, 'data', 'data') self.l('return data') # Generate parts that are referenced and not yet generated - while len(self._validation_functions) > 0: - uri, name = self._validation_functions.popitem() + while len(self._needed_validation_functions) > 0: + # Normal for ... in... loop not works here, because new functions + # can be found and therefore added in self._needed_validation_functions + # during it generation and dictionary cannot changed during iteration. + uri, name = self._needed_validation_functions.popitem() self._validation_functions_done.add(uri) - self.l('') - with self._resolver.resolving(uri) as definition: - with self.l('def {}(data):', name): - self.generate_func_code_block(definition, 'data', 'data', clear_variables=True) - self.l('return data') + self.generate_validation_function(uri, name) + + def generate_validation_function(self, uri, name): + """ + Generate validation function for given uri with given name + """ + self.l('') + with self._resolver.resolving(uri) as definition: + with self.l('def {}(data):', name): + self.generate_func_code_block(definition, 'data', 'data', clear_variables=True) + self.l('return data') def generate_func_code_block(self, definition, variable, variable_name, clear_variables=False): """ @@ -208,10 +219,6 @@ def generate_func_code_block(self, definition, variable, variable_name, clear_va if '$ref' in definition: # needed because ref overrides any sibling keywords self.generate_ref() - elif 'id' in definition: - id = definition['id'] - with self._resolver.in_scope(id): - self.run_generate_functions(definition) else: self.run_generate_functions(definition) @@ -238,27 +245,14 @@ def generate_ref(self): } } """ - ref = self._definition['$ref'] - with self._resolver.in_scope(ref): - name = self.generate_validation_function() + with self._resolver.in_scope(self._definition['$ref']): + name = self._resolver.get_scope_name() + uri = self._resolver.get_uri() + if uri not in self._validation_functions_done: + self._needed_validation_functions[uri] = name + # call validation function self.l('{}({variable})', name) - def generate_validation_function(self): - name = self._resolver.get_scope_name() - if 'validate' == name: - name = self._name - uri = self._resolver.get_uri() - if uri not in self._validation_functions_done: - self._validation_functions[uri] = name - return name - - def generate_defitions(self): - definitions = self._definition['definitions'] - for _, value in definitions.items(): - if 'id' in value: - id = value['id'] - self._resolver.store[id] = value - def generate_type(self): """ Validation of type. Can be one type or list of types. diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index 8cdefe5..1420c83 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -1,6 +1,8 @@ """ JSON Schema URI resolution scopes and dereferencing +https://tools.ietf.org/id/draft-zyp-json-schema-04.html#rfc.section.7 + Code adapted from https://github.com/Julian/jsonschema """ import contextlib @@ -86,13 +88,14 @@ class RefResolver(object): """ - def __init__(self, base_uri, schema, store=(), cache=True, handlers={}): + def __init__(self, base_uri, schema, store={}, cache=True, handlers={}): self.base_uri = base_uri self.resolution_scope = base_uri self.schema = schema - self.store = dict(store) + self.store = store self.cache = cache self.handlers = handlers + self.walk(schema) @classmethod def from_schema(cls, schema, handlers={}, **kwargs): @@ -156,3 +159,22 @@ def get_scope_name(self): name = re.sub('[:/#\.\-\%]', '_', name) name = name.lower().rstrip('_') return name + + def walk(self, node: dict): + """ + Walk thru schema and Normalize ``id`` and ``$ref`` instances + """ + if '$ref' in node: + ref = node['$ref'] + node['$ref'] = urlparse.urljoin(self.resolution_scope, ref) + if 'id' in node: + id = node['id'] + with self.in_scope(id): + self.store[normalize(self.resolution_scope)] = node + for _, item in node.items(): + if isinstance(item, dict): + self.walk(item) + else: + for _, item in node.items(): + if isinstance(item, dict): + self.walk(item) diff --git a/tests/test_json_schema_test_suits.py b/tests/test_json_schema_test_suits.py index c80b894..77fd6f7 100644 --- a/tests/test_json_schema_test_suits.py +++ b/tests/test_json_schema_test_suits.py @@ -32,7 +32,6 @@ def pytest_generate_tests(metafunc): 'ecmascript-regex.json', ] ignore_tests = [ - "base URI change - change folder in subschema", ] suite_dir_path = Path(suite_dir).resolve() From 89b100d6dc1c4071e4b70140d4825f7a5d100717 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Tue, 19 Jun 2018 15:29:44 +0300 Subject: [PATCH 053/201] Update documentation about 'definitions' and 'ref' --- fastjsonschema/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index bedf065..b2295b3 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -34,8 +34,6 @@ Note that there are some differences compared to JSON schema standard: - * ``definitions`` and ``ref`` for sharing JSON schema are not implemented yet. Future implementation will - not change the speed. * Regular expressions are full Python ones, not only what JSON schema allows. It's easier to allow everything and also it's faster to compile without limits. So keep in mind that when you will use more advanced regular expression, it may not work with other library. From e2573d84a9179fda990d4312b239937f16f005b0 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Tue, 19 Jun 2018 16:26:12 +0300 Subject: [PATCH 054/201] Better explanation --- fastjsonschema/generator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 955287d..5eacdf3 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -189,9 +189,9 @@ def generate_func_code(self, definition): self.l('return data') # Generate parts that are referenced and not yet generated while len(self._needed_validation_functions) > 0: - # Normal for ... in... loop not works here, because new functions - # can be found and therefore added in self._needed_validation_functions - # during it generation and dictionary cannot changed during iteration. + # During generation of validation function, could be needed to generate + # new one that is added again to `_needed_validation_functions`. + # Therefore usage of while instead of for loop. uri, name = self._needed_validation_functions.popitem() self._validation_functions_done.add(uri) self.generate_validation_function(uri, name) From 51b5229705e6516bb1a7594027cbc1f9445b6c84 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Tue, 19 Jun 2018 16:55:54 +0300 Subject: [PATCH 055/201] main function generation moved in `generate_validation_function` --- fastjsonschema/__init__.py | 4 +++- fastjsonschema/generator.py | 11 +++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index b2295b3..75c2c4d 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -78,8 +78,10 @@ def compile(definition, handlers={}): Exception :any:`JsonSchemaException` is thrown when validation fails. """ resolver = RefResolver.from_schema(definition, handlers=handlers) + # get main function name + name = resolver.get_scope_name() code_generator = CodeGenerator(definition, resolver=resolver) # Do not pass local state so it can recursively call itself. global_state = code_generator.global_state exec(code_generator.func_code, global_state) - return global_state['validate'] + return global_state[name] diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 5eacdf3..f12d634 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -66,8 +66,8 @@ def __init__(self, definition, resolver=None): if resolver == None: resolver = RefResolver.from_schema(definition) self._resolver = resolver - if 'id' in definition: - self._needed_validation_functions[self._resolver.get_uri()] = self._resolver.get_scope_name() + # add main function to `self._needed_validation_functions` + self._needed_validation_functions[self._resolver.get_uri()] = self._resolver.get_scope_name() self._json_keywords_to_function = OrderedDict(( ('type', self.generate_type), @@ -181,25 +181,20 @@ def generate_func_code(self, definition): Creates base code of validation function and calls helper for creating code by definition. """ - self._validation_functions_done.add(self._resolver.get_uri()) self.l('NoneType = type(None)') - self.l('') - with self.l('def validate(data):'): - self.generate_func_code_block(definition, 'data', 'data') - self.l('return data') # Generate parts that are referenced and not yet generated while len(self._needed_validation_functions) > 0: # During generation of validation function, could be needed to generate # new one that is added again to `_needed_validation_functions`. # Therefore usage of while instead of for loop. uri, name = self._needed_validation_functions.popitem() - self._validation_functions_done.add(uri) self.generate_validation_function(uri, name) def generate_validation_function(self, uri, name): """ Generate validation function for given uri with given name """ + self._validation_functions_done.add(uri) self.l('') with self._resolver.resolving(uri) as definition: with self.l('def {}(data):', name): From e4ec4fd501e7ec2a34a6afafb530054ced329fbd Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Tue, 19 Jun 2018 20:08:32 +0300 Subject: [PATCH 056/201] format: regex validated agains python re.compile --- fastjsonschema/generator.py | 8 ++++++-- tests/test_string.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index f12d634..5e95e96 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -398,8 +398,12 @@ def generate_format(self): self._generate_format('ipv4', 'ipv4_re_pattern', r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$') self._generate_format('ipv6', 'ipv6_re_pattern', r'^(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|::(?:[0-9A-Fa-f]{1,4}:){5}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){4}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){3}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,2}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){2}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,3}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}:(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,4}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,5}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}|(?:(?:[0-9A-Fa-f]{1,4}:){,6}[0-9A-Fa-f]{1,4})?::)$') self._generate_format('hostname', 'hostname_re_pattern', r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{1,62}[A-Za-z0-9])$') - # TODO real pattern for regex - self._generate_format('regex', 'regex_re_pattern', r'^.+$') + # format regex is used only in meta schemas + if self._definition['format'] == 'regex': + with self.l('try:'): + self.l('re.compile({variable})') + with self.l('except Exception:'): + self.l('raise JsonSchemaException("{name} must be a valid regex")') def _generate_format(self, format_name, regexp_name, regexp): if self._definition['format'] == format_name: diff --git a/tests/test_string.py b/tests/test_string.py index 93bf123..ee45203 100644 --- a/tests/test_string.py +++ b/tests/test_string.py @@ -60,3 +60,14 @@ def test_pattern(asserter, value, expected): 'type': 'string', 'pattern': '^[ab]*[^ab]+(c{2}|d)$', }, value, expected) + +exc = JsonSchemaException('data must be a valid regex') +@pytest.mark.parametrize('value, expected', [ + ('[a-z]', '[a-z]'), + ('[a-z', exc), +]) +def test_pattern(asserter, value, expected): + asserter({ + 'format': 'regex', + 'type': 'string' + }, value, expected) From 75aa2a92f6be63f4e2918c507f75d1a8c7d9f077 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Wed, 20 Jun 2018 02:34:58 +0300 Subject: [PATCH 057/201] write_code implementation --- fastjsonschema/__init__.py | 24 ++++++++++++++ fastjsonschema/generator.py | 62 +++++++++++++++++++++++++++++-------- 2 files changed, 73 insertions(+), 13 deletions(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 75c2c4d..7985d49 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -85,3 +85,27 @@ def compile(definition, handlers={}): global_state = code_generator.global_state exec(code_generator.func_code, global_state) return global_state[name] + +def write_code(filename, definition, handlers={}): + """ + Generates validation function for validating JSON schema by ``definition`` + and write it to file. Example: + + .. code-block:: python + + import fastjsonschema + + validate = fastjsonschema.write_code('validator.py', {'type': 'string'}) + + Exception :any:`JsonSchemaException` is thrown when validation fails. + """ + resolver = RefResolver.from_schema(definition, handlers=handlers) + # get main function name + name = resolver.get_scope_name() + code_generator = CodeGenerator(definition, resolver=resolver) + # Do not pass local state so it can recursively call itself. + global_state_code = code_generator.global_state_code + with open(filename, 'w', encoding='UTF-8') as out: + print(global_state_code, file=out) + print(code_generator.func_code, file=out) + return name diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 5e95e96..05eb5eb 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -15,6 +15,16 @@ from .ref_resolver import RefResolver +FORMAT_REGEXS = { + 'date-time': r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+(?:[+-][0-2]\d:[0-5]\d|Z)?$', + 'uri': r'^\w+:(\/?\/?)[^\s]+$', + 'email': r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$', + 'ipv4': r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$', + 'ipv6': r'^(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|::(?:[0-9A-Fa-f]{1,4}:){5}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){4}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){3}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,2}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){2}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,3}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}:(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,4}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,5}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}|(?:(?:[0-9A-Fa-f]{1,4}:){,6}[0-9A-Fa-f]{1,4})?::)$', + 'hostname': r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{1,62}[A-Za-z0-9])$', +} + + def enforce_list(variable): if isinstance(variable, list): return variable @@ -113,11 +123,39 @@ def global_state(self): time when validation function is called. """ return dict( - self._compile_regexps, + REGEX_PATTERNS=self._compile_regexps, re=re, JsonSchemaException=JsonSchemaException, ) + @property + def global_state_code(self): + """ + Returns global variables for generating function from ``func_code`` as code. + Includes compiled regular expressions and imports. + """ + if len(self._compile_regexps) == 0: + return '\n'.join( + [ + 'from fastjsonschema import JsonSchemaException', + '', + '', + ] + ) + regexs = ['"{}": {}'.format(key, value) for key, value in self._compile_regexps.items()] + return '\n'.join( + [ + 'import re', + 'from fastjsonschema import JsonSchemaException', + '', + '', + 'REGEX_PATTERNS = {', + ' ' + ',\n '.join(regexs), + '}', + '', + ] + ) + @indent def l(self, line, *args, **kwds): """ @@ -386,20 +424,18 @@ def generate_max_length(self): def generate_pattern(self): with self.l('if isinstance({variable}, str):'): - self._compile_regexps['{}_re'.format(self._definition['pattern'])] = re.compile(self._definition['pattern']) - with self.l('if not globals()["{}_re"].search({variable}):', self._definition['pattern']): + self._compile_regexps['{}'.format(self._definition['pattern'])] = re.compile(self._definition['pattern']) + with self.l('if not REGEX_PATTERNS["{}"].search({variable}):', self._definition['pattern']): self.l('raise JsonSchemaException("{name} must match pattern {pattern}")') def generate_format(self): with self.l('if isinstance({variable}, str):'): - self._generate_format('date-time', 'date_time_re_pattern', r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+(?:[+-][0-2]\d:[0-5]\d|Z)?$') - self._generate_format('uri', 'uri_re_pattern', r'^\w+:(\/?\/?)[^\s]+$') - self._generate_format('email', 'email_re_pattern', r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$') - self._generate_format('ipv4', 'ipv4_re_pattern', r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$') - self._generate_format('ipv6', 'ipv6_re_pattern', r'^(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|::(?:[0-9A-Fa-f]{1,4}:){5}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){4}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){3}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,2}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){2}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,3}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}:(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,4}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,5}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}|(?:(?:[0-9A-Fa-f]{1,4}:){,6}[0-9A-Fa-f]{1,4})?::)$') - self._generate_format('hostname', 'hostname_re_pattern', r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{1,62}[A-Za-z0-9])$') + format = self._definition['format'] + if format in FORMAT_REGEXS: + format_regex = FORMAT_REGEXS[format] + self._generate_format(format, format + '_re_pattern', format_regex) # format regex is used only in meta schemas - if self._definition['format'] == 'regex': + elif format == 'regex': with self.l('try:'): self.l('re.compile({variable})') with self.l('except Exception:'): @@ -408,7 +444,7 @@ def generate_format(self): def _generate_format(self, format_name, regexp_name, regexp): if self._definition['format'] == format_name: self._compile_regexps[regexp_name] = re.compile(regexp) - with self.l('if not {}.match({variable}):', regexp_name): + with self.l('if not REGEX_PATTERNS["{}"].match({variable}):', regexp_name): self.l('raise JsonSchemaException("{name} must be {}")', format_name) def generate_minimum(self): @@ -538,10 +574,10 @@ def generate_pattern_properties(self): with self.l('if isinstance({variable}, dict):'): self.create_variable_keys() for pattern, definition in self._definition['patternProperties'].items(): - self._compile_regexps['{}_re'.format(pattern)] = re.compile(pattern) + self._compile_regexps['{}'.format(pattern)] = re.compile(pattern) with self.l('for key, val in {variable}.items():'): for pattern, definition in self._definition['patternProperties'].items(): - with self.l('if globals()["{}_re"].search(key):', pattern): + with self.l('if REGEX_PATTERNS["{}"].search(key):', pattern): with self.l('if key in {variable}_keys:'): self.l('{variable}_keys.remove(key)') self.generate_func_code_block( From ba27e48d883fb06189510c4adad83c82d980e043 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Wed, 20 Jun 2018 16:09:25 +0300 Subject: [PATCH 058/201] tox.ini + front page README.markdown updates --- .gitignore | 3 +++ README.markdown | 13 +++++++++++++ fastjsonschema/__init__.py | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 12459fc..370406e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ .cache .vscode .testmondata +__pycache__ +.tox +.pytest_cache build/ _build/ diff --git a/README.markdown b/README.markdown index 3193721..ee7c995 100644 --- a/README.markdown +++ b/README.markdown @@ -1,5 +1,18 @@ # Fast JSON schema for Python +This project was made to come up with fast JSON validations. See +documentation https://seznam.github.io/python-fastjsonschema/ for +performance test details. + +Current version is implementation of [json-schema](http://json-schema.org/) draft-04. Note that there are some differences compared to JSON schema standard: + + * Regular expressions are full Python ones, not only what JSON schema + allows. It's easier to allow everything and also it's faster to + compile without limits. So keep in mind that when you will use more advanced regular expression, it may not work with other library. + * JSON schema says you can use keyword ``default`` for providing default + values. This implementation uses that and always returns transformed + input data. + ## Install `pip install fastjsonschema` diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 75c2c4d..9af2200 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -28,7 +28,7 @@ validictory valid ==> 0.6349993739277124 validictory invalid ==> 0.03125431900843978 -This library follows and implements `JSON schema v4 `_. Sometimes +This library follows and implements `JSON schema draft-04 `_. Sometimes it's not perfectly clear so I recommend also check out this `understaning json schema `_. From bb2860d17ae8d87991b8874f26b70ad35c539bbc Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Wed, 20 Jun 2018 18:00:05 +0300 Subject: [PATCH 059/201] tox.ini missed from previous commit --- tox.ini | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tox.ini diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..6eec2d9 --- /dev/null +++ b/tox.ini @@ -0,0 +1,12 @@ +# tox (https://tox.readthedocs.io/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = py33, py34, py35, py36 + +[testenv] +commands = pytest tests +deps = + pytest From 0292b575f50837ac5a409b4d40dae85fcc710463 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Wed, 20 Jun 2018 19:02:18 +0300 Subject: [PATCH 060/201] Json-Schema-Test-Suite test for draft-06 and draft-07 with ignores --- tests/test_json_schema_test_suits.py | 22 ++---- tests/test_json_schema_test_suits_draft4.py | 21 ++++++ tests/test_json_schema_test_suits_draft6.py | 65 ++++++++++++++++++ tests/test_json_schema_test_suits_draft7.py | 74 +++++++++++++++++++++ 4 files changed, 164 insertions(+), 18 deletions(-) create mode 100644 tests/test_json_schema_test_suits_draft4.py create mode 100644 tests/test_json_schema_test_suits_draft6.py create mode 100644 tests/test_json_schema_test_suits_draft7.py diff --git a/tests/test_json_schema_test_suits.py b/tests/test_json_schema_test_suits.py index 77fd6f7..cc62592 100644 --- a/tests/test_json_schema_test_suits.py +++ b/tests/test_json_schema_test_suits.py @@ -1,9 +1,6 @@ import json -from pathlib import Path - import pytest import requests - from fastjsonschema import CodeGenerator, RefResolver, JsonSchemaException, compile remotes = { @@ -26,22 +23,12 @@ def remotes_handler(uri): return remotes[uri] return requests.get(uri).json() -def pytest_generate_tests(metafunc): - suite_dir = 'JSON-Schema-Test-Suite/tests/draft4' - ignored_suite_files = [ - 'ecmascript-regex.json', - ] - ignore_tests = [ - ] - - suite_dir_path = Path(suite_dir).resolve() - test_file_paths = sorted(set(suite_dir_path.glob("**/*.json"))) +def resolve_param_values_and_ids(test_file_paths, ignored_suite_files, ignore_tests): param_values = [] param_ids = [] - for test_file_path in test_file_paths: - with test_file_path.open() as test_file: + with test_file_path.open(encoding='UTF-8') as test_file: test_cases = json.load(test_file) for test_case in test_cases: for test_data in test_case['tests']: @@ -59,11 +46,10 @@ def pytest_generate_tests(metafunc): test_case['description'], test_data['description'], )) - - metafunc.parametrize(['schema', 'data', 'is_valid'], param_values, ids=param_ids) + return param_values, param_ids -def test(schema, data, is_valid): +def template_test(schema, data, is_valid): # For debug purposes. When test fails, it will print stdout. resolver = RefResolver.from_schema(schema, handlers={'http': remotes_handler}) print(CodeGenerator(schema, resolver=resolver).func_code) diff --git a/tests/test_json_schema_test_suits_draft4.py b/tests/test_json_schema_test_suits_draft4.py new file mode 100644 index 0000000..55fd21e --- /dev/null +++ b/tests/test_json_schema_test_suits_draft4.py @@ -0,0 +1,21 @@ +from pathlib import Path +import pytest +from tests.test_json_schema_test_suits import template_test, resolve_param_values_and_ids + + +def pytest_generate_tests(metafunc): + suite_dir = 'JSON-Schema-Test-Suite/tests/draft4' + ignored_suite_files = [ + 'ecmascript-regex.json', + ] + ignore_tests = [] + + suite_dir_path = Path(suite_dir).resolve() + test_file_paths = sorted(set(suite_dir_path.glob("**/*.json"))) + + param_values, param_ids = resolve_param_values_and_ids( + test_file_paths, ignored_suite_files, ignore_tests + ) + metafunc.parametrize(['schema', 'data', 'is_valid'], param_values, ids=param_ids) + +test = template_test diff --git a/tests/test_json_schema_test_suits_draft6.py b/tests/test_json_schema_test_suits_draft6.py new file mode 100644 index 0000000..acef625 --- /dev/null +++ b/tests/test_json_schema_test_suits_draft6.py @@ -0,0 +1,65 @@ +from pathlib import Path +import pytest +from tests.test_json_schema_test_suits import template_test, resolve_param_values_and_ids + +def pytest_generate_tests(metafunc): + suite_dir = 'JSON-Schema-Test-Suite/tests/draft6' + ignored_suite_files = [ + 'bignum.json', + 'ecmascript-regex.json', + 'zeroTerminatedFloats.json', + 'boolean_schema.json', + 'contains.json', + 'const.json', + ] + ignore_tests = [ + 'invalid definition', + 'valid definition', + 'Recursive references between schemas', + 'remote ref, containing refs itself', + 'dependencies with boolean subschemas', + 'dependencies with empty array', + 'exclusiveMaximum validation', + 'exclusiveMinimum validation', + 'format: uri-template', + 'validation of URI References', + 'items with boolean schema (true)', + 'items with boolean schema (false)', + 'items with boolean schema', + 'items with boolean schemas', + 'not with boolean schema true', + 'not with boolean schema false', + 'properties with boolean schema', + 'propertyNames with boolean schema false', + 'propertyNames validation', + 'base URI change - change folder', + 'base URI change - change folder in subschema', + 'base URI change', + 'root ref in remote ref', + 'allOf with boolean schemas, all true', + 'allOf with boolean schemas, some false', + 'allOf with boolean schemas, all false', + 'anyOf with boolean schemas, all true', + 'anyOf with boolean schemas, some false', + 'anyOf with boolean schemas, all false', + 'anyOf with boolean schemas, some true', + 'oneOf with boolean schemas, all true', + 'oneOf with boolean schemas, some false', + 'oneOf with boolean schemas, all false', + 'oneOf with boolean schemas, one true', + 'oneOf with boolean schemas, more than one true', + 'validation of JSON-pointers (JSON String Representation)', + 'patternProperties with boolean schemas', + '$ref to boolean schema true', + '$ref to boolean schema false', + ] + + suite_dir_path = Path(suite_dir).resolve() + test_file_paths = sorted(set(suite_dir_path.glob("**/*.json"))) + + param_values, param_ids = resolve_param_values_and_ids( + test_file_paths, ignored_suite_files, ignore_tests + ) + metafunc.parametrize(['schema', 'data', 'is_valid'], param_values, ids=param_ids) + +test = template_test diff --git a/tests/test_json_schema_test_suits_draft7.py b/tests/test_json_schema_test_suits_draft7.py new file mode 100644 index 0000000..4bc6a03 --- /dev/null +++ b/tests/test_json_schema_test_suits_draft7.py @@ -0,0 +1,74 @@ +from pathlib import Path +import pytest +from tests.test_json_schema_test_suits import template_test, resolve_param_values_and_ids + +def pytest_generate_tests(metafunc): + suite_dir = 'JSON-Schema-Test-Suite/tests/draft7' + ignored_suite_files = [ + 'bignum.json', + 'ecmascript-regex.json', + 'zeroTerminatedFloats.json', + 'boolean_schema.json', + 'contains.json', + 'content.json', + 'if-then-else.json', + 'idn-email.json', + 'idn-hostname.json', + 'iri-reference.json', + 'iri.json', + 'relative-json-pointer.json', + 'time.json', + 'const.json', + ] + ignore_tests = [ + 'invalid definition', + 'valid definition', + 'Recursive references between schemas', + 'remote ref, containing refs itself', + 'dependencies with boolean subschemas', + 'dependencies with empty array', + 'exclusiveMaximum validation', + 'exclusiveMinimum validation', + 'format: uri-template', + 'validation of URI References', + 'items with boolean schema (true)', + 'items with boolean schema (false)', + 'items with boolean schema', + 'items with boolean schemas', + 'not with boolean schema true', + 'not with boolean schema false', + 'properties with boolean schema', + 'propertyNames with boolean schema false', + 'propertyNames validation', + 'base URI change - change folder', + 'base URI change - change folder in subschema', + 'base URI change', + 'root ref in remote ref', + 'validation of date strings', + 'allOf with boolean schemas, all true', + 'allOf with boolean schemas, some false', + 'allOf with boolean schemas, all false', + 'anyOf with boolean schemas, all true', + 'anyOf with boolean schemas, some false', + 'anyOf with boolean schemas, all false', + 'anyOf with boolean schemas, some true', + 'oneOf with boolean schemas, all true', + 'oneOf with boolean schemas, some false', + 'oneOf with boolean schemas, all false', + 'oneOf with boolean schemas, one true', + 'oneOf with boolean schemas, more than one true', + 'validation of JSON-pointers (JSON String Representation)', + 'patternProperties with boolean schemas', + '$ref to boolean schema true', + '$ref to boolean schema false', + ] + + suite_dir_path = Path(suite_dir).resolve() + test_file_paths = sorted(set(suite_dir_path.glob("**/*.json"))) + + param_values, param_ids = resolve_param_values_and_ids( + test_file_paths, ignored_suite_files, ignore_tests + ) + metafunc.parametrize(['schema', 'data', 'is_valid'], param_values, ids=param_ids) + +test = template_test From 5cd9051e3592e8b1953df7d81035a52e6c34f10f Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 20 Jun 2018 16:23:46 +0000 Subject: [PATCH 061/201] v1.5 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 29a83af..711bc48 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='fastjsonschema', - version='1.4', + version='1.5', packages=['fastjsonschema'], install_requires=[ From 7fcf49d24b63d798703e9f8817c04a29579aa12a Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Wed, 20 Jun 2018 19:56:55 +0300 Subject: [PATCH 062/201] Args documentation for compile --- fastjsonschema/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 9af2200..c39e027 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -75,6 +75,11 @@ def compile(definition, handlers={}): data = validate({}) assert data == {'a': 42} + Args: + definition (dict): Json schema definition + handlers (dict): A mapping from URI schemes to functions + that should be used to retrieve them. + Exception :any:`JsonSchemaException` is thrown when validation fails. """ resolver = RefResolver.from_schema(definition, handlers=handlers) From f0c8a0bfe39a2d016c69f72598bc41ef0c1ac917 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Thu, 21 Jun 2018 04:55:09 +0300 Subject: [PATCH 063/201] fix 2 RefResolver bugs, 'id', and '$ref' properties handled as they are actual schema 'id' and '$ref' --- fastjsonschema/ref_resolver.py | 6 +++--- tests/test_object.py | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index 1420c83..c4e3843 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -162,12 +162,12 @@ def get_scope_name(self): def walk(self, node: dict): """ - Walk thru schema and Normalize ``id`` and ``$ref`` instances + Walk thru schema and dereferencing ``id`` and ``$ref`` instances """ - if '$ref' in node: + if '$ref' in node and isinstance(node['$ref'], str): ref = node['$ref'] node['$ref'] = urlparse.urljoin(self.resolution_scope, ref) - if 'id' in node: + elif 'id' in node and isinstance(node['id'], str): id = node['id'] with self.in_scope(id): self.store[normalize(self.resolution_scope)] = node diff --git a/tests/test_object.py b/tests/test_object.py index 8d425b8..c486d52 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -129,3 +129,29 @@ def test_pattern_properties(asserter, value, expected): }, 'additionalProperties': False, }, value, expected) + +@pytest.mark.parametrize('value, expected', [ + ({'id': 1}, {'id': 1}), + ({'id': 'a'}, JsonSchemaException('data.id must be integer')), +]) + +def test_object_with_id_property(asserter, value, expected): + asserter({ + "type": "object", + "properties": { + "id": {"type": "integer"} + } + }, value, expected) + +@pytest.mark.parametrize('value, expected', [ + ({'$ref': 'ref://to.somewhere'}, {'$ref': 'ref://to.somewhere'}), + ({'$ref': 1}, JsonSchemaException('data.$ref must be string')), +]) + +def test_object_with_id_property(asserter, value, expected): + asserter({ + "type": "object", + "properties": { + "$ref": {"type": "string"} + } + }, value, expected) From aa9995e93fa8e98390b5ac32f6f2a440e5433fd3 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Thu, 21 Jun 2018 06:25:37 +0300 Subject: [PATCH 064/201] better documentation --- fastjsonschema/__init__.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 7985d49..20b477b 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -89,16 +89,42 @@ def compile(definition, handlers={}): def write_code(filename, definition, handlers={}): """ Generates validation function for validating JSON schema by ``definition`` - and write it to file. Example: + and write it to file. + + Arguments: + filename (str): Filename where generated code is written + definition (dict): JSON Schema defining validation rules + handlers (dict): A mapping from URI schemes to functions + that should be used to retrieve them. + + Returns: + validation function name (str) + + Raises: + JsonSchemaException: is thrown when generation fails. + + + Create validator Example: .. code-block:: python import fastjsonschema - validate = fastjsonschema.write_code('validator.py', {'type': 'string'}) + schema = {'type': 'string'} + name = fastjsonschema.write_code('validator.py', schema) + print(name) + + Generated file usage Example: + + .. code-block:: python + + from validator import validate + + validate('example') Exception :any:`JsonSchemaException` is thrown when validation fails. """ + resolver = RefResolver.from_schema(definition, handlers=handlers) # get main function name name = resolver.get_scope_name() From 6b74045ef28cb91617248556b3c3c0c4fd3e8b27 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Thu, 21 Jun 2018 11:24:07 +0300 Subject: [PATCH 065/201] fix test name --- tests/test_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_object.py b/tests/test_object.py index c486d52..2c25d89 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -148,7 +148,7 @@ def test_object_with_id_property(asserter, value, expected): ({'$ref': 1}, JsonSchemaException('data.$ref must be string')), ]) -def test_object_with_id_property(asserter, value, expected): +def test_object_with_ref_property(asserter, value, expected): asserter({ "type": "object", "properties": { From f6ae54707d65c70d2993f4159aedda2e0014423b Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Thu, 21 Jun 2018 14:14:42 +0300 Subject: [PATCH 066/201] test for write_code, prevent overwrite if not flagged, .gitignore temp/* added --- .gitignore | 1 + fastjsonschema/__init__.py | 9 +++++++-- fastjsonschema/generator.py | 7 ++++--- tests/test_write_code.py | 24 ++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 tests/test_write_code.py diff --git a/.gitignore b/.gitignore index 12459fc..2dcb41b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ dist/ *.tar.gz *egg-info MANIFEST +temp/* diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 20b477b..7228c69 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -43,11 +43,13 @@ Support only for Python 3.3 and higher. """ +from os.path import exists + from .exceptions import JsonSchemaException from .generator import CodeGenerator from .ref_resolver import RefResolver -__all__ = ('JsonSchemaException', 'compile') +__all__ = ('JsonSchemaException', 'compile', 'write_code') def compile(definition, handlers={}): @@ -86,7 +88,7 @@ def compile(definition, handlers={}): exec(code_generator.func_code, global_state) return global_state[name] -def write_code(filename, definition, handlers={}): +def write_code(filename, definition, handlers={}, overwrite=False): """ Generates validation function for validating JSON schema by ``definition`` and write it to file. @@ -96,6 +98,7 @@ def write_code(filename, definition, handlers={}): definition (dict): JSON Schema defining validation rules handlers (dict): A mapping from URI schemes to functions that should be used to retrieve them. + overwrite (bool): Set to `True` to overwrite existing file Returns: validation function name (str) @@ -125,6 +128,8 @@ def write_code(filename, definition, handlers={}): Exception :any:`JsonSchemaException` is thrown when validation fails. """ + if exists(filename) and overwrite == False: + raise JsonSchemaException('file {} already exists'.format(filename)) resolver = RefResolver.from_schema(definition, handlers=handlers) # get main function name name = resolver.get_scope_name() diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 05eb5eb..58b1f05 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -442,10 +442,11 @@ def generate_format(self): self.l('raise JsonSchemaException("{name} must be a valid regex")') def _generate_format(self, format_name, regexp_name, regexp): - if self._definition['format'] == format_name: + if self._definition['format'] == format_name: + if not regexp_name in self._compile_regexps: self._compile_regexps[regexp_name] = re.compile(regexp) - with self.l('if not REGEX_PATTERNS["{}"].match({variable}):', regexp_name): - self.l('raise JsonSchemaException("{name} must be {}")', format_name) + with self.l('if not REGEX_PATTERNS["{}"].match({variable}):', regexp_name): + self.l('raise JsonSchemaException("{name} must be {}")', format_name) def generate_minimum(self): with self.l('if isinstance({variable}, (int, float)):'): diff --git a/tests/test_write_code.py b/tests/test_write_code.py new file mode 100644 index 0000000..074563a --- /dev/null +++ b/tests/test_write_code.py @@ -0,0 +1,24 @@ +import os +import pytest + +from fastjsonschema import JsonSchemaException, write_code + +def test_write_code(): + if not os.path.isdir('temp'): + os.makedirs('temp') + if os.path.exists('temp/schema.py'): + os.remove('temp/schema.py') + name = write_code( + 'temp/schema.py', + {'properties': {'a': {'type': 'string'}, 'b': {'type': 'integer'}}} + ) + assert name == 'validate' + assert os.path.exists('temp/schema.py') + with pytest.raises(JsonSchemaException) as exc: + name = write_code( + 'temp/schema.py', + {'type': 'string'} + ) + assert exc.value.message == 'file temp/schema.py already exists' + from temp.schema import validate + assert validate({'a': 'a', 'b': 1}) == {'a': 'a', 'b': 1} From ac5b71e74ef1796d22d364e665f0341ab142574f Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Thu, 21 Jun 2018 14:48:54 +0300 Subject: [PATCH 067/201] New performance test: fast_file added. --- performance.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/performance.py b/performance.py index beaed05..d0812f4 100644 --- a/performance.py +++ b/performance.py @@ -73,6 +73,14 @@ fastjsonschema_validate = fastjsonschema.compile(JSON_SCHEMA) fast_compiled = lambda value, _: fastjsonschema_validate(value) fast_not_compiled = lambda value, json_schema: fastjsonschema.compile(json_schema)(value) +validate = fastjsonschema.write_code( + filename='temp/performance.py', + definition=JSON_SCHEMA, + overwrite=True +) +def fast_file(value, schema): + from temp.performance import validate + validate(value) jsonspec = load(JSON_SCHEMA) @@ -88,6 +96,7 @@ def t(func, valid_values=True): jsonschema, jsonspec, fast_compiled, + fast_file, fast_not_compiled, ) """ @@ -115,6 +124,9 @@ def t(func, valid_values=True): t('fast_compiled') t('fast_compiled', valid_values=False) +t('fast_file') +t('fast_file', valid_values=False) + t('fast_not_compiled') t('fast_not_compiled', valid_values=False) From 37669b07b33057e09628814c275efbf382c3ad79 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 21 Jun 2018 17:19:52 +0000 Subject: [PATCH 068/201] v1.6 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 711bc48..b2102d0 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='fastjsonschema', - version='1.5', + version='1.6', packages=['fastjsonschema'], install_requires=[ From f7953c774de9f1319a8740428071c0f0ec3cf617 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Thu, 21 Jun 2018 20:34:59 +0300 Subject: [PATCH 069/201] README changed to rst so it can be directly used in PyPI thru setup.py --- .gitignore | 1 + README.markdown | 24 ------------------------ README.rst | 44 ++++++++++++++++++++++++++++++++++++++++++++ setup.py | 10 ++++++++++ tox.ini | 2 +- 5 files changed, 56 insertions(+), 25 deletions(-) delete mode 100644 README.markdown create mode 100644 README.rst diff --git a/.gitignore b/.gitignore index 370406e..ac4aa36 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ __pycache__ .tox .pytest_cache +.coverage build/ _build/ diff --git a/README.markdown b/README.markdown deleted file mode 100644 index ee7c995..0000000 --- a/README.markdown +++ /dev/null @@ -1,24 +0,0 @@ -# Fast JSON schema for Python - -This project was made to come up with fast JSON validations. See -documentation https://seznam.github.io/python-fastjsonschema/ for -performance test details. - -Current version is implementation of [json-schema](http://json-schema.org/) draft-04. Note that there are some differences compared to JSON schema standard: - - * Regular expressions are full Python ones, not only what JSON schema - allows. It's easier to allow everything and also it's faster to - compile without limits. So keep in mind that when you will use more advanced regular expression, it may not work with other library. - * JSON schema says you can use keyword ``default`` for providing default - values. This implementation uses that and always returns transformed - input data. - -## Install - -`pip install fastjsonschema` - -Support for Python 3.3 and higher. - -## Documentation - -Documentation: https://seznam.github.io/python-fastjsonschema/ diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..6b3c54a --- /dev/null +++ b/README.rst @@ -0,0 +1,44 @@ +=========================== +Fast JSON schema for Python +=========================== + +|PyPI| |Pythons| + +.. |PyPI| image:: https://img.shields.io/pypi/v/fastjsonschema.svg + :alt: PyPI version + :target: https://pypi.python.org/pypi/fastjsonschema + +.. |Pythons| image:: https://img.shields.io/pypi/pyversions/fastjsonschema.svg + :alt: Supported Python versions + :target: https://pypi.python.org/pypi/fastjsonschema + +This project was made to come up with fast JSON validations. It is at +least an order of magnitude faster than other Python implemantaions. +See `documentation `_ for +performance test details. + +Current version is implementation of `json-schema `_ draft-04. +Note that there are some differences compared to JSON schema standard: + +* Regular expressions are full Python ones, not only what JSON schema + allows. It's easier to allow everything and also it's faster to + compile without limits. So keep in mind that when you will use more + advanced regular expression, it may not work with other library. +* JSON schema says you can use keyword ``default`` for providing default + values. This implementation uses that and always returns transformed + input data. + +Install +------- + +.. code-block:: bash + + pip install fastjsonschema + +Support for Python 3.3 and higher. + +Documentation +------------- + +Documentation: `https://seznam.github.io/python-fastjsonschema/ +`_ diff --git a/setup.py b/setup.py index 711bc48..024f5e5 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,13 @@ #!/usr/bin/env python +import os try: from setuptools import setup except ImportError: from distutils.core import setup +with open(os.path.join(os.path.dirname(__file__), "README.rst")) as readme: + long_description = readme.read() setup( name='fastjsonschema', @@ -28,11 +31,18 @@ author='Michal Horejsek', author_email='horejsekmichal@gmail.com', description='Fastest Python implementation of JSON schema', + long_description=long_description, license='BSD', classifiers=[ 'Programming Language :: Python', 'Programming Language :: Python :: 3', + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Development Status :: 5 - Production/Stable', diff --git a/tox.ini b/tox.ini index 6eec2d9..415800c 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py33, py34, py35, py36 +envlist = py33, py34, py35, py36, pypy [testenv] commands = pytest tests From 01ad6769056b215e76f5e9863ba7a4b15d112e98 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Fri, 22 Jun 2018 01:07:26 +0300 Subject: [PATCH 070/201] Working tox.ini --- setup.py | 1 - test-requirements.txt | 3 +++ tests/test_json_schema_test_suits_draft4.py | 2 +- tests/test_json_schema_test_suits_draft6.py | 2 +- tests/test_json_schema_test_suits_draft7.py | 2 +- tox.ini | 9 ++++++--- 6 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 test-requirements.txt diff --git a/setup.py b/setup.py index 27602b8..20ae26b 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,6 @@ "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Development Status :: 5 - Production/Stable', diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..a3fa396 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,3 @@ +-e . +pytest +sphinx diff --git a/tests/test_json_schema_test_suits_draft4.py b/tests/test_json_schema_test_suits_draft4.py index 55fd21e..4c87f12 100644 --- a/tests/test_json_schema_test_suits_draft4.py +++ b/tests/test_json_schema_test_suits_draft4.py @@ -1,6 +1,6 @@ from pathlib import Path import pytest -from tests.test_json_schema_test_suits import template_test, resolve_param_values_and_ids +from test_json_schema_test_suits import template_test, resolve_param_values_and_ids def pytest_generate_tests(metafunc): diff --git a/tests/test_json_schema_test_suits_draft6.py b/tests/test_json_schema_test_suits_draft6.py index acef625..948d99a 100644 --- a/tests/test_json_schema_test_suits_draft6.py +++ b/tests/test_json_schema_test_suits_draft6.py @@ -1,6 +1,6 @@ from pathlib import Path import pytest -from tests.test_json_schema_test_suits import template_test, resolve_param_values_and_ids +from test_json_schema_test_suits import template_test, resolve_param_values_and_ids def pytest_generate_tests(metafunc): suite_dir = 'JSON-Schema-Test-Suite/tests/draft6' diff --git a/tests/test_json_schema_test_suits_draft7.py b/tests/test_json_schema_test_suits_draft7.py index 4bc6a03..b149d5a 100644 --- a/tests/test_json_schema_test_suits_draft7.py +++ b/tests/test_json_schema_test_suits_draft7.py @@ -1,6 +1,6 @@ from pathlib import Path import pytest -from tests.test_json_schema_test_suits import template_test, resolve_param_values_and_ids +from test_json_schema_test_suits import template_test, resolve_param_values_and_ids def pytest_generate_tests(metafunc): suite_dir = 'JSON-Schema-Test-Suite/tests/draft7' diff --git a/tox.ini b/tox.ini index 415800c..fe0f0fe 100644 --- a/tox.ini +++ b/tox.ini @@ -4,9 +4,12 @@ # and then run "tox" from this directory. [tox] -envlist = py33, py34, py35, py36, pypy +envlist = py{34,35,36} [testenv] -commands = pytest tests -deps = +whitelist_externals = + pip + pytest +commands = + {envbindir}/pip install -r {toxinidir}/test-requirements.txt pytest From dde6db2c11e9137f2f1ead003316c02362351c38 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Fri, 22 Jun 2018 16:47:47 +0300 Subject: [PATCH 071/201] LICENSE file --- LICENSE | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1d77bbf --- /dev/null +++ b/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2018, Michal Horejsek +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + + Neither the name of the {organization} nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From f89a62b3966f619a10b4574590d91071c27a97c0 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Sat, 23 Jun 2018 20:40:37 +0300 Subject: [PATCH 072/201] remove unused method + unused import --- fastjsonschema/generator.py | 2 -- fastjsonschema/ref_resolver.py | 5 ----- 2 files changed, 7 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 5e95e96..7200615 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -8,8 +8,6 @@ from collections import OrderedDict import re -import requests - from .exceptions import JsonSchemaException from .indent import indent from .ref_resolver import RefResolver diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index c4e3843..0baf683 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -146,11 +146,6 @@ def resolving(self, ref): finally: self.base_uri, self.schema = old_base_uri, old_schema - def store_id(self, schema): - id = self.resolution_scope - if normalize(id) not in self.store: - self.store[normalize(id)] = schema - def get_uri(self): return normalize(self.resolution_scope) From 0c5de5ad0be4d7f1ca1ff856594320b939b06f2b Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 25 Jun 2018 16:11:43 +0200 Subject: [PATCH 073/201] Compile to code, pylint, Makefile with venv --- .gitignore | 26 ++++++--- Makefile | 48 +++++++++------ fastjsonschema/__init__.py | 103 ++++++++++++++------------------- fastjsonschema/__main__.py | 19 ++++++ fastjsonschema/exceptions.py | 2 +- fastjsonschema/generator.py | 24 ++++---- fastjsonschema/indent.py | 4 +- fastjsonschema/ref_resolver.py | 6 +- fastjsonschema/version.py | 1 + performance.py | 14 ++--- pylintrc | 33 +++++++++++ setup.py | 22 ++++--- tests/test_compile_to_code.py | 19 ++++++ tests/test_write_code.py | 24 -------- 14 files changed, 203 insertions(+), 142 deletions(-) create mode 100644 fastjsonschema/__main__.py create mode 100644 fastjsonschema/version.py create mode 100644 pylintrc create mode 100644 tests/test_compile_to_code.py delete mode 100644 tests/test_write_code.py diff --git a/.gitignore b/.gitignore index 2dcb41b..79e0da3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,22 @@ - +# Python files +__pycache__/ +.cache/ *.pyc -.cache -.vscode -.testmondata +*.pyo + +# Test files +.pytest_cache/ +temp/ + +# Deb files +debian/files +debian/*debhelper* +debian/*substvars +# Build files +.eggs/ +*.egg-info/ build/ _build/ -deb_dist/ dist/ -*.tar.gz -*egg-info -MANIFEST -temp/* +venv/ diff --git a/Makefile b/Makefile index 640bd11..cf93e63 100644 --- a/Makefile +++ b/Makefile @@ -1,33 +1,47 @@ +.PHONY: all venv test lint lint-js lint-web test test-web watch compile run makemessages compilemessages clean-dev deb +SHELL=/bin/bash + +VENV_NAME?=venv +VENV_BIN=$(shell pwd)/${VENV_NAME}/bin + +PYTHON=${VENV_BIN}/python3 + all: - @echo "make install - Install on local system" @echo "make test - Run tests during development" @echo "make performance - Run performance test of this and other implementation" @echo "make doc - Make documentation" @echo "make clean - Get rid of scratch and byte files" -deb: - python3 setup.py --command-packages=stdeb.command bdist_deb - -upload: - python3 setup.py register sdist upload +venv: $(VENV_NAME)/bin/activate +$(VENV_NAME)/bin/activate: setup.py + test -d $(VENV_NAME) || virtualenv -p python3 $(VENV_NAME) + # Some problem in latest version of setuptools during extracting translations. + ${PYTHON} -m pip install -U pip setuptools==39.1.0 + ${PYTHON} -m pip install -e .[devel] + touch $(VENV_NAME)/bin/activate -install: - pip install --editable .[test] +test: venv + ${PYTHON} -m pytest +test-lf: venv + ${PYTHON} -m pytest --last-failed tests -test: - python3 -m pytest tests +lint: venv + ${PYTHON} -m pylint fastjsonschema -test-lf: - python3 -m pytest --last-fail tests - -performance: - python3 performance.py +performance: venv + ${PYTHON} performance.py doc: cd docs; make +upload: venv + ${PYTHON} setup.py register sdist upload + +deb: venv + ${PYTHON} setup.py --command-packages=stdeb.command bdist_deb + clean: - python3 setup.py clean - find . -name '*.pyc' -delete + find . -name '*.pyc' -exec rm --force {} + + rm -rf $(VENV_NAME) *.eggs *.egg-info dist build docs/_build .mypy_cache .cache diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 7228c69..8be67e1 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -1,32 +1,34 @@ """ This project was made to come up with fast JSON validations. Just let's see some numbers first: - * Probalby most popular ``jsonschema`` can take in tests up to 7 seconds for valid inputs - and 1.6 seconds for invalid inputs. - * Secondly most popular ``json-spec`` is even worse with up to 11 and 2.6 seconds. - * Lastly ``validictory`` is much better with 640 or 30 miliseconds, but it does not + * Probalby most popular ``jsonschema`` can take in tests up to 5 seconds for valid inputs + and 1.2 seconds for invalid inputs. + * Secondly most popular ``json-spec`` is even worse with up to 7.2 and 1.7 seconds. + * Lastly ``validictory`` is much better with 370 or 23 miliseconds, but it does not follow all standards and it can be still slow for some purposes. That's why this project exists. It compiles definition into Python most stupid code which people would had hard time to write by themselfs because of not-written-rule DRY -(don't repeat yourself). When you compile definition, then times are 90 miliseconds for -valid inputs and 5 miliseconds for invalid inputs. Pretty amazing, right? :-) +(don't repeat yourself). When you compile definition, then times are 25 miliseconds for +valid inputs and less than 2 miliseconds for invalid inputs. Pretty amazing, right? :-) You can try it for yourself with included script: .. code-block:: bash $ make performance - fast_compiled valid ==> 0.09240092901140451 - fast_compiled invalid ==> 0.004246290685236454 - fast_not_compiled valid ==> 6.710726021323353 - fast_not_compiled invalid ==> 1.5449269418604672 - jsonschema valid ==> 6.963333621155471 - jsonschema invalid ==> 1.6309524956159294 - jsonspec valid ==> 10.576010060030967 - jsonspec invalid ==> 2.6199211929924786 - validictory valid ==> 0.6349993739277124 - validictory invalid ==> 0.03125431900843978 + fast_compiled valid ==> 0.026877017982769758 + fast_compiled invalid ==> 0.0015628149849362671 + fast_file valid ==> 0.025493122986517847 + fast_file invalid ==> 0.0012430319911800325 + fast_not_compiled valid ==> 4.790547857992351 + fast_not_compiled invalid ==> 1.2642899919883348 + jsonschema valid ==> 5.036152001994196 + jsonschema invalid ==> 1.1929481109953485 + jsonspec valid ==> 7.196442283981014 + jsonspec invalid ==> 1.7245555499684997 + validictory valid ==> 0.36818933801259845 + validictory invalid ==> 0.022672351042274386 This library follows and implements `JSON schema v4 `_. Sometimes it's not perfectly clear so I recommend also check out this `understaning json schema @@ -48,10 +50,12 @@ from .exceptions import JsonSchemaException from .generator import CodeGenerator from .ref_resolver import RefResolver +from .version import VERSION -__all__ = ('JsonSchemaException', 'compile', 'write_code') +__all__ = ('VERSION', 'JsonSchemaException', 'compile', 'compile_to_code') +# pylint: disable=redefined-builtin,dangerous-default-value,exec-used def compile(definition, handlers={}): """ Generates validation function for validating JSON schema by ``definition``. Example: @@ -79,64 +83,45 @@ def compile(definition, handlers={}): Exception :any:`JsonSchemaException` is thrown when validation fails. """ - resolver = RefResolver.from_schema(definition, handlers=handlers) - # get main function name - name = resolver.get_scope_name() - code_generator = CodeGenerator(definition, resolver=resolver) - # Do not pass local state so it can recursively call itself. + resolver, code_generator = _factory(definition, handlers) global_state = code_generator.global_state + # Do not pass local state so it can recursively call itself. exec(code_generator.func_code, global_state) - return global_state[name] + return global_state[resolver.get_scope_name()] + -def write_code(filename, definition, handlers={}, overwrite=False): +# pylint: disable=dangerous-default-value +def compile_to_code(definition, handlers={}): """ Generates validation function for validating JSON schema by ``definition`` - and write it to file. - - Arguments: - filename (str): Filename where generated code is written - definition (dict): JSON Schema defining validation rules - handlers (dict): A mapping from URI schemes to functions - that should be used to retrieve them. - overwrite (bool): Set to `True` to overwrite existing file - - Returns: - validation function name (str) - - Raises: - JsonSchemaException: is thrown when generation fails. - - - Create validator Example: + and returns compiled code. Example: .. code-block:: python import fastjsonschema - schema = {'type': 'string'} - name = fastjsonschema.write_code('validator.py', schema) - print(name) - - Generated file usage Example: + code = fastjsonschema.compile_to_code({'type': 'string'}) + with open('your_file.py', 'w') as f: + f.write(code) - .. code-block:: python + You can also use it as a script: - from validator import validate + .. code-block:: bash - validate('example') + echo "{'type': 'string'}" | pytohn3 -m fastjsonschema > your_file.py + pytohn3 -m fastjsonschema "{'type': 'string'}" > your_file.py Exception :any:`JsonSchemaException` is thrown when validation fails. """ + _, code_generator = _factory(definition, handlers) + return ( + 'VERSION = "' + VERSION + '"\n' + + code_generator.global_state_code + '\n' + + code_generator.func_code + ) + - if exists(filename) and overwrite == False: - raise JsonSchemaException('file {} already exists'.format(filename)) +def _factory(definition, handlers): resolver = RefResolver.from_schema(definition, handlers=handlers) - # get main function name - name = resolver.get_scope_name() code_generator = CodeGenerator(definition, resolver=resolver) - # Do not pass local state so it can recursively call itself. - global_state_code = code_generator.global_state_code - with open(filename, 'w', encoding='UTF-8') as out: - print(global_state_code, file=out) - print(code_generator.func_code, file=out) - return name + return resolver, code_generator diff --git a/fastjsonschema/__main__.py b/fastjsonschema/__main__.py new file mode 100644 index 0000000..e5f3aa7 --- /dev/null +++ b/fastjsonschema/__main__.py @@ -0,0 +1,19 @@ +import json +import sys + +from . import compile_to_code + + +def main(): + if len(sys.argv) == 2: + definition = sys.argv[1] + else: + definition = sys.stdin.read() + + definition = json.loads(definition) + code = compile_to_code(definition) + print(code) + + +if __name__ == '__main__': + main() diff --git a/fastjsonschema/exceptions.py b/fastjsonschema/exceptions.py index d950798..ac12512 100644 --- a/fastjsonschema/exceptions.py +++ b/fastjsonschema/exceptions.py @@ -1,4 +1,3 @@ - class JsonSchemaException(ValueError): """ Exception raised by validation function. Contains ``message`` with @@ -6,4 +5,5 @@ class JsonSchemaException(ValueError): """ def __init__(self, message): + super().__init__() self.message = message diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index b90e85a..419bf83 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -1,4 +1,3 @@ - # ___ # \./ DANGER: This module implements some code generation # .--.O.--. techniques involving string concatenation. @@ -13,6 +12,7 @@ from .ref_resolver import RefResolver +# pylint: disable=line-too-long FORMAT_REGEXS = { 'date-time': r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+(?:[+-][0-2]\d:[0-5]\d|Z)?$', 'uri': r'^\w+:(\/?\/?)[^\s]+$', @@ -29,6 +29,7 @@ def enforce_list(variable): return [variable] +# pylint: disable=too-many-instance-attributes,too-many-public-methods class CodeGenerator: """ This class is not supposed to be used directly. Anything @@ -71,7 +72,7 @@ def __init__(self, definition, resolver=None): # validation function names that are already done self._validation_functions_done = set() - if resolver == None: + if resolver is None: resolver = RefResolver.from_schema(definition) self._resolver = resolver # add main function to `self._needed_validation_functions` @@ -104,7 +105,7 @@ def __init__(self, definition, resolver=None): ('dependencies', self.generate_dependencies), )) - self.generate_func_code(definition) + self.generate_func_code() @property def func_code(self): @@ -132,7 +133,7 @@ def global_state_code(self): Returns global variables for generating function from ``func_code`` as code. Includes compiled regular expressions and imports. """ - if len(self._compile_regexps) == 0: + if self._compile_regexps: return '\n'.join( [ 'from fastjsonschema import JsonSchemaException', @@ -154,6 +155,7 @@ def global_state_code(self): ] ) + # pylint: disable=invalid-name @indent def l(self, line, *args, **kwds): """ @@ -212,14 +214,14 @@ def create_variable_keys(self): self._variables.add(variable_name) self.l('{variable}_keys = set({variable}.keys())') - def generate_func_code(self, definition): + def generate_func_code(self): """ Creates base code of validation function and calls helper for creating code by definition. """ self.l('NoneType = type(None)') # Generate parts that are referenced and not yet generated - while len(self._needed_validation_functions) > 0: + while self._needed_validation_functions: # During generation of validation function, could be needed to generate # new one that is added again to `_needed_validation_functions`. # Therefore usage of while instead of for loop. @@ -428,12 +430,12 @@ def generate_pattern(self): def generate_format(self): with self.l('if isinstance({variable}, str):'): - format = self._definition['format'] - if format in FORMAT_REGEXS: - format_regex = FORMAT_REGEXS[format] - self._generate_format(format, format + '_re_pattern', format_regex) + format_ = self._definition['format'] + if format_ in FORMAT_REGEXS: + format_regex = FORMAT_REGEXS[format_] + self._generate_format(format_, format_ + '_re_pattern', format_regex) # format regex is used only in meta schemas - elif format == 'regex': + elif format_ == 'regex': with self.l('try:'): self.l('re.compile({variable})') with self.l('except Exception:'): diff --git a/fastjsonschema/indent.py b/fastjsonschema/indent.py index 89c9493..e27fced 100644 --- a/fastjsonschema/indent.py +++ b/fastjsonschema/indent.py @@ -1,5 +1,3 @@ - - def indent(func): """ Decorator for allowing to use method as normal method or with @@ -18,5 +16,5 @@ def __init__(self, instance): def __enter__(self): self.instance._indent += 1 - def __exit__(self, type, value, traceback): + def __exit__(self, type_, value, traceback): self.instance._indent -= 1 diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index 0baf683..d6b7cec 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -88,6 +88,7 @@ class RefResolver(object): """ + # pylint: disable=dangerous-default-value,too-many-arguments def __init__(self, base_uri, schema, store={}, cache=True, handlers={}): self.base_uri = base_uri self.resolution_scope = base_uri @@ -151,7 +152,7 @@ def get_uri(self): def get_scope_name(self): name = 'validate_' + unquote(self.resolution_scope).replace('~1', '_').replace('~0', '_') - name = re.sub('[:/#\.\-\%]', '_', name) + name = re.sub(r'[:/#\.\-\%]', '_', name) name = name.lower().rstrip('_') return name @@ -163,8 +164,7 @@ def walk(self, node: dict): ref = node['$ref'] node['$ref'] = urlparse.urljoin(self.resolution_scope, ref) elif 'id' in node and isinstance(node['id'], str): - id = node['id'] - with self.in_scope(id): + with self.in_scope(node['id']): self.store[normalize(self.resolution_scope)] = node for _, item in node.items(): if isinstance(item, dict): diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py new file mode 100644 index 0000000..1dae207 --- /dev/null +++ b/fastjsonschema/version.py @@ -0,0 +1 @@ +VERSION = '1.6' diff --git a/performance.py b/performance.py index d0812f4..6d77ede 100644 --- a/performance.py +++ b/performance.py @@ -72,15 +72,13 @@ fastjsonschema_validate = fastjsonschema.compile(JSON_SCHEMA) fast_compiled = lambda value, _: fastjsonschema_validate(value) + fast_not_compiled = lambda value, json_schema: fastjsonschema.compile(json_schema)(value) -validate = fastjsonschema.write_code( - filename='temp/performance.py', - definition=JSON_SCHEMA, - overwrite=True -) -def fast_file(value, schema): - from temp.performance import validate - validate(value) + +with open('temp/performance.py', 'w') as f: + f.write(fastjsonschema.compile_to_code(JSON_SCHEMA)) +from temp.performance import validate +fast_file = lambda value, _: validate(value) jsonspec = load(JSON_SCHEMA) diff --git a/pylintrc b/pylintrc new file mode 100644 index 0000000..a5abbfe --- /dev/null +++ b/pylintrc @@ -0,0 +1,33 @@ + +[MASTER] +ignore=tests + +[MESSAGES CONTROL] +# missing-docstring only for now, remove after this issue is deployed https://github.com/PyCQA/pylint/issues/1164 +disable=missing-docstring + +[REPORTS] +output-format=colorized + +[VARIABLES] +init-import=no +dummy-variables-rgx=_|dummy + +[TYPECHECK] +ignore-mixin-members=yes + +[BASIC] +no-docstring-rgx=_.* +docstring-min-length=3 +good-names=_ +bad-names=foo,bar,baz,foobar + +[DESIGN] +min-public-methods=1 +max-public-methods=20 + +[FORMAT] +max-line-length=120 + +[MISCELLANEOUS] +notes=FIXME,XXX,TODO diff --git a/setup.py b/setup.py index b2102d0..d86a380 100644 --- a/setup.py +++ b/setup.py @@ -5,22 +5,30 @@ except ImportError: from distutils.core import setup +# https://packaging.python.org/en/latest/single_source_version.html +try: + execfile('fastjsonschema/version.py') +except NameError: + exec(open('fastjsonschema/version.py').read()) + setup( name='fastjsonschema', - version='1.6', + version=VERSION, packages=['fastjsonschema'], install_requires=[ 'requests', ], extras_require={ - "test": [ - "colorama", - "jsonschema", - "json-spec", - "pytest", - "validictory", + 'devel': [ + 'colorama', + 'jsonschema', + 'json-spec', + 'pylint', + 'pytest', + 'pytest-cache', + 'validictory', ], }, diff --git a/tests/test_compile_to_code.py b/tests/test_compile_to_code.py new file mode 100644 index 0000000..617e812 --- /dev/null +++ b/tests/test_compile_to_code.py @@ -0,0 +1,19 @@ +import os +import pytest + +from fastjsonschema import JsonSchemaException, compile_to_code + + +def test_compile_to_code(): + code = compile_to_code({ + 'properties': { + 'a': {'type': 'string'}, + 'b': {'type': 'integer'}, + } + }) + if not os.path.isdir('temp'): + os.makedirs('temp') + with open('temp/schema.py', 'w') as f: + f.write(code) + from temp.schema import validate + assert validate({'a': 'a', 'b': 1}) == {'a': 'a', 'b': 1} diff --git a/tests/test_write_code.py b/tests/test_write_code.py deleted file mode 100644 index 074563a..0000000 --- a/tests/test_write_code.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import pytest - -from fastjsonschema import JsonSchemaException, write_code - -def test_write_code(): - if not os.path.isdir('temp'): - os.makedirs('temp') - if os.path.exists('temp/schema.py'): - os.remove('temp/schema.py') - name = write_code( - 'temp/schema.py', - {'properties': {'a': {'type': 'string'}, 'b': {'type': 'integer'}}} - ) - assert name == 'validate' - assert os.path.exists('temp/schema.py') - with pytest.raises(JsonSchemaException) as exc: - name = write_code( - 'temp/schema.py', - {'type': 'string'} - ) - assert exc.value.message == 'file temp/schema.py already exists' - from temp.schema import validate - assert validate({'a': 'a', 'b': 1}) == {'a': 'a', 'b': 1} From 068865e8834f396ff591fabe1af1110083f8d96a Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Thu, 28 Jun 2018 10:06:23 +0300 Subject: [PATCH 074/201] remove test-requirements.txt that is not needed --- test-requirements.txt | 3 --- tox.ini | 2 -- 2 files changed, 5 deletions(-) delete mode 100644 test-requirements.txt diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index a3fa396..0000000 --- a/test-requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ --e . -pytest -sphinx diff --git a/tox.ini b/tox.ini index fe0f0fe..ff5d2d5 100644 --- a/tox.ini +++ b/tox.ini @@ -8,8 +8,6 @@ envlist = py{34,35,36} [testenv] whitelist_externals = - pip pytest commands = - {envbindir}/pip install -r {toxinidir}/test-requirements.txt pytest From 2489fc4d3a09d830aab8cd061820e34fcfdfb293 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Thu, 28 Jun 2018 10:15:42 +0300 Subject: [PATCH 075/201] remove duplicate code --- tests/test_json_schema_test_suits.py | 7 ++++++- tests/test_json_schema_test_suits_draft4.py | 6 +----- tests/test_json_schema_test_suits_draft6.py | 8 ++------ tests/test_json_schema_test_suits_draft7.py | 6 +----- 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/tests/test_json_schema_test_suits.py b/tests/test_json_schema_test_suits.py index cc62592..6443cc7 100644 --- a/tests/test_json_schema_test_suits.py +++ b/tests/test_json_schema_test_suits.py @@ -1,5 +1,6 @@ import json import pytest +from pathlib import Path import requests from fastjsonschema import CodeGenerator, RefResolver, JsonSchemaException, compile @@ -24,7 +25,11 @@ def remotes_handler(uri): return requests.get(uri).json() -def resolve_param_values_and_ids(test_file_paths, ignored_suite_files, ignore_tests): +def resolve_param_values_and_ids(suite_dir, ignored_suite_files, ignore_tests): + + suite_dir_path = Path(suite_dir).resolve() + test_file_paths = sorted(set(suite_dir_path.glob("**/*.json"))) + param_values = [] param_ids = [] for test_file_path in test_file_paths: diff --git a/tests/test_json_schema_test_suits_draft4.py b/tests/test_json_schema_test_suits_draft4.py index 4c87f12..c88c97d 100644 --- a/tests/test_json_schema_test_suits_draft4.py +++ b/tests/test_json_schema_test_suits_draft4.py @@ -1,4 +1,3 @@ -from pathlib import Path import pytest from test_json_schema_test_suits import template_test, resolve_param_values_and_ids @@ -10,11 +9,8 @@ def pytest_generate_tests(metafunc): ] ignore_tests = [] - suite_dir_path = Path(suite_dir).resolve() - test_file_paths = sorted(set(suite_dir_path.glob("**/*.json"))) - param_values, param_ids = resolve_param_values_and_ids( - test_file_paths, ignored_suite_files, ignore_tests + suite_dir, ignored_suite_files, ignore_tests ) metafunc.parametrize(['schema', 'data', 'is_valid'], param_values, ids=param_ids) diff --git a/tests/test_json_schema_test_suits_draft6.py b/tests/test_json_schema_test_suits_draft6.py index 948d99a..0104de4 100644 --- a/tests/test_json_schema_test_suits_draft6.py +++ b/tests/test_json_schema_test_suits_draft6.py @@ -1,4 +1,3 @@ -from pathlib import Path import pytest from test_json_schema_test_suits import template_test, resolve_param_values_and_ids @@ -54,12 +53,9 @@ def pytest_generate_tests(metafunc): '$ref to boolean schema false', ] - suite_dir_path = Path(suite_dir).resolve() - test_file_paths = sorted(set(suite_dir_path.glob("**/*.json"))) - param_values, param_ids = resolve_param_values_and_ids( - test_file_paths, ignored_suite_files, ignore_tests - ) + suite_dir, ignored_suite_files, ignore_tests + ) metafunc.parametrize(['schema', 'data', 'is_valid'], param_values, ids=param_ids) test = template_test diff --git a/tests/test_json_schema_test_suits_draft7.py b/tests/test_json_schema_test_suits_draft7.py index b149d5a..2da6270 100644 --- a/tests/test_json_schema_test_suits_draft7.py +++ b/tests/test_json_schema_test_suits_draft7.py @@ -1,4 +1,3 @@ -from pathlib import Path import pytest from test_json_schema_test_suits import template_test, resolve_param_values_and_ids @@ -63,11 +62,8 @@ def pytest_generate_tests(metafunc): '$ref to boolean schema false', ] - suite_dir_path = Path(suite_dir).resolve() - test_file_paths = sorted(set(suite_dir_path.glob("**/*.json"))) - param_values, param_ids = resolve_param_values_and_ids( - test_file_paths, ignored_suite_files, ignore_tests + suite_dir, ignored_suite_files, ignore_tests ) metafunc.parametrize(['schema', 'data', 'is_valid'], param_values, ids=param_ids) From f1b4b121c9cb30fd7d20960130dde4370204ce59 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Thu, 28 Jun 2018 13:16:37 +0300 Subject: [PATCH 076/201] rename test, because name already in use --- tests/test_string.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_string.py b/tests/test_string.py index ee45203..acfaa89 100644 --- a/tests/test_string.py +++ b/tests/test_string.py @@ -66,7 +66,7 @@ def test_pattern(asserter, value, expected): ('[a-z]', '[a-z]'), ('[a-z', exc), ]) -def test_pattern(asserter, value, expected): +def test_regex_pattern(asserter, value, expected): asserter({ 'format': 'regex', 'type': 'string' From ca113f65869f4dbceebad3e05d20870ffca46333 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Thu, 28 Jun 2018 13:40:32 +0300 Subject: [PATCH 077/201] fix docstring intendation, so sphinx do not complain when making docs --- fastjsonschema/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 75c6232..472c62f 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -84,7 +84,7 @@ def compile(definition, handlers={}): Args: definition (dict): Json schema definition handlers (dict): A mapping from URI schemes to functions - that should be used to retrieve them. + that should be used to retrieve them. Exception :any:`JsonSchemaException` is thrown when validation fails. """ From 5909b6bd9202d9b4c2398c55e00950b92566be01 Mon Sep 17 00:00:00 2001 From: Antti Jokipii Date: Thu, 28 Jun 2018 17:00:43 +0300 Subject: [PATCH 078/201] .gitignore --- .gitignore | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 7e2dba1..cca1b83 100644 --- a/.gitignore +++ b/.gitignore @@ -2,26 +2,20 @@ __pycache__/ .cache/ *.pyc -<<<<<<< HEAD .cache -.vscode -.testmondata -__pycache__ -.tox -.pytest_cache -.coverage -======= *.pyo # Test files .pytest_cache/ temp/ +.tox +.coverage +.testmondata # Deb files debian/files debian/*debhelper* debian/*substvars ->>>>>>> upstream/master # Build files .eggs/ @@ -30,3 +24,6 @@ build/ _build/ dist/ venv/ + +# Editors +.vscode From 655d08fd4483618d49b0b549f454fec71c7b7a59 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Sat, 21 Jul 2018 08:00:37 +0000 Subject: [PATCH 079/201] Clean up --- .gitignore | 5 +- Makefile | 2 +- README.rst | 5 +- fastjsonschema/ref_resolver.py | 2 +- tests/json_schema/__init__.py | 0 tests/json_schema/test_draft4.py | 18 +++++ tests/json_schema/test_draft6.py | 63 ++++++++++++++++ tests/json_schema/test_draft7.py | 72 +++++++++++++++++++ .../utils.py} | 19 +++-- tests/test_json_schema_test_suits_draft4.py | 17 ----- tests/test_json_schema_test_suits_draft6.py | 61 ---------------- tests/test_json_schema_test_suits_draft7.py | 70 ------------------ 12 files changed, 172 insertions(+), 162 deletions(-) create mode 100644 tests/json_schema/__init__.py create mode 100644 tests/json_schema/test_draft4.py create mode 100644 tests/json_schema/test_draft6.py create mode 100644 tests/json_schema/test_draft7.py rename tests/{test_json_schema_test_suits.py => json_schema/utils.py} (91%) delete mode 100644 tests/test_json_schema_test_suits_draft4.py delete mode 100644 tests/test_json_schema_test_suits_draft6.py delete mode 100644 tests/test_json_schema_test_suits_draft7.py diff --git a/.gitignore b/.gitignore index cca1b83..ae7c3cf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,8 @@ # Python files -__pycache__/ -.cache/ *.pyc -.cache *.pyo +__pycache__/ +.cache # Test files .pytest_cache/ diff --git a/Makefile b/Makefile index cf93e63..0f33b02 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ $(VENV_NAME)/bin/activate: setup.py test: venv ${PYTHON} -m pytest test-lf: venv - ${PYTHON} -m pytest --last-failed tests + ${PYTHON} -m pytest --last-failed lint: venv ${PYTHON} -m pylint fastjsonschema diff --git a/README.rst b/README.rst index 6b3c54a..d6e0515 100644 --- a/README.rst +++ b/README.rst @@ -14,7 +14,7 @@ Fast JSON schema for Python This project was made to come up with fast JSON validations. It is at least an order of magnitude faster than other Python implemantaions. -See `documentation `_ for +See `documentation `_ for performance test details. Current version is implementation of `json-schema `_ draft-04. @@ -40,5 +40,4 @@ Support for Python 3.3 and higher. Documentation ------------- -Documentation: `https://seznam.github.io/python-fastjsonschema/ -`_ +Documentation: `https://horejsek.github.io/python-fastjsonschema`_ diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index d6b7cec..d32e2e3 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -74,7 +74,7 @@ def resolve_remote(uri, handlers): return result -class RefResolver(object): +class RefResolver: """ Resolve JSON References. diff --git a/tests/json_schema/__init__.py b/tests/json_schema/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/json_schema/test_draft4.py b/tests/json_schema/test_draft4.py new file mode 100644 index 0000000..5c76970 --- /dev/null +++ b/tests/json_schema/test_draft4.py @@ -0,0 +1,18 @@ +import pytest + +from .utils import template_test, resolve_param_values_and_ids + + +def pytest_generate_tests(metafunc): + param_values, param_ids = resolve_param_values_and_ids( + suite_dir='JSON-Schema-Test-Suite/tests/draft4', + ignored_suite_files=[ + 'ecmascript-regex.json', + ], + ignore_tests=[], + ) + metafunc.parametrize(['schema', 'data', 'is_valid'], param_values, ids=param_ids) + + +# Real test function to be used with parametrization by previous hook function. +test = template_test diff --git a/tests/json_schema/test_draft6.py b/tests/json_schema/test_draft6.py new file mode 100644 index 0000000..b2dae1d --- /dev/null +++ b/tests/json_schema/test_draft6.py @@ -0,0 +1,63 @@ +import pytest + +from .utils import template_test, resolve_param_values_and_ids + + +def pytest_generate_tests(metafunc): + param_values, param_ids = resolve_param_values_and_ids( + suite_dir='JSON-Schema-Test-Suite/tests/draft6', + ignored_suite_files=[ + 'bignum.json', + 'ecmascript-regex.json', + 'zeroTerminatedFloats.json', + 'boolean_schema.json', + 'contains.json', + 'const.json', + ], + ignore_tests=[ + 'invalid definition', + 'valid definition', + 'Recursive references between schemas', + 'remote ref, containing refs itself', + 'dependencies with boolean subschemas', + 'dependencies with empty array', + 'exclusiveMaximum validation', + 'exclusiveMinimum validation', + 'format: uri-template', + 'validation of URI References', + 'items with boolean schema (true)', + 'items with boolean schema (false)', + 'items with boolean schema', + 'items with boolean schemas', + 'not with boolean schema true', + 'not with boolean schema false', + 'properties with boolean schema', + 'propertyNames with boolean schema false', + 'propertyNames validation', + 'base URI change - change folder', + 'base URI change - change folder in subschema', + 'base URI change', + 'root ref in remote ref', + 'allOf with boolean schemas, all true', + 'allOf with boolean schemas, some false', + 'allOf with boolean schemas, all false', + 'anyOf with boolean schemas, all true', + 'anyOf with boolean schemas, some false', + 'anyOf with boolean schemas, all false', + 'anyOf with boolean schemas, some true', + 'oneOf with boolean schemas, all true', + 'oneOf with boolean schemas, some false', + 'oneOf with boolean schemas, all false', + 'oneOf with boolean schemas, one true', + 'oneOf with boolean schemas, more than one true', + 'validation of JSON-pointers (JSON String Representation)', + 'patternProperties with boolean schemas', + '$ref to boolean schema true', + '$ref to boolean schema false', + ], + ) + metafunc.parametrize(['schema', 'data', 'is_valid'], param_values, ids=param_ids) + + +# Real test function to be used with parametrization by previous hook function. +test = template_test diff --git a/tests/json_schema/test_draft7.py b/tests/json_schema/test_draft7.py new file mode 100644 index 0000000..d26b9c4 --- /dev/null +++ b/tests/json_schema/test_draft7.py @@ -0,0 +1,72 @@ +import pytest + +from .utils import template_test, resolve_param_values_and_ids + + +def pytest_generate_tests(metafunc): + param_values, param_ids = resolve_param_values_and_ids( + suite_dir='JSON-Schema-Test-Suite/tests/draft7', + ignored_suite_files=[ + 'bignum.json', + 'ecmascript-regex.json', + 'zeroTerminatedFloats.json', + 'boolean_schema.json', + 'contains.json', + 'content.json', + 'if-then-else.json', + 'idn-email.json', + 'idn-hostname.json', + 'iri-reference.json', + 'iri.json', + 'relative-json-pointer.json', + 'time.json', + 'const.json', + ], + ignore_tests=[ + 'invalid definition', + 'valid definition', + 'Recursive references between schemas', + 'remote ref, containing refs itself', + 'dependencies with boolean subschemas', + 'dependencies with empty array', + 'exclusiveMaximum validation', + 'exclusiveMinimum validation', + 'format: uri-template', + 'validation of URI References', + 'items with boolean schema (true)', + 'items with boolean schema (false)', + 'items with boolean schema', + 'items with boolean schemas', + 'not with boolean schema true', + 'not with boolean schema false', + 'properties with boolean schema', + 'propertyNames with boolean schema false', + 'propertyNames validation', + 'base URI change - change folder', + 'base URI change - change folder in subschema', + 'base URI change', + 'root ref in remote ref', + 'validation of date strings', + 'allOf with boolean schemas, all true', + 'allOf with boolean schemas, some false', + 'allOf with boolean schemas, all false', + 'anyOf with boolean schemas, all true', + 'anyOf with boolean schemas, some false', + 'anyOf with boolean schemas, all false', + 'anyOf with boolean schemas, some true', + 'oneOf with boolean schemas, all true', + 'oneOf with boolean schemas, some false', + 'oneOf with boolean schemas, all false', + 'oneOf with boolean schemas, one true', + 'oneOf with boolean schemas, more than one true', + 'validation of JSON-pointers (JSON String Representation)', + 'patternProperties with boolean schemas', + '$ref to boolean schema true', + '$ref to boolean schema false', + ], + ) + metafunc.parametrize(['schema', 'data', 'is_valid'], param_values, ids=param_ids) + + +# Real test function to be used with parametrization by previous hook function. +test = template_test diff --git a/tests/test_json_schema_test_suits.py b/tests/json_schema/utils.py similarity index 91% rename from tests/test_json_schema_test_suits.py rename to tests/json_schema/utils.py index 6443cc7..16fe43a 100644 --- a/tests/test_json_schema_test_suits.py +++ b/tests/json_schema/utils.py @@ -1,10 +1,13 @@ import json -import pytest from pathlib import Path + +import pytest import requests + from fastjsonschema import CodeGenerator, RefResolver, JsonSchemaException, compile -remotes = { + +REMOTES = { 'http://localhost:1234/integer.json': {'type': 'integer'}, 'http://localhost:1234/name.json': { 'type': 'string', @@ -18,15 +21,15 @@ }, 'http://localhost:1234/folder/folderInteger.json': {'type': 'integer'} } + + def remotes_handler(uri): - print(uri) - if uri in remotes: - return remotes[uri] + if uri in REMOTES: + return REMOTES[uri] return requests.get(uri).json() def resolve_param_values_and_ids(suite_dir, ignored_suite_files, ignore_tests): - suite_dir_path = Path(suite_dir).resolve() test_file_paths = sorted(set(suite_dir_path.glob("**/*.json"))) @@ -55,6 +58,10 @@ def resolve_param_values_and_ids(suite_dir, ignored_suite_files, ignore_tests): def template_test(schema, data, is_valid): + """ + Test function to be used (imported) in final test file to run the tests + which are generated by `pytest_generate_tests` hook. + """ # For debug purposes. When test fails, it will print stdout. resolver = RefResolver.from_schema(schema, handlers={'http': remotes_handler}) print(CodeGenerator(schema, resolver=resolver).func_code) diff --git a/tests/test_json_schema_test_suits_draft4.py b/tests/test_json_schema_test_suits_draft4.py deleted file mode 100644 index c88c97d..0000000 --- a/tests/test_json_schema_test_suits_draft4.py +++ /dev/null @@ -1,17 +0,0 @@ -import pytest -from test_json_schema_test_suits import template_test, resolve_param_values_and_ids - - -def pytest_generate_tests(metafunc): - suite_dir = 'JSON-Schema-Test-Suite/tests/draft4' - ignored_suite_files = [ - 'ecmascript-regex.json', - ] - ignore_tests = [] - - param_values, param_ids = resolve_param_values_and_ids( - suite_dir, ignored_suite_files, ignore_tests - ) - metafunc.parametrize(['schema', 'data', 'is_valid'], param_values, ids=param_ids) - -test = template_test diff --git a/tests/test_json_schema_test_suits_draft6.py b/tests/test_json_schema_test_suits_draft6.py deleted file mode 100644 index 0104de4..0000000 --- a/tests/test_json_schema_test_suits_draft6.py +++ /dev/null @@ -1,61 +0,0 @@ -import pytest -from test_json_schema_test_suits import template_test, resolve_param_values_and_ids - -def pytest_generate_tests(metafunc): - suite_dir = 'JSON-Schema-Test-Suite/tests/draft6' - ignored_suite_files = [ - 'bignum.json', - 'ecmascript-regex.json', - 'zeroTerminatedFloats.json', - 'boolean_schema.json', - 'contains.json', - 'const.json', - ] - ignore_tests = [ - 'invalid definition', - 'valid definition', - 'Recursive references between schemas', - 'remote ref, containing refs itself', - 'dependencies with boolean subschemas', - 'dependencies with empty array', - 'exclusiveMaximum validation', - 'exclusiveMinimum validation', - 'format: uri-template', - 'validation of URI References', - 'items with boolean schema (true)', - 'items with boolean schema (false)', - 'items with boolean schema', - 'items with boolean schemas', - 'not with boolean schema true', - 'not with boolean schema false', - 'properties with boolean schema', - 'propertyNames with boolean schema false', - 'propertyNames validation', - 'base URI change - change folder', - 'base URI change - change folder in subschema', - 'base URI change', - 'root ref in remote ref', - 'allOf with boolean schemas, all true', - 'allOf with boolean schemas, some false', - 'allOf with boolean schemas, all false', - 'anyOf with boolean schemas, all true', - 'anyOf with boolean schemas, some false', - 'anyOf with boolean schemas, all false', - 'anyOf with boolean schemas, some true', - 'oneOf with boolean schemas, all true', - 'oneOf with boolean schemas, some false', - 'oneOf with boolean schemas, all false', - 'oneOf with boolean schemas, one true', - 'oneOf with boolean schemas, more than one true', - 'validation of JSON-pointers (JSON String Representation)', - 'patternProperties with boolean schemas', - '$ref to boolean schema true', - '$ref to boolean schema false', - ] - - param_values, param_ids = resolve_param_values_and_ids( - suite_dir, ignored_suite_files, ignore_tests - ) - metafunc.parametrize(['schema', 'data', 'is_valid'], param_values, ids=param_ids) - -test = template_test diff --git a/tests/test_json_schema_test_suits_draft7.py b/tests/test_json_schema_test_suits_draft7.py deleted file mode 100644 index 2da6270..0000000 --- a/tests/test_json_schema_test_suits_draft7.py +++ /dev/null @@ -1,70 +0,0 @@ -import pytest -from test_json_schema_test_suits import template_test, resolve_param_values_and_ids - -def pytest_generate_tests(metafunc): - suite_dir = 'JSON-Schema-Test-Suite/tests/draft7' - ignored_suite_files = [ - 'bignum.json', - 'ecmascript-regex.json', - 'zeroTerminatedFloats.json', - 'boolean_schema.json', - 'contains.json', - 'content.json', - 'if-then-else.json', - 'idn-email.json', - 'idn-hostname.json', - 'iri-reference.json', - 'iri.json', - 'relative-json-pointer.json', - 'time.json', - 'const.json', - ] - ignore_tests = [ - 'invalid definition', - 'valid definition', - 'Recursive references between schemas', - 'remote ref, containing refs itself', - 'dependencies with boolean subschemas', - 'dependencies with empty array', - 'exclusiveMaximum validation', - 'exclusiveMinimum validation', - 'format: uri-template', - 'validation of URI References', - 'items with boolean schema (true)', - 'items with boolean schema (false)', - 'items with boolean schema', - 'items with boolean schemas', - 'not with boolean schema true', - 'not with boolean schema false', - 'properties with boolean schema', - 'propertyNames with boolean schema false', - 'propertyNames validation', - 'base URI change - change folder', - 'base URI change - change folder in subschema', - 'base URI change', - 'root ref in remote ref', - 'validation of date strings', - 'allOf with boolean schemas, all true', - 'allOf with boolean schemas, some false', - 'allOf with boolean schemas, all false', - 'anyOf with boolean schemas, all true', - 'anyOf with boolean schemas, some false', - 'anyOf with boolean schemas, all false', - 'anyOf with boolean schemas, some true', - 'oneOf with boolean schemas, all true', - 'oneOf with boolean schemas, some false', - 'oneOf with boolean schemas, all false', - 'oneOf with boolean schemas, one true', - 'oneOf with boolean schemas, more than one true', - 'validation of JSON-pointers (JSON String Representation)', - 'patternProperties with boolean schemas', - '$ref to boolean schema true', - '$ref to boolean schema false', - ] - - param_values, param_ids = resolve_param_values_and_ids( - suite_dir, ignored_suite_files, ignore_tests - ) - metafunc.parametrize(['schema', 'data', 'is_valid'], param_values, ids=param_ids) - -test = template_test From 2ace14a90ee7ecfd6ab7aa4e09bfbcf119cd5a4a Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Sat, 21 Jul 2018 08:10:59 +0000 Subject: [PATCH 080/201] Fixed Makefile --- Makefile | 8 ++++---- docs/Makefile | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 0f33b02..e1f3716 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all venv test lint lint-js lint-web test test-web watch compile run makemessages compilemessages clean-dev deb +.PHONY: all venv lint test test-lf performance doc upload deb clean SHELL=/bin/bash VENV_NAME?=venv @@ -22,14 +22,14 @@ $(VENV_NAME)/bin/activate: setup.py ${PYTHON} -m pip install -e .[devel] touch $(VENV_NAME)/bin/activate +lint: venv + ${PYTHON} -m pylint fastjsonschema + test: venv ${PYTHON} -m pytest test-lf: venv ${PYTHON} -m pytest --last-failed -lint: venv - ${PYTHON} -m pylint fastjsonschema - performance: venv ${PYTHON} performance.py diff --git a/docs/Makefile b/docs/Makefile index fefaf1a..3ba9e91 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -8,5 +8,6 @@ SPHINXOPTS=-n -W -d $(BUILDDIR)/doctrees . sphinx: sphinx-build -b html $(SPHINXOPTS) $(BUILDDIR)/html +.PHONY: clean clean: rm -rf build From 298ba220bb2dd200eed97e4f3de031323b28d787 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 26 Jul 2018 13:17:22 +0200 Subject: [PATCH 081/201] Benchmarks --- .gitignore | 1 + Makefile | 32 ++++++++++-- setup.py | 1 + tests/benchmarks/test_benchmark.py | 83 ++++++++++++++++++++++++++++++ tests/conftest.py | 1 - 5 files changed, 112 insertions(+), 6 deletions(-) create mode 100644 tests/benchmarks/test_benchmark.py diff --git a/.gitignore b/.gitignore index ae7c3cf..6bc9168 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ temp/ .tox .coverage .testmondata +.benchmarks # Deb files debian/files diff --git a/Makefile b/Makefile index e1f3716..d4616cf 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all venv lint test test-lf performance doc upload deb clean +.PHONY: all venv lint test test-lf benchmark benchmark-save performance doc upload deb clean SHELL=/bin/bash VENV_NAME?=venv @@ -25,10 +25,32 @@ $(VENV_NAME)/bin/activate: setup.py lint: venv ${PYTHON} -m pylint fastjsonschema -test: venv - ${PYTHON} -m pytest -test-lf: venv - ${PYTHON} -m pytest --last-failed +jsonschemasuitcases: + git submodule init + git submodule update + +test: venv jsonschemasuitcases + ${PYTHON} -m pytest --benchmark-skip +test-lf: venv jsonschemasuitcases + ${PYTHON} -m pytest --benchmark-skip --last-failed + +# Call make benchmark-save before change and then make benchmark to compare results. +benchmark: venv jsonschemasuitcases + ${PYTHON} -m pytest \ + --benchmark-only \ + --benchmark-sort=name \ + --benchmark-group-by=fullfunc \ + --benchmark-disable-gc \ + --benchmark-compare \ + --benchmark-compare-fail='min:5%' +benchmark-save: venv jsonschemasuitcases + ${PYTHON} -m pytest \ + --benchmark-only \ + --benchmark-sort=name \ + --benchmark-group-by=fullfunc \ + --benchmark-disable-gc \ + --benchmark-save=benchmark \ + --benchmark-save-data performance: venv ${PYTHON} performance.py diff --git a/setup.py b/setup.py index 873d7bf..5a9c478 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ 'json-spec', 'pylint', 'pytest', + 'pytest-benchmark', 'pytest-cache', 'validictory', ], diff --git a/tests/benchmarks/test_benchmark.py b/tests/benchmarks/test_benchmark.py new file mode 100644 index 0000000..8da2c8a --- /dev/null +++ b/tests/benchmarks/test_benchmark.py @@ -0,0 +1,83 @@ +import pytest + +import fastjsonschema + + +JSON_SCHEMA = { + 'type': 'array', + 'items': [ + { + 'type': 'number', + 'maximum': 10, + 'exclusiveMaximum': True, + }, + { + 'type': 'string', + 'enum': ['hello', 'world'], + }, + { + 'type': 'array', + 'minItems': 1, + 'maxItems': 3, + 'items': [ + {'type': 'number'}, + {'type': 'string'}, + {'type': 'boolean'}, + ], + }, + { + 'type': 'object', + 'required': ['a', 'b'], + 'minProperties': 3, + 'properties': { + 'a': {'type': ['null', 'string']}, + 'b': {'type': ['null', 'string']}, + 'c': {'type': ['null', 'string'], 'default': 'abc'} + }, + 'additionalProperties': {'type': 'string'}, + }, + {'not': {'type': ['null']}}, + {'oneOf': [ + {'type': 'number', 'multipleOf': 3}, + {'type': 'number', 'multipleOf': 5}, + ]}, + ], +} + + +fastjsonschema_validate = fastjsonschema.compile(JSON_SCHEMA) + + +@pytest.mark.benchmark(min_rounds=20) +@pytest.mark.parametrize('value', ( + [9, 'hello', [1, 'a', True], {'a': 'a', 'b': 'b', 'd': 'd'}, 42, 3], + [9, 'world', [1, 'a', True], {'a': 'a', 'b': 'b', 'd': 'd'}, 42, 3], + [9, 'world', [1, 'a', True], {'a': 'a', 'b': 'b', 'c': 'xy'}, 42, 3], + [9, 'world', [1, 'a', True], {'a': 'a', 'b': 'b', 'c': 'xy'}, 'str', 5], +)) +def test_benchmark_ok_values(benchmark, value): + @benchmark + def f(): + fastjsonschema_validate(value) + + +@pytest.mark.benchmark(min_rounds=20) +@pytest.mark.parametrize('value', ( + [10, 'world', [1, 'a', True], {'a': 'a', 'b': 'b', 'c': 'xy'}, 'str', 5], + [9, 'xxx', [1, 'a', True], {'a': 'a', 'b': 'b', 'c': 'xy'}, 'str', 5], + [9, 'hello', [], {'a': 'a', 'b': 'b', 'c': 'xy'}, 'str', 5], + [9, 'hello', [1, 2, 3], {'a': 'a', 'b': 'b', 'c': 'xy'}, 'str', 5], + [9, 'hello', [1, 'a', True], {'a': 'a', 'x': 'x', 'y': 'y'}, 'str', 5], + [9, 'hello', [1, 'a', True], {}, 'str', 5], + [9, 'hello', [1, 'a', True], {'a': 'a', 'b': 'b', 'x': 'x'}, None, 5], + [9, 'hello', [1, 'a', True], {'a': 'a', 'b': 'b', 'x': 'x'}, 42, 15], +)) +def test_benchmark_bad_values(benchmark, value): + @benchmark + def f(): + try: + fastjsonschema_validate(value) + except fastjsonschema.JsonSchemaException: + pass + else: + pytest.fail('Exception is not raised') diff --git a/tests/conftest.py b/tests/conftest.py index 87bfbb7..2d77249 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,3 @@ - import os import sys From 9a60247afa1ed7cc1ccb598dbc92f01dcb85dd9f Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 26 Jul 2018 16:06:23 +0200 Subject: [PATCH 082/201] Optimalization of isinstance of dicts and lists --- Makefile | 3 ++ fastjsonschema/generator.py | 56 ++++++++++++++++++++++++++++--------- schema.json | 40 ++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 13 deletions(-) create mode 100644 schema.json diff --git a/Makefile b/Makefile index d4616cf..a74376e 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,9 @@ benchmark-save: venv jsonschemasuitcases performance: venv ${PYTHON} performance.py +printcode: venv + cat schema.json | python3 -m fastjsonschema + doc: cd docs; make diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 419bf83..13e6de7 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -205,8 +205,7 @@ def create_variable_with_length(self): def create_variable_keys(self): """ Append code for creating variable with keys of that variable (dictionary) - with name ``{variable}_len``. It can be called several times and always - it's done only when that variable still does not exists. + with a name ``{variable}_keys``. Similar to `create_variable_with_length`. """ variable_name = '{}_keys'.format(self._variable) if variable_name in self._variables: @@ -214,6 +213,28 @@ def create_variable_keys(self): self._variables.add(variable_name) self.l('{variable}_keys = set({variable}.keys())') + def create_variable_is_list(self): + """ + Append code for creating variable with bool if it's instance of list + with a name ``{variable}_is_list``. Similar to `create_variable_with_length`. + """ + variable_name = '{}_is_list'.format(self._variable) + if variable_name in self._variables: + return + self._variables.add(variable_name) + self.l('{variable}_is_list = isinstance({variable}, list)') + + def create_variable_is_dict(self): + """ + Append code for creating variable with bool if it's instance of list + with a name ``{variable}_is_dict``. Similar to `create_variable_with_length`. + """ + variable_name = '{}_is_dict'.format(self._variable) + if variable_name in self._variables: + return + self._variables.add(variable_name) + self.l('{variable}_is_dict = isinstance({variable}, dict)') + def generate_func_code(self): """ Creates base code of validation function and calls helper @@ -305,7 +326,6 @@ def generate_type(self): with self.l('if not isinstance({variable}, ({})){}:', python_types, extra): self.l('raise JsonSchemaException("{name} must be {}")', ' or '.join(types)) - def generate_enum(self): """ Means that only value specified in the enum is valid. @@ -473,13 +493,15 @@ def generate_multiple_of(self): self.l('raise JsonSchemaException("{name} must be multiple of {multipleOf}")') def generate_min_items(self): - with self.l('if isinstance({variable}, list):'): + self.create_variable_is_list() + with self.l('if {variable}_is_list:'): self.create_variable_with_length() with self.l('if {variable}_len < {minItems}:'): self.l('raise JsonSchemaException("{name} must contain at least {minItems} items")') def generate_max_items(self): - with self.l('if isinstance({variable}, list):'): + self.create_variable_is_list() + with self.l('if {variable}_is_list:'): self.create_variable_with_length() with self.l('if {variable}_len > {maxItems}:'): self.l('raise JsonSchemaException("{name} must contain less than or equal to {maxItems} items")') @@ -504,7 +526,8 @@ def generate_unique_items(self): self.l('raise JsonSchemaException("{name} must contain unique items")') def generate_items(self): - with self.l('if isinstance({variable}, list):'): + self.create_variable_is_list() + with self.l('if {variable}_is_list:'): self.create_variable_with_length() if isinstance(self._definition['items'], list): for x, item_definition in enumerate(self._definition['items']): @@ -538,25 +561,29 @@ def generate_items(self): ) def generate_min_properties(self): - with self.l('if isinstance({variable}, dict):'): + self.create_variable_is_dict() + with self.l('if {variable}_is_dict:'): self.create_variable_with_length() with self.l('if {variable}_len < {minProperties}:'): self.l('raise JsonSchemaException("{name} must contain at least {minProperties} properties")') def generate_max_properties(self): - with self.l('if isinstance({variable}, dict):'): + self.create_variable_is_dict() + with self.l('if {variable}_is_dict:'): self.create_variable_with_length() with self.l('if {variable}_len > {maxProperties}:'): self.l('raise JsonSchemaException("{name} must contain less than or equal to {maxProperties} properties")') def generate_required(self): - with self.l('if isinstance({variable}, dict):'): + self.create_variable_is_dict() + with self.l('if {variable}_is_dict:'): self.create_variable_with_length() with self.l('if not all(prop in {variable} for prop in {required}):'): self.l('raise JsonSchemaException("{name} must contain {required} properties")') def generate_properties(self): - with self.l('if isinstance({variable}, dict):'): + self.create_variable_is_dict() + with self.l('if {variable}_is_dict:'): self.create_variable_keys() for key, prop_definition in self._definition['properties'].items(): key_name = re.sub(r'($[^a-zA-Z]|[^a-zA-Z0-9])', '', key) @@ -572,7 +599,8 @@ def generate_properties(self): self.l('else: {variable}["{}"] = {}', key, repr(prop_definition['default'])) def generate_pattern_properties(self): - with self.l('if isinstance({variable}, dict):'): + self.create_variable_is_dict() + with self.l('if {variable}_is_dict:'): self.create_variable_keys() for pattern, definition in self._definition['patternProperties'].items(): self._compile_regexps['{}'.format(pattern)] = re.compile(pattern) @@ -588,7 +616,8 @@ def generate_pattern_properties(self): ) def generate_additional_properties(self): - with self.l('if isinstance({variable}, dict):'): + self.create_variable_is_dict() + with self.l('if {variable}_is_dict:'): self.create_variable_keys() add_prop_definition = self._definition["additionalProperties"] if add_prop_definition: @@ -606,7 +635,8 @@ def generate_additional_properties(self): self.l('raise JsonSchemaException("{name} must contain only specified properties")') def generate_dependencies(self): - with self.l('if isinstance({variable}, dict):'): + self.create_variable_is_dict() + with self.l('if {variable}_is_dict:'): self.create_variable_keys() for key, values in self._definition["dependencies"].items(): with self.l('if "{}" in {variable}_keys:', key): diff --git a/schema.json b/schema.json new file mode 100644 index 0000000..a5babda --- /dev/null +++ b/schema.json @@ -0,0 +1,40 @@ +{ + "type": "array", + "items": [ + { + "type": "number", + "maximum": 10, + "exclusiveMaximum": true + }, + { + "type": "string", + "enum": ["hello", "world"] + }, + { + "type": "array", + "minItems": 1, + "maxItems": 3, + "items": [ + {"type": "number"}, + {"type": "string"}, + {"type": "boolean"} + ] + }, + { + "type": "object", + "required": ["a", "b"], + "minProperties": 3, + "properties": { + "a": {"type": ["null", "string"]}, + "b": {"type": ["null", "string"]}, + "c": {"type": ["null", "string"], "default": "abc"} + }, + "additionalProperties": {"type": "string"} + }, + {"not": {"type": ["null"]}}, + {"oneOf": [ + {"type": "number", "multipleOf": 3}, + {"type": "number", "multipleOf": 5} + ]} + ] +} From 475073c4a6e7fb09bad29eb0cfe2cd36038ab70f Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 26 Jul 2018 16:12:39 +0200 Subject: [PATCH 083/201] Fix of additional properties --- fastjsonschema/generator.py | 4 ++-- tests/test_object.py | 22 ++++++++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 13e6de7..76b6f4d 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -621,9 +621,9 @@ def generate_additional_properties(self): self.create_variable_keys() add_prop_definition = self._definition["additionalProperties"] if add_prop_definition: - properties_keys = self._definition.get("properties", {}).keys() + properties_keys = list(self._definition.get("properties", {}).keys()) with self.l('for {variable}_key in {variable}_keys:'): - with self.l('if {variable}_key not in "{}":', properties_keys): + with self.l('if {variable}_key not in {}:', properties_keys): self.l('{variable}_value = {variable}.get({variable}_key)') self.generate_func_code_block( add_prop_definition, diff --git a/tests/test_object.py b/tests/test_object.py index 2c25d89..51d6c6a 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -130,11 +130,29 @@ def test_pattern_properties(asserter, value, expected): 'additionalProperties': False, }, value, expected) + +@pytest.mark.parametrize('value, expected', [ + ({}, {}), + ({'a': 1}, {'a': 1}), + ({'b': True}, {'b': True}), + ({'c': ''}, {'c': ''}), + ({'d': 1}, JsonSchemaException('data.d must be string')), +]) +def test_additional_properties(asserter, value, expected): + asserter({ + 'type': 'object', + "properties": { + "a": {"type": "number"}, + "b": {"type": "boolean"}, + }, + "additionalProperties": {"type": "string"} + }, value, expected) + + @pytest.mark.parametrize('value, expected', [ ({'id': 1}, {'id': 1}), ({'id': 'a'}, JsonSchemaException('data.id must be integer')), ]) - def test_object_with_id_property(asserter, value, expected): asserter({ "type": "object", @@ -143,11 +161,11 @@ def test_object_with_id_property(asserter, value, expected): } }, value, expected) + @pytest.mark.parametrize('value, expected', [ ({'$ref': 'ref://to.somewhere'}, {'$ref': 'ref://to.somewhere'}), ({'$ref': 1}, JsonSchemaException('data.$ref must be string')), ]) - def test_object_with_ref_property(asserter, value, expected): asserter({ "type": "object", From d6f0eb46d2e6966b0b52799a6679ad1bf19fa894 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 26 Jul 2018 16:23:59 +0200 Subject: [PATCH 084/201] Optimalization of one of counting --- fastjsonschema/generator.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 76b6f4d..80d8a26 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -376,6 +376,7 @@ def generate_any_of(self): """ self.l('{variable}_any_of_count = 0') for definition_item in self._definition['anyOf']: + # When we know it's passing (at least once), we do not need to do another expensive try-except. with self.l('if not {variable}_any_of_count:'): with self.l('try:'): self.generate_func_code_block(definition_item, self._variable, self._variable_name, clear_variables=True) @@ -403,10 +404,12 @@ def generate_one_of(self): """ self.l('{variable}_one_of_count = 0') for definition_item in self._definition['oneOf']: - with self.l('try:'): - self.generate_func_code_block(definition_item, self._variable, self._variable_name, clear_variables=True) - self.l('{variable}_one_of_count += 1') - self.l('except JsonSchemaException: pass') + # When we know it's failing (one of means exactly once), we do not need to do another expensive try-except. + with self.l('if {variable}_one_of_count < 2:'): + with self.l('try:'): + self.generate_func_code_block(definition_item, self._variable, self._variable_name, clear_variables=True) + self.l('{variable}_one_of_count += 1') + self.l('except JsonSchemaException: pass') with self.l('if {variable}_one_of_count != 1:'): self.l('raise JsonSchemaException("{name} must be valid exactly by one of oneOf definition")') From d32022c4bebd474f84674de65cf4ccf5fc55cb0a Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 26 Jul 2018 16:29:43 +0200 Subject: [PATCH 085/201] Authors --- AUTHORS | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 AUTHORS diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..f0d8503 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,8 @@ +MAINTAINER +Michal Hořejšek + +CONTRIBUTORS +anentropic +Antti Jokipii +Guillaume Desvé +Kris Molendyke From 8069e1641bdaa7d35e6388b10bd65e3570fa7732 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Fri, 10 Aug 2018 14:25:29 +0000 Subject: [PATCH 086/201] Prepared classes for JSON draft 6 and 7 --- fastjsonschema/__init__.py | 32 +- fastjsonschema/draft04.py | 391 ++++++++++++++++++ fastjsonschema/draft06.py | 5 + fastjsonschema/draft07.py | 5 + fastjsonschema/generator.py | 388 ----------------- fastjsonschema/ref_resolver.py | 6 +- tests/conftest.py | 4 +- .../{test_draft4.py => test_draft04.py} | 3 +- .../{test_draft6.py => test_draft06.py} | 3 +- .../{test_draft7.py => test_draft07.py} | 3 +- tests/json_schema/utils.py | 11 +- 11 files changed, 442 insertions(+), 409 deletions(-) create mode 100644 fastjsonschema/draft04.py create mode 100644 fastjsonschema/draft06.py create mode 100644 fastjsonschema/draft07.py rename tests/json_schema/{test_draft4.py => test_draft04.py} (79%) rename tests/json_schema/{test_draft6.py => test_draft06.py} (95%) rename tests/json_schema/{test_draft7.py => test_draft07.py} (95%) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 472c62f..71b83da 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -1,3 +1,9 @@ +# ___ +# \./ DANGER: This project implements some code generation +# .--.O.--. techniques involving string concatenation. +# \/ \/ If you look at it, you might die. +# + """ This project was made to come up with fast JSON validations. Just let's see some numbers first: @@ -48,7 +54,9 @@ from os.path import exists from .exceptions import JsonSchemaException -from .generator import CodeGenerator +from .draft04 import CodeGeneratorDraft04 +from .draft06 import CodeGeneratorDraft06 +from .draft07 import CodeGeneratorDraft07 from .ref_resolver import RefResolver from .version import VERSION @@ -56,7 +64,7 @@ # pylint: disable=redefined-builtin,dangerous-default-value,exec-used -def compile(definition, handlers={}): +def compile(definition, version=7, handlers={}): """ Generates validation function for validating JSON schema by ``definition``. Example: @@ -88,7 +96,7 @@ def compile(definition, handlers={}): Exception :any:`JsonSchemaException` is thrown when validation fails. """ - resolver, code_generator = _factory(definition, handlers) + resolver, code_generator = _factory(definition, version, handlers) global_state = code_generator.global_state # Do not pass local state so it can recursively call itself. exec(code_generator.func_code, global_state) @@ -96,7 +104,7 @@ def compile(definition, handlers={}): # pylint: disable=dangerous-default-value -def compile_to_code(definition, handlers={}): +def compile_to_code(definition, version=7, handlers={}): """ Generates validation function for validating JSON schema by ``definition`` and returns compiled code. Example: @@ -118,7 +126,7 @@ def compile_to_code(definition, handlers={}): Exception :any:`JsonSchemaException` is thrown when validation fails. """ - _, code_generator = _factory(definition, handlers) + _, code_generator = _factory(definition, version, handlers) return ( 'VERSION = "' + VERSION + '"\n' + code_generator.global_state_code + '\n' + @@ -126,7 +134,17 @@ def compile_to_code(definition, handlers={}): ) -def _factory(definition, handlers): +def _factory(definition, version, handlers): resolver = RefResolver.from_schema(definition, handlers=handlers) - code_generator = CodeGenerator(definition, resolver=resolver) + code_generator = _get_code_generator_class(version)(definition, resolver=resolver) return resolver, code_generator + + +def _get_code_generator_class(version): + if version == 4: + return CodeGeneratorDraft04 + if version == 6: + return CodeGeneratorDraft06 + if version == 7: + return CodeGeneratorDraft07 + raise JsonSchemaException('Unsupported JSON schema version. Supported are 4, 6 and 7.') diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py new file mode 100644 index 0000000..6904a62 --- /dev/null +++ b/fastjsonschema/draft04.py @@ -0,0 +1,391 @@ +from collections import OrderedDict +import re + +from .generator import CodeGenerator, enforce_list + + +# pylint: disable=line-too-long +FORMAT_REGEXS = { + 'date-time': r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+(?:[+-][0-2]\d:[0-5]\d|Z)?$', + 'uri': r'^\w+:(\/?\/?)[^\s]+$', + 'email': r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$', + 'ipv4': r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$', + 'ipv6': r'^(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|::(?:[0-9A-Fa-f]{1,4}:){5}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){4}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){3}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,2}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){2}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,3}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}:(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,4}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,5}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}|(?:(?:[0-9A-Fa-f]{1,4}:){,6}[0-9A-Fa-f]{1,4})?::)$', + 'hostname': r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{1,62}[A-Za-z0-9])$', +} + + +# pylint: disable=too-many-instance-attributes,too-many-public-methods +class CodeGeneratorDraft04(CodeGenerator): + def __init__(self, definition, resolver=None): + self._json_keywords_to_function = OrderedDict(( + ('type', self.generate_type), + ('enum', self.generate_enum), + ('allOf', self.generate_all_of), + ('anyOf', self.generate_any_of), + ('oneOf', self.generate_one_of), + ('not', self.generate_not), + ('minLength', self.generate_min_length), + ('maxLength', self.generate_max_length), + ('pattern', self.generate_pattern), + ('format', self.generate_format), + ('minimum', self.generate_minimum), + ('maximum', self.generate_maximum), + ('multipleOf', self.generate_multiple_of), + ('minItems', self.generate_min_items), + ('maxItems', self.generate_max_items), + ('uniqueItems', self.generate_unique_items), + ('items', self.generate_items), + ('minProperties', self.generate_min_properties), + ('maxProperties', self.generate_max_properties), + ('required', self.generate_required), + ('properties', self.generate_properties), + ('patternProperties', self.generate_pattern_properties), + ('additionalProperties', self.generate_additional_properties), + ('dependencies', self.generate_dependencies), + )) + + super().__init__(definition, resolver) + + def generate_type(self): + """ + Validation of type. Can be one type or list of types. + + .. code-block:: python + + {'type': 'string'} + {'type': ['string', 'number']} + """ + types = enforce_list(self._definition['type']) + python_types = ', '.join(self.JSON_TYPE_TO_PYTHON_TYPE.get(t) for t in types) + + extra = '' + if ('number' in types or 'integer' in types) and 'boolean' not in types: + extra = ' or isinstance({variable}, bool)'.format(variable=self._variable) + + with self.l('if not isinstance({variable}, ({})){}:', python_types, extra): + self.l('raise JsonSchemaException("{name} must be {}")', ' or '.join(types)) + + def generate_enum(self): + """ + Means that only value specified in the enum is valid. + + .. code-block:: python + + { + 'enum': ['a', 'b'], + } + """ + with self.l('if {variable} not in {enum}:'): + self.l('raise JsonSchemaException("{name} must be one of {enum}")') + + def generate_all_of(self): + """ + Means that value have to be valid by all of those definitions. It's like put it in + one big definition. + + .. code-block:: python + + { + 'allOf': [ + {'type': 'number'}, + {'minimum': 5}, + ], + } + + Valid values for this definition are 5, 6, 7, ... but not 4 or 'abc' for example. + """ + for definition_item in self._definition['allOf']: + self.generate_func_code_block(definition_item, self._variable, self._variable_name, clear_variables=True) + + def generate_any_of(self): + """ + Means that value have to be valid by any of those definitions. It can also be valid + by all of them. + + .. code-block:: python + + { + 'anyOf': [ + {'type': 'number', 'minimum': 10}, + {'type': 'number', 'maximum': 5}, + ], + } + + Valid values for this definition are 3, 4, 5, 10, 11, ... but not 8 for example. + """ + self.l('{variable}_any_of_count = 0') + for definition_item in self._definition['anyOf']: + # When we know it's passing (at least once), we do not need to do another expensive try-except. + with self.l('if not {variable}_any_of_count:'): + with self.l('try:'): + self.generate_func_code_block(definition_item, self._variable, self._variable_name, clear_variables=True) + self.l('{variable}_any_of_count += 1') + self.l('except JsonSchemaException: pass') + + with self.l('if not {variable}_any_of_count:'): + self.l('raise JsonSchemaException("{name} must be valid by one of anyOf definition")') + + def generate_one_of(self): + """ + Means that value have to be valid by only one of those definitions. It can't be valid + by two or more of them. + + .. code-block:: python + + { + 'oneOf': [ + {'type': 'number', 'multipleOf': 3}, + {'type': 'number', 'multipleOf': 5}, + ], + } + + Valid values for this definitions are 3, 5, 6, ... but not 15 for example. + """ + self.l('{variable}_one_of_count = 0') + for definition_item in self._definition['oneOf']: + # When we know it's failing (one of means exactly once), we do not need to do another expensive try-except. + with self.l('if {variable}_one_of_count < 2:'): + with self.l('try:'): + self.generate_func_code_block(definition_item, self._variable, self._variable_name, clear_variables=True) + self.l('{variable}_one_of_count += 1') + self.l('except JsonSchemaException: pass') + + with self.l('if {variable}_one_of_count != 1:'): + self.l('raise JsonSchemaException("{name} must be valid exactly by one of oneOf definition")') + + def generate_not(self): + """ + Means that value have not to be valid by this definition. + + .. code-block:: python + + {'not': {'type': 'null'}} + + Valid values for this definitions are 'hello', 42, {} ... but not None. + """ + if not self._definition['not']: + with self.l('if {}:', self._variable): + self.l('raise JsonSchemaException("{name} must not be valid by not definition")') + else: + with self.l('try:'): + self.generate_func_code_block(self._definition['not'], self._variable, self._variable_name) + self.l('except JsonSchemaException: pass') + self.l('else: raise JsonSchemaException("{name} must not be valid by not definition")') + + def generate_min_length(self): + with self.l('if isinstance({variable}, str):'): + self.create_variable_with_length() + with self.l('if {variable}_len < {minLength}:'): + self.l('raise JsonSchemaException("{name} must be longer than or equal to {minLength} characters")') + + def generate_max_length(self): + with self.l('if isinstance({variable}, str):'): + self.create_variable_with_length() + with self.l('if {variable}_len > {maxLength}:'): + self.l('raise JsonSchemaException("{name} must be shorter than or equal to {maxLength} characters")') + + def generate_pattern(self): + with self.l('if isinstance({variable}, str):'): + self._compile_regexps['{}'.format(self._definition['pattern'])] = re.compile(self._definition['pattern']) + with self.l('if not REGEX_PATTERNS["{}"].search({variable}):', self._definition['pattern']): + self.l('raise JsonSchemaException("{name} must match pattern {pattern}")') + + def generate_format(self): + with self.l('if isinstance({variable}, str):'): + format_ = self._definition['format'] + if format_ in FORMAT_REGEXS: + format_regex = FORMAT_REGEXS[format_] + self._generate_format(format_, format_ + '_re_pattern', format_regex) + # format regex is used only in meta schemas + elif format_ == 'regex': + with self.l('try:'): + self.l('re.compile({variable})') + with self.l('except Exception:'): + self.l('raise JsonSchemaException("{name} must be a valid regex")') + + def _generate_format(self, format_name, regexp_name, regexp): + if self._definition['format'] == format_name: + if not regexp_name in self._compile_regexps: + self._compile_regexps[regexp_name] = re.compile(regexp) + with self.l('if not REGEX_PATTERNS["{}"].match({variable}):', regexp_name): + self.l('raise JsonSchemaException("{name} must be {}")', format_name) + + def generate_minimum(self): + with self.l('if isinstance({variable}, (int, float)):'): + if self._definition.get('exclusiveMinimum', False): + with self.l('if {variable} <= {minimum}:'): + self.l('raise JsonSchemaException("{name} must be bigger than {minimum}")') + else: + with self.l('if {variable} < {minimum}:'): + self.l('raise JsonSchemaException("{name} must be bigger than or equal to {minimum}")') + + def generate_maximum(self): + with self.l('if isinstance({variable}, (int, float)):'): + if self._definition.get('exclusiveMaximum', False): + with self.l('if {variable} >= {maximum}:'): + self.l('raise JsonSchemaException("{name} must be smaller than {maximum}")') + else: + with self.l('if {variable} > {maximum}:'): + self.l('raise JsonSchemaException("{name} must be smaller than or equal to {maximum}")') + + def generate_multiple_of(self): + with self.l('if isinstance({variable}, (int, float)):'): + self.l('quotient = {variable} / {multipleOf}') + with self.l('if int(quotient) != quotient:'): + self.l('raise JsonSchemaException("{name} must be multiple of {multipleOf}")') + + def generate_min_items(self): + self.create_variable_is_list() + with self.l('if {variable}_is_list:'): + self.create_variable_with_length() + with self.l('if {variable}_len < {minItems}:'): + self.l('raise JsonSchemaException("{name} must contain at least {minItems} items")') + + def generate_max_items(self): + self.create_variable_is_list() + with self.l('if {variable}_is_list:'): + self.create_variable_with_length() + with self.l('if {variable}_len > {maxItems}:'): + self.l('raise JsonSchemaException("{name} must contain less than or equal to {maxItems} items")') + + def generate_unique_items(self): + """ + With Python 3.4 module ``timeit`` recommended this solutions: + + .. code-block:: python + + >>> timeit.timeit("len(x) > len(set(x))", "x=range(100)+range(100)", number=100000) + 0.5839540958404541 + >>> timeit.timeit("len({}.fromkeys(x)) == len(x)", "x=range(100)+range(100)", number=100000) + 0.7094449996948242 + >>> timeit.timeit("seen = set(); any(i in seen or seen.add(i) for i in x)", "x=range(100)+range(100)", number=100000) + 2.0819358825683594 + >>> timeit.timeit("np.unique(x).size == len(x)", "x=range(100)+range(100); import numpy as np", number=100000) + 2.1439831256866455 + """ + self.create_variable_with_length() + with self.l('if {variable}_len > len(set(str(x) for x in {variable})):'): + self.l('raise JsonSchemaException("{name} must contain unique items")') + + def generate_items(self): + self.create_variable_is_list() + with self.l('if {variable}_is_list:'): + self.create_variable_with_length() + if isinstance(self._definition['items'], list): + for x, item_definition in enumerate(self._definition['items']): + with self.l('if {variable}_len > {}:', x): + self.l('{variable}_{0} = {variable}[{0}]', x) + self.generate_func_code_block( + item_definition, + '{}_{}'.format(self._variable, x), + '{}[{}]'.format(self._variable_name, x), + ) + if 'default' in item_definition: + self.l('else: {variable}.append({})', repr(item_definition['default'])) + + if 'additionalItems' in self._definition: + if self._definition['additionalItems'] is False: + self.l('if {variable}_len > {}: raise JsonSchemaException("{name} must contain only specified items")', len(self._definition['items'])) + else: + with self.l('for {variable}_x, {variable}_item in enumerate({variable}[{0}:], {0}):', len(self._definition['items'])): + self.generate_func_code_block( + self._definition['additionalItems'], + '{}_item'.format(self._variable), + '{}[{{{}_x}}]'.format(self._variable_name, self._variable), + ) + else: + if self._definition['items']: + with self.l('for {variable}_x, {variable}_item in enumerate({variable}):'): + self.generate_func_code_block( + self._definition['items'], + '{}_item'.format(self._variable), + '{}[{{{}_x}}]'.format(self._variable_name, self._variable), + ) + + def generate_min_properties(self): + self.create_variable_is_dict() + with self.l('if {variable}_is_dict:'): + self.create_variable_with_length() + with self.l('if {variable}_len < {minProperties}:'): + self.l('raise JsonSchemaException("{name} must contain at least {minProperties} properties")') + + def generate_max_properties(self): + self.create_variable_is_dict() + with self.l('if {variable}_is_dict:'): + self.create_variable_with_length() + with self.l('if {variable}_len > {maxProperties}:'): + self.l('raise JsonSchemaException("{name} must contain less than or equal to {maxProperties} properties")') + + def generate_required(self): + self.create_variable_is_dict() + with self.l('if {variable}_is_dict:'): + self.create_variable_with_length() + with self.l('if not all(prop in {variable} for prop in {required}):'): + self.l('raise JsonSchemaException("{name} must contain {required} properties")') + + def generate_properties(self): + self.create_variable_is_dict() + with self.l('if {variable}_is_dict:'): + self.create_variable_keys() + for key, prop_definition in self._definition['properties'].items(): + key_name = re.sub(r'($[^a-zA-Z]|[^a-zA-Z0-9])', '', key) + with self.l('if "{}" in {variable}_keys:', key): + self.l('{variable}_keys.remove("{}")', key) + self.l('{variable}_{0} = {variable}["{1}"]', key_name, key) + self.generate_func_code_block( + prop_definition, + '{}_{}'.format(self._variable, key_name), + '{}.{}'.format(self._variable_name, key), + ) + if 'default' in prop_definition: + self.l('else: {variable}["{}"] = {}', key, repr(prop_definition['default'])) + + def generate_pattern_properties(self): + self.create_variable_is_dict() + with self.l('if {variable}_is_dict:'): + self.create_variable_keys() + for pattern, definition in self._definition['patternProperties'].items(): + self._compile_regexps['{}'.format(pattern)] = re.compile(pattern) + with self.l('for key, val in {variable}.items():'): + for pattern, definition in self._definition['patternProperties'].items(): + with self.l('if REGEX_PATTERNS["{}"].search(key):', pattern): + with self.l('if key in {variable}_keys:'): + self.l('{variable}_keys.remove(key)') + self.generate_func_code_block( + definition, + 'val', + '{}.{{key}}'.format(self._variable_name), + ) + + def generate_additional_properties(self): + self.create_variable_is_dict() + with self.l('if {variable}_is_dict:'): + self.create_variable_keys() + add_prop_definition = self._definition["additionalProperties"] + if add_prop_definition: + properties_keys = list(self._definition.get("properties", {}).keys()) + with self.l('for {variable}_key in {variable}_keys:'): + with self.l('if {variable}_key not in {}:', properties_keys): + self.l('{variable}_value = {variable}.get({variable}_key)') + self.generate_func_code_block( + add_prop_definition, + '{}_value'.format(self._variable), + '{}.{{{}_key}}'.format(self._variable_name, self._variable), + ) + else: + with self.l('if {variable}_keys:'): + self.l('raise JsonSchemaException("{name} must contain only specified properties")') + + def generate_dependencies(self): + self.create_variable_is_dict() + with self.l('if {variable}_is_dict:'): + self.create_variable_keys() + for key, values in self._definition["dependencies"].items(): + with self.l('if "{}" in {variable}_keys:', key): + if isinstance(values, list): + for value in values: + with self.l('if "{}" not in {variable}_keys:', value): + self.l('raise JsonSchemaException("{name} missing dependency {} for {}")', value, key) + else: + self.generate_func_code_block(values, self._variable, self._variable_name, clear_variables=True) diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py new file mode 100644 index 0000000..a711047 --- /dev/null +++ b/fastjsonschema/draft06.py @@ -0,0 +1,5 @@ +from .draft04 import CodeGeneratorDraft04 + + +class CodeGeneratorDraft06(CodeGeneratorDraft04): + pass diff --git a/fastjsonschema/draft07.py b/fastjsonschema/draft07.py new file mode 100644 index 0000000..c9f44b7 --- /dev/null +++ b/fastjsonschema/draft07.py @@ -0,0 +1,5 @@ +from .draft06 import CodeGeneratorDraft06 + + +class CodeGeneratorDraft07(CodeGeneratorDraft06): + pass diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 80d8a26..1407b91 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -1,10 +1,3 @@ -# ___ -# \./ DANGER: This module implements some code generation -# .--.O.--. techniques involving string concatenation. -# \/ \/ If you look at it, you might die. -# - -from collections import OrderedDict import re from .exceptions import JsonSchemaException @@ -12,17 +5,6 @@ from .ref_resolver import RefResolver -# pylint: disable=line-too-long -FORMAT_REGEXS = { - 'date-time': r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+(?:[+-][0-2]\d:[0-5]\d|Z)?$', - 'uri': r'^\w+:(\/?\/?)[^\s]+$', - 'email': r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$', - 'ipv4': r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$', - 'ipv6': r'^(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|::(?:[0-9A-Fa-f]{1,4}:){5}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){4}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){3}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,2}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){2}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,3}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}:(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,4}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,5}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}|(?:(?:[0-9A-Fa-f]{1,4}:){,6}[0-9A-Fa-f]{1,4})?::)$', - 'hostname': r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{1,62}[A-Za-z0-9])$', -} - - def enforce_list(variable): if isinstance(variable, list): return variable @@ -78,33 +60,6 @@ def __init__(self, definition, resolver=None): # add main function to `self._needed_validation_functions` self._needed_validation_functions[self._resolver.get_uri()] = self._resolver.get_scope_name() - self._json_keywords_to_function = OrderedDict(( - ('type', self.generate_type), - ('enum', self.generate_enum), - ('allOf', self.generate_all_of), - ('anyOf', self.generate_any_of), - ('oneOf', self.generate_one_of), - ('not', self.generate_not), - ('minLength', self.generate_min_length), - ('maxLength', self.generate_max_length), - ('pattern', self.generate_pattern), - ('format', self.generate_format), - ('minimum', self.generate_minimum), - ('maximum', self.generate_maximum), - ('multipleOf', self.generate_multiple_of), - ('minItems', self.generate_min_items), - ('maxItems', self.generate_max_items), - ('uniqueItems', self.generate_unique_items), - ('items', self.generate_items), - ('minProperties', self.generate_min_properties), - ('maxProperties', self.generate_max_properties), - ('required', self.generate_required), - ('properties', self.generate_properties), - ('patternProperties', self.generate_pattern_properties), - ('additionalProperties', self.generate_additional_properties), - ('dependencies', self.generate_dependencies), - )) - self.generate_func_code() @property @@ -306,346 +261,3 @@ def generate_ref(self): self._needed_validation_functions[uri] = name # call validation function self.l('{}({variable})', name) - - def generate_type(self): - """ - Validation of type. Can be one type or list of types. - - .. code-block:: python - - {'type': 'string'} - {'type': ['string', 'number']} - """ - types = enforce_list(self._definition['type']) - python_types = ', '.join(self.JSON_TYPE_TO_PYTHON_TYPE.get(t) for t in types) - - extra = '' - if ('number' in types or 'integer' in types) and 'boolean' not in types: - extra = ' or isinstance({variable}, bool)'.format(variable=self._variable) - - with self.l('if not isinstance({variable}, ({})){}:', python_types, extra): - self.l('raise JsonSchemaException("{name} must be {}")', ' or '.join(types)) - - def generate_enum(self): - """ - Means that only value specified in the enum is valid. - - .. code-block:: python - - { - 'enum': ['a', 'b'], - } - """ - with self.l('if {variable} not in {enum}:'): - self.l('raise JsonSchemaException("{name} must be one of {enum}")') - - def generate_all_of(self): - """ - Means that value have to be valid by all of those definitions. It's like put it in - one big definition. - - .. code-block:: python - - { - 'allOf': [ - {'type': 'number'}, - {'minimum': 5}, - ], - } - - Valid values for this definition are 5, 6, 7, ... but not 4 or 'abc' for example. - """ - for definition_item in self._definition['allOf']: - self.generate_func_code_block(definition_item, self._variable, self._variable_name, clear_variables=True) - - def generate_any_of(self): - """ - Means that value have to be valid by any of those definitions. It can also be valid - by all of them. - - .. code-block:: python - - { - 'anyOf': [ - {'type': 'number', 'minimum': 10}, - {'type': 'number', 'maximum': 5}, - ], - } - - Valid values for this definition are 3, 4, 5, 10, 11, ... but not 8 for example. - """ - self.l('{variable}_any_of_count = 0') - for definition_item in self._definition['anyOf']: - # When we know it's passing (at least once), we do not need to do another expensive try-except. - with self.l('if not {variable}_any_of_count:'): - with self.l('try:'): - self.generate_func_code_block(definition_item, self._variable, self._variable_name, clear_variables=True) - self.l('{variable}_any_of_count += 1') - self.l('except JsonSchemaException: pass') - - with self.l('if not {variable}_any_of_count:'): - self.l('raise JsonSchemaException("{name} must be valid by one of anyOf definition")') - - def generate_one_of(self): - """ - Means that value have to be valid by only one of those definitions. It can't be valid - by two or more of them. - - .. code-block:: python - - { - 'oneOf': [ - {'type': 'number', 'multipleOf': 3}, - {'type': 'number', 'multipleOf': 5}, - ], - } - - Valid values for this definitions are 3, 5, 6, ... but not 15 for example. - """ - self.l('{variable}_one_of_count = 0') - for definition_item in self._definition['oneOf']: - # When we know it's failing (one of means exactly once), we do not need to do another expensive try-except. - with self.l('if {variable}_one_of_count < 2:'): - with self.l('try:'): - self.generate_func_code_block(definition_item, self._variable, self._variable_name, clear_variables=True) - self.l('{variable}_one_of_count += 1') - self.l('except JsonSchemaException: pass') - - with self.l('if {variable}_one_of_count != 1:'): - self.l('raise JsonSchemaException("{name} must be valid exactly by one of oneOf definition")') - - def generate_not(self): - """ - Means that value have not to be valid by this definition. - - .. code-block:: python - - {'not': {'type': 'null'}} - - Valid values for this definitions are 'hello', 42, {} ... but not None. - """ - if not self._definition['not']: - with self.l('if {}:', self._variable): - self.l('raise JsonSchemaException("{name} must not be valid by not definition")') - else: - with self.l('try:'): - self.generate_func_code_block(self._definition['not'], self._variable, self._variable_name) - self.l('except JsonSchemaException: pass') - self.l('else: raise JsonSchemaException("{name} must not be valid by not definition")') - - def generate_min_length(self): - with self.l('if isinstance({variable}, str):'): - self.create_variable_with_length() - with self.l('if {variable}_len < {minLength}:'): - self.l('raise JsonSchemaException("{name} must be longer than or equal to {minLength} characters")') - - def generate_max_length(self): - with self.l('if isinstance({variable}, str):'): - self.create_variable_with_length() - with self.l('if {variable}_len > {maxLength}:'): - self.l('raise JsonSchemaException("{name} must be shorter than or equal to {maxLength} characters")') - - def generate_pattern(self): - with self.l('if isinstance({variable}, str):'): - self._compile_regexps['{}'.format(self._definition['pattern'])] = re.compile(self._definition['pattern']) - with self.l('if not REGEX_PATTERNS["{}"].search({variable}):', self._definition['pattern']): - self.l('raise JsonSchemaException("{name} must match pattern {pattern}")') - - def generate_format(self): - with self.l('if isinstance({variable}, str):'): - format_ = self._definition['format'] - if format_ in FORMAT_REGEXS: - format_regex = FORMAT_REGEXS[format_] - self._generate_format(format_, format_ + '_re_pattern', format_regex) - # format regex is used only in meta schemas - elif format_ == 'regex': - with self.l('try:'): - self.l('re.compile({variable})') - with self.l('except Exception:'): - self.l('raise JsonSchemaException("{name} must be a valid regex")') - - def _generate_format(self, format_name, regexp_name, regexp): - if self._definition['format'] == format_name: - if not regexp_name in self._compile_regexps: - self._compile_regexps[regexp_name] = re.compile(regexp) - with self.l('if not REGEX_PATTERNS["{}"].match({variable}):', regexp_name): - self.l('raise JsonSchemaException("{name} must be {}")', format_name) - - def generate_minimum(self): - with self.l('if isinstance({variable}, (int, float)):'): - if self._definition.get('exclusiveMinimum', False): - with self.l('if {variable} <= {minimum}:'): - self.l('raise JsonSchemaException("{name} must be bigger than {minimum}")') - else: - with self.l('if {variable} < {minimum}:'): - self.l('raise JsonSchemaException("{name} must be bigger than or equal to {minimum}")') - - def generate_maximum(self): - with self.l('if isinstance({variable}, (int, float)):'): - if self._definition.get('exclusiveMaximum', False): - with self.l('if {variable} >= {maximum}:'): - self.l('raise JsonSchemaException("{name} must be smaller than {maximum}")') - else: - with self.l('if {variable} > {maximum}:'): - self.l('raise JsonSchemaException("{name} must be smaller than or equal to {maximum}")') - - def generate_multiple_of(self): - with self.l('if isinstance({variable}, (int, float)):'): - self.l('quotient = {variable} / {multipleOf}') - with self.l('if int(quotient) != quotient:'): - self.l('raise JsonSchemaException("{name} must be multiple of {multipleOf}")') - - def generate_min_items(self): - self.create_variable_is_list() - with self.l('if {variable}_is_list:'): - self.create_variable_with_length() - with self.l('if {variable}_len < {minItems}:'): - self.l('raise JsonSchemaException("{name} must contain at least {minItems} items")') - - def generate_max_items(self): - self.create_variable_is_list() - with self.l('if {variable}_is_list:'): - self.create_variable_with_length() - with self.l('if {variable}_len > {maxItems}:'): - self.l('raise JsonSchemaException("{name} must contain less than or equal to {maxItems} items")') - - def generate_unique_items(self): - """ - With Python 3.4 module ``timeit`` recommended this solutions: - - .. code-block:: python - - >>> timeit.timeit("len(x) > len(set(x))", "x=range(100)+range(100)", number=100000) - 0.5839540958404541 - >>> timeit.timeit("len({}.fromkeys(x)) == len(x)", "x=range(100)+range(100)", number=100000) - 0.7094449996948242 - >>> timeit.timeit("seen = set(); any(i in seen or seen.add(i) for i in x)", "x=range(100)+range(100)", number=100000) - 2.0819358825683594 - >>> timeit.timeit("np.unique(x).size == len(x)", "x=range(100)+range(100); import numpy as np", number=100000) - 2.1439831256866455 - """ - self.create_variable_with_length() - with self.l('if {variable}_len > len(set(str(x) for x in {variable})):'): - self.l('raise JsonSchemaException("{name} must contain unique items")') - - def generate_items(self): - self.create_variable_is_list() - with self.l('if {variable}_is_list:'): - self.create_variable_with_length() - if isinstance(self._definition['items'], list): - for x, item_definition in enumerate(self._definition['items']): - with self.l('if {variable}_len > {}:', x): - self.l('{variable}_{0} = {variable}[{0}]', x) - self.generate_func_code_block( - item_definition, - '{}_{}'.format(self._variable, x), - '{}[{}]'.format(self._variable_name, x), - ) - if 'default' in item_definition: - self.l('else: {variable}.append({})', repr(item_definition['default'])) - - if 'additionalItems' in self._definition: - if self._definition['additionalItems'] is False: - self.l('if {variable}_len > {}: raise JsonSchemaException("{name} must contain only specified items")', len(self._definition['items'])) - else: - with self.l('for {variable}_x, {variable}_item in enumerate({variable}[{0}:], {0}):', len(self._definition['items'])): - self.generate_func_code_block( - self._definition['additionalItems'], - '{}_item'.format(self._variable), - '{}[{{{}_x}}]'.format(self._variable_name, self._variable), - ) - else: - if self._definition['items']: - with self.l('for {variable}_x, {variable}_item in enumerate({variable}):'): - self.generate_func_code_block( - self._definition['items'], - '{}_item'.format(self._variable), - '{}[{{{}_x}}]'.format(self._variable_name, self._variable), - ) - - def generate_min_properties(self): - self.create_variable_is_dict() - with self.l('if {variable}_is_dict:'): - self.create_variable_with_length() - with self.l('if {variable}_len < {minProperties}:'): - self.l('raise JsonSchemaException("{name} must contain at least {minProperties} properties")') - - def generate_max_properties(self): - self.create_variable_is_dict() - with self.l('if {variable}_is_dict:'): - self.create_variable_with_length() - with self.l('if {variable}_len > {maxProperties}:'): - self.l('raise JsonSchemaException("{name} must contain less than or equal to {maxProperties} properties")') - - def generate_required(self): - self.create_variable_is_dict() - with self.l('if {variable}_is_dict:'): - self.create_variable_with_length() - with self.l('if not all(prop in {variable} for prop in {required}):'): - self.l('raise JsonSchemaException("{name} must contain {required} properties")') - - def generate_properties(self): - self.create_variable_is_dict() - with self.l('if {variable}_is_dict:'): - self.create_variable_keys() - for key, prop_definition in self._definition['properties'].items(): - key_name = re.sub(r'($[^a-zA-Z]|[^a-zA-Z0-9])', '', key) - with self.l('if "{}" in {variable}_keys:', key): - self.l('{variable}_keys.remove("{}")', key) - self.l('{variable}_{0} = {variable}["{1}"]', key_name, key) - self.generate_func_code_block( - prop_definition, - '{}_{}'.format(self._variable, key_name), - '{}.{}'.format(self._variable_name, key), - ) - if 'default' in prop_definition: - self.l('else: {variable}["{}"] = {}', key, repr(prop_definition['default'])) - - def generate_pattern_properties(self): - self.create_variable_is_dict() - with self.l('if {variable}_is_dict:'): - self.create_variable_keys() - for pattern, definition in self._definition['patternProperties'].items(): - self._compile_regexps['{}'.format(pattern)] = re.compile(pattern) - with self.l('for key, val in {variable}.items():'): - for pattern, definition in self._definition['patternProperties'].items(): - with self.l('if REGEX_PATTERNS["{}"].search(key):', pattern): - with self.l('if key in {variable}_keys:'): - self.l('{variable}_keys.remove(key)') - self.generate_func_code_block( - definition, - 'val', - '{}.{{key}}'.format(self._variable_name), - ) - - def generate_additional_properties(self): - self.create_variable_is_dict() - with self.l('if {variable}_is_dict:'): - self.create_variable_keys() - add_prop_definition = self._definition["additionalProperties"] - if add_prop_definition: - properties_keys = list(self._definition.get("properties", {}).keys()) - with self.l('for {variable}_key in {variable}_keys:'): - with self.l('if {variable}_key not in {}:', properties_keys): - self.l('{variable}_value = {variable}.get({variable}_key)') - self.generate_func_code_block( - add_prop_definition, - '{}_value'.format(self._variable), - '{}.{{{}_key}}'.format(self._variable_name, self._variable), - ) - else: - with self.l('if {variable}_keys:'): - self.l('raise JsonSchemaException("{name} must contain only specified properties")') - - def generate_dependencies(self): - self.create_variable_is_dict() - with self.l('if {variable}_is_dict:'): - self.create_variable_keys() - for key, values in self._definition["dependencies"].items(): - with self.l('if "{}" in {variable}_keys:', key): - if isinstance(values, list): - for value in values: - with self.l('if "{}" not in {variable}_keys:', value): - self.l('raise JsonSchemaException("{name} missing dependency {} for {}")', value, key) - else: - self.generate_func_code_block(values, self._variable, self._variable_name, clear_variables=True) diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index d32e2e3..8bd1ead 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -5,12 +5,13 @@ Code adapted from https://github.com/Julian/jsonschema """ + import contextlib +import json import re from urllib import parse as urlparse from urllib.parse import unquote from urllib.request import urlopen -import json import requests @@ -85,7 +86,6 @@ class RefResolver: first resolution :argument dict handlers: a mapping from URI schemes to functions that should be used to retrieve them - """ # pylint: disable=dangerous-default-value,too-many-arguments @@ -105,7 +105,6 @@ def from_schema(cls, schema, handlers={}, **kwargs): :argument schema schema: the referring schema :rtype: :class:`RefResolver` - """ return cls(schema.get('id', ''), schema, handlers=handlers, **kwargs) @@ -125,7 +124,6 @@ def resolving(self, ref): resolution scope of this ref. :argument str ref: reference to resolve - """ new_uri = urlparse.urljoin(self.resolution_scope, ref) uri, fragment = urlparse.urldefrag(new_uri) diff --git a/tests/conftest.py b/tests/conftest.py index 2d77249..fb7f61d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,14 +10,14 @@ import pytest from fastjsonschema import JsonSchemaException, compile -from fastjsonschema.generator import CodeGenerator +from fastjsonschema.draft07 import CodeGeneratorDraft07 @pytest.fixture def asserter(): def f(definition, value, expected): # When test fails, it will show up code. - code_generator = CodeGenerator(definition) + code_generator = CodeGeneratorDraft07(definition) print(code_generator.func_code) pprint(code_generator.global_state) diff --git a/tests/json_schema/test_draft4.py b/tests/json_schema/test_draft04.py similarity index 79% rename from tests/json_schema/test_draft4.py rename to tests/json_schema/test_draft04.py index 5c76970..5c00861 100644 --- a/tests/json_schema/test_draft4.py +++ b/tests/json_schema/test_draft04.py @@ -5,13 +5,14 @@ def pytest_generate_tests(metafunc): param_values, param_ids = resolve_param_values_and_ids( + version=4, suite_dir='JSON-Schema-Test-Suite/tests/draft4', ignored_suite_files=[ 'ecmascript-regex.json', ], ignore_tests=[], ) - metafunc.parametrize(['schema', 'data', 'is_valid'], param_values, ids=param_ids) + metafunc.parametrize(['version', 'schema', 'data', 'is_valid'], param_values, ids=param_ids) # Real test function to be used with parametrization by previous hook function. diff --git a/tests/json_schema/test_draft6.py b/tests/json_schema/test_draft06.py similarity index 95% rename from tests/json_schema/test_draft6.py rename to tests/json_schema/test_draft06.py index b2dae1d..d933587 100644 --- a/tests/json_schema/test_draft6.py +++ b/tests/json_schema/test_draft06.py @@ -5,6 +5,7 @@ def pytest_generate_tests(metafunc): param_values, param_ids = resolve_param_values_and_ids( + version=6, suite_dir='JSON-Schema-Test-Suite/tests/draft6', ignored_suite_files=[ 'bignum.json', @@ -56,7 +57,7 @@ def pytest_generate_tests(metafunc): '$ref to boolean schema false', ], ) - metafunc.parametrize(['schema', 'data', 'is_valid'], param_values, ids=param_ids) + metafunc.parametrize(['version', 'schema', 'data', 'is_valid'], param_values, ids=param_ids) # Real test function to be used with parametrization by previous hook function. diff --git a/tests/json_schema/test_draft7.py b/tests/json_schema/test_draft07.py similarity index 95% rename from tests/json_schema/test_draft7.py rename to tests/json_schema/test_draft07.py index d26b9c4..120b639 100644 --- a/tests/json_schema/test_draft7.py +++ b/tests/json_schema/test_draft07.py @@ -5,6 +5,7 @@ def pytest_generate_tests(metafunc): param_values, param_ids = resolve_param_values_and_ids( + version=7, suite_dir='JSON-Schema-Test-Suite/tests/draft7', ignored_suite_files=[ 'bignum.json', @@ -65,7 +66,7 @@ def pytest_generate_tests(metafunc): '$ref to boolean schema false', ], ) - metafunc.parametrize(['schema', 'data', 'is_valid'], param_values, ids=param_ids) + metafunc.parametrize(['version', 'schema', 'data', 'is_valid'], param_values, ids=param_ids) # Real test function to be used with parametrization by previous hook function. diff --git a/tests/json_schema/utils.py b/tests/json_schema/utils.py index 16fe43a..c1375e2 100644 --- a/tests/json_schema/utils.py +++ b/tests/json_schema/utils.py @@ -4,7 +4,7 @@ import pytest import requests -from fastjsonschema import CodeGenerator, RefResolver, JsonSchemaException, compile +from fastjsonschema import RefResolver, JsonSchemaException, compile, _get_code_generator_class REMOTES = { @@ -29,7 +29,7 @@ def remotes_handler(uri): return requests.get(uri).json() -def resolve_param_values_and_ids(suite_dir, ignored_suite_files, ignore_tests): +def resolve_param_values_and_ids(version, suite_dir, ignored_suite_files, ignore_tests): suite_dir_path = Path(suite_dir).resolve() test_file_paths = sorted(set(suite_dir_path.glob("**/*.json"))) @@ -41,6 +41,7 @@ def resolve_param_values_and_ids(suite_dir, ignored_suite_files, ignore_tests): for test_case in test_cases: for test_data in test_case['tests']: param_values.append(pytest.param( + version, test_case['schema'], test_data['data'], test_data['valid'], @@ -57,16 +58,16 @@ def resolve_param_values_and_ids(suite_dir, ignored_suite_files, ignore_tests): return param_values, param_ids -def template_test(schema, data, is_valid): +def template_test(version, schema, data, is_valid): """ Test function to be used (imported) in final test file to run the tests which are generated by `pytest_generate_tests` hook. """ # For debug purposes. When test fails, it will print stdout. resolver = RefResolver.from_schema(schema, handlers={'http': remotes_handler}) - print(CodeGenerator(schema, resolver=resolver).func_code) + print(_get_code_generator_class(version)(schema, resolver=resolver).func_code) - validate = compile(schema, handlers={'http': remotes_handler}) + validate = compile(schema, version=version, handlers={'http': remotes_handler}) try: result = validate(data) print('Validate result:', result) From ddb56c17449190e81c7056506abce93a1d792019 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 13 Aug 2018 16:11:23 +0200 Subject: [PATCH 087/201] Support of new keywords from JSON schema draft 06 and 07 --- fastjsonschema/draft04.py | 16 ++- fastjsonschema/draft06.py | 115 ++++++++++++++++++++- fastjsonschema/draft07.py | 97 +++++++++++++++++- fastjsonschema/generator.py | 159 ++++++++++++++++-------------- tests/conftest.py | 6 +- tests/json_schema/test_draft06.py | 7 -- tests/json_schema/test_draft07.py | 9 -- 7 files changed, 306 insertions(+), 103 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 6904a62..cd75213 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -1,4 +1,3 @@ -from collections import OrderedDict import re from .generator import CodeGenerator, enforce_list @@ -18,7 +17,8 @@ # pylint: disable=too-many-instance-attributes,too-many-public-methods class CodeGeneratorDraft04(CodeGenerator): def __init__(self, definition, resolver=None): - self._json_keywords_to_function = OrderedDict(( + super().__init__(definition, resolver) + self._json_keywords_to_function.update(( ('type', self.generate_type), ('enum', self.generate_enum), ('allOf', self.generate_all_of), @@ -45,8 +45,6 @@ def __init__(self, definition, resolver=None): ('dependencies', self.generate_dependencies), )) - super().__init__(definition, resolver) - def generate_type(self): """ Validation of type. Can be one type or list of types. @@ -273,13 +271,13 @@ def generate_items(self): with self.l('if {variable}_is_list:'): self.create_variable_with_length() if isinstance(self._definition['items'], list): - for x, item_definition in enumerate(self._definition['items']): - with self.l('if {variable}_len > {}:', x): - self.l('{variable}_{0} = {variable}[{0}]', x) + for idx, item_definition in enumerate(self._definition['items']): + with self.l('if {variable}_len > {}:', idx): + self.l('{variable}_{0} = {variable}[{0}]', idx) self.generate_func_code_block( item_definition, - '{}_{}'.format(self._variable, x), - '{}[{}]'.format(self._variable_name, x), + '{}_{}'.format(self._variable, idx), + '{}[{}]'.format(self._variable_name, idx), ) if 'default' in item_definition: self.l('else: {variable}.append({})', repr(item_definition['default'])) diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py index a711047..7118537 100644 --- a/fastjsonschema/draft06.py +++ b/fastjsonschema/draft06.py @@ -2,4 +2,117 @@ class CodeGeneratorDraft06(CodeGeneratorDraft04): - pass + def __init__(self, definition, resolver=None): + super().__init__(definition, resolver) + self._json_keywords_to_function.update(( + ('exclusiveMinimum', self.generate_exclusive_minimum), + ('exclusiveMaximum', self.generate_exclusive_maximum), + ('propertyNames', self.generate_property_names), + ('contains', self.generate_contains), + ('const', self.generate_const), + )) + + def generate_exclusive_minimum(self): + with self.l('if isinstance({variable}, (int, float)):'): + with self.l('if {variable} <= {exclusiveMinimum}:'): + self.l('raise JsonSchemaException("{name} must be bigger than {exclusiveMinimum}")') + + def generate_exclusive_maximum(self): + with self.l('if isinstance({variable}, (int, float)):'): + with self.l('if {variable} >= {exclusiveMaximum}:'): + self.l('raise JsonSchemaException("{name} must be smaller than {exclusiveMaximum}")') + + def generate_property_names(self): + """ + Means that keys of object must to follow this definition. + + .. code-block:: python + + { + 'propertyNames': { + 'maxLength': 3, + }, + } + + Valid keys of object for this definition are foo, bar, ... but not foobar for example. + """ + property_names_definition = self._definition.get('propertyNames', {}) + if property_names_definition is True: + pass + elif property_names_definition is False: + self.create_variable_keys() + with self.l('if {variable}_keys:'): + self.l('raise JsonSchemaException("{name} must not be there")') + else: + self.create_variable_is_dict() + with self.l('if {variable}_is_dict:'): + self.create_variable_with_length() + with self.l('if {variable}_len != 0:'): + self.l('{variable}_property_names = True') + with self.l('for key in {variable}:'): + with self.l('try:'): + self.generate_func_code_block( + property_names_definition, + 'key', + self._variable_name, + clear_variables=True, + ) + with self.l('except JsonSchemaException:'): + self.l('{variable}_property_names = False') + with self.l('if not {variable}_property_names:'): + self.l('raise JsonSchemaException("{name} must be named by propertyName definition")') + + def generate_contains(self): + """ + Means that array must contain at least one defined item. + + .. code-block:: python + + { + 'contains': { + 'type': 'number', + }, + } + + Valid array is any with at least one number. + """ + self.create_variable_is_list() + with self.l('if {variable}_is_list:'): + contains_definition = self._definition['contains'] + + if contains_definition is False: + self.l('raise JsonSchemaException("{name} is always invalid")') + elif contains_definition is True: + with self.l('if not {variable}:'): + self.l('raise JsonSchemaException("{name} must not be empty")') + else: + self.l('{variable}_contains = False') + with self.l('for key in {variable}:'): + with self.l('try:'): + self.generate_func_code_block( + contains_definition, + 'key', + self._variable_name, + clear_variables=True, + ) + self.l('{variable}_contains = True') + self.l('break') + self.l('except JsonSchemaException: pass') + + with self.l('if not {variable}_contains:'): + self.l('raise JsonSchemaException("{name} must contain one of contains definition")') + + def generate_const(self): + """ + Means that value is valid when is equeal to const definition. + + .. code-block:: python + + { + 'const': 42, + } + + Only valid value is 42 in this example. + """ + with self.l('if {variable} != {}:', self._definition['const']): + self.l('raise JsonSchemaException("{name} must be same as const definition")') diff --git a/fastjsonschema/draft07.py b/fastjsonschema/draft07.py index c9f44b7..9db483e 100644 --- a/fastjsonschema/draft07.py +++ b/fastjsonschema/draft07.py @@ -2,4 +2,99 @@ class CodeGeneratorDraft07(CodeGeneratorDraft06): - pass + def __init__(self, definition, resolver=None): + super().__init__(definition, resolver) + self._json_keywords_to_function.update(( + ('if', self.generate_if_then_else), + ('contentEncoding', self.generate_content_encoding), + ('contentMediaType', self.generate_content_media_type), + )) + + def generate_if_then_else(self): + """ + Implementation of if-then-else. + + .. code-block:: python + + { + 'if': { + 'exclusiveMaximum': 0, + }, + 'then': { + 'minimum': -10, + }, + 'else': { + 'multipleOf': 2, + }, + } + + Valid values are any between -10 and 0 or any multiplication of two. + """ + with self.l('try:'): + self.generate_func_code_block( + self._definition['if'], + self._variable, + self._variable_name, + clear_variables=True + ) + with self.l('except JsonSchemaException:'): + if 'else' in self._definition: + self.generate_func_code_block( + self._definition['else'], + self._variable, + self._variable_name, + clear_variables=True + ) + else: + self.l('pass') + if 'then' in self._definition: + with self.l('else:'): + self.generate_func_code_block( + self._definition['then'], + self._variable, + self._variable_name, + clear_variables=True + ) + + def generate_content_encoding(self): + """ + Means decoding value when it's encoded by base64. + + .. code-block:: python + + { + 'contentEncoding': 'base64', + } + """ + if self._definition['contentEncoding'] == 'base64': + with self.l('if isinstance({variable}, str):'): + with self.l('try:'): + self.l('import base64') + self.l('{variable} = base64.b64decode({variable})') + with self.l('except Exception:'): + self.l('raise JsonSchemaException("{name} must be encoded by base64")') + with self.l('if {variable} == "":'): + self.l('raise JsonSchemaException("contentEncoding must be base64")') + + def generate_content_media_type(self): + """ + Means loading value when it's specified as JSON. + + .. code-block:: python + + { + 'contentMediaType': 'application/json', + } + """ + if self._definition['contentMediaType'] == 'application/json': + with self.l('if isinstance({variable}, bytes):'): + with self.l('try:'): + self.l('{variable} = {variable}.decode("utf-8")') + with self.l('except Exception:'): + self.l('raise JsonSchemaException("{name} must encoded by utf8")') + with self.l('if isinstance({variable}, str):'): + with self.l('try:'): + self.l('import json') + self.l('{variable} = json.loads({variable})') + with self.l('except Exception:'): + self.l('raise JsonSchemaException("{name} must be valid JSON")') diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 1407b91..221ff9e 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -1,3 +1,4 @@ +from collections import OrderedDict import re from .exceptions import JsonSchemaException @@ -60,13 +61,15 @@ def __init__(self, definition, resolver=None): # add main function to `self._needed_validation_functions` self._needed_validation_functions[self._resolver.get_uri()] = self._resolver.get_scope_name() - self.generate_func_code() + self._json_keywords_to_function = OrderedDict() @property def func_code(self): """ Returns generated code of whole validation function as string. """ + self._generate_func_code() + return '\n'.join(self._code) @property @@ -76,6 +79,8 @@ def global_state(self): compiled regular expressions and imports, so it does not have to do it every time when validation function is called. """ + self._generate_func_code() + return dict( REGEX_PATTERNS=self._compile_regexps, re=re, @@ -88,6 +93,8 @@ def global_state_code(self): Returns global variables for generating function from ``func_code`` as code. Includes compiled regular expressions and imports. """ + self._generate_func_code() + if self._compile_regexps: return '\n'.join( [ @@ -110,6 +117,84 @@ def global_state_code(self): ] ) + + def _generate_func_code(self): + if not self._code: + self.generate_func_code() + + def generate_func_code(self): + """ + Creates base code of validation function and calls helper + for creating code by definition. + """ + self.l('NoneType = type(None)') + # Generate parts that are referenced and not yet generated + while self._needed_validation_functions: + # During generation of validation function, could be needed to generate + # new one that is added again to `_needed_validation_functions`. + # Therefore usage of while instead of for loop. + uri, name = self._needed_validation_functions.popitem() + self.generate_validation_function(uri, name) + + def generate_validation_function(self, uri, name): + """ + Generate validation function for given uri with given name + """ + self._validation_functions_done.add(uri) + self.l('') + with self._resolver.resolving(uri) as definition: + with self.l('def {}(data):', name): + self.generate_func_code_block(definition, 'data', 'data', clear_variables=True) + self.l('return data') + + def generate_func_code_block(self, definition, variable, variable_name, clear_variables=False): + """ + Creates validation rules for current definition. + """ + backup = self._definition, self._variable, self._variable_name + self._definition, self._variable, self._variable_name = definition, variable, variable_name + if clear_variables: + backup_variables = self._variables + self._variables = set() + + if '$ref' in definition: + # needed because ref overrides any sibling keywords + self.generate_ref() + else: + self.run_generate_functions(definition) + + self._definition, self._variable, self._variable_name = backup + if clear_variables: + self._variables = backup_variables + + def run_generate_functions(self, definition): + for key, func in self._json_keywords_to_function.items(): + if key in definition: + func() + + def generate_ref(self): + """ + Ref can be link to remote or local definition. + + .. code-block:: python + + {'$ref': 'http://json-schema.org/draft-04/schema#'} + { + 'properties': { + 'foo': {'type': 'integer'}, + 'bar': {'$ref': '#/properties/foo'} + } + } + """ + with self._resolver.in_scope(self._definition['$ref']): + name = self._resolver.get_scope_name() + uri = self._resolver.get_uri() + if uri not in self._validation_functions_done: + self._needed_validation_functions[uri] = name + # call validation function + self.l('{}({variable})', name) + + # pylint: disable=invalid-name @indent def l(self, line, *args, **kwds): @@ -189,75 +274,3 @@ def create_variable_is_dict(self): return self._variables.add(variable_name) self.l('{variable}_is_dict = isinstance({variable}, dict)') - - def generate_func_code(self): - """ - Creates base code of validation function and calls helper - for creating code by definition. - """ - self.l('NoneType = type(None)') - # Generate parts that are referenced and not yet generated - while self._needed_validation_functions: - # During generation of validation function, could be needed to generate - # new one that is added again to `_needed_validation_functions`. - # Therefore usage of while instead of for loop. - uri, name = self._needed_validation_functions.popitem() - self.generate_validation_function(uri, name) - - def generate_validation_function(self, uri, name): - """ - Generate validation function for given uri with given name - """ - self._validation_functions_done.add(uri) - self.l('') - with self._resolver.resolving(uri) as definition: - with self.l('def {}(data):', name): - self.generate_func_code_block(definition, 'data', 'data', clear_variables=True) - self.l('return data') - - def generate_func_code_block(self, definition, variable, variable_name, clear_variables=False): - """ - Creates validation rules for current definition. - """ - backup = self._definition, self._variable, self._variable_name - self._definition, self._variable, self._variable_name = definition, variable, variable_name - if clear_variables: - backup_variables = self._variables - self._variables = set() - - if '$ref' in definition: - # needed because ref overrides any sibling keywords - self.generate_ref() - else: - self.run_generate_functions(definition) - - self._definition, self._variable, self._variable_name = backup - if clear_variables: - self._variables = backup_variables - - def run_generate_functions(self, definition): - for key, func in self._json_keywords_to_function.items(): - if key in definition: - func() - - def generate_ref(self): - """ - Ref can be link to remote or local definition. - - .. code-block:: python - - {'$ref': 'http://json-schema.org/draft-04/schema#'} - { - 'properties': { - 'foo': {'type': 'integer'}, - 'bar': {'$ref': '#/properties/foo'} - } - } - """ - with self._resolver.in_scope(self._definition['$ref']): - name = self._resolver.get_scope_name() - uri = self._resolver.get_uri() - if uri not in self._validation_functions_done: - self._needed_validation_functions[uri] = name - # call validation function - self.l('{}({variable})', name) diff --git a/tests/conftest.py b/tests/conftest.py index fb7f61d..68ad5c7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,18 +10,18 @@ import pytest from fastjsonschema import JsonSchemaException, compile -from fastjsonschema.draft07 import CodeGeneratorDraft07 +from fastjsonschema.draft04 import CodeGeneratorDraft04 @pytest.fixture def asserter(): def f(definition, value, expected): # When test fails, it will show up code. - code_generator = CodeGeneratorDraft07(definition) + code_generator = CodeGeneratorDraft04(definition) print(code_generator.func_code) pprint(code_generator.global_state) - validator = compile(definition) + validator = compile(definition, version=4) if isinstance(expected, JsonSchemaException): with pytest.raises(JsonSchemaException) as exc: validator(value) diff --git a/tests/json_schema/test_draft06.py b/tests/json_schema/test_draft06.py index d933587..6488119 100644 --- a/tests/json_schema/test_draft06.py +++ b/tests/json_schema/test_draft06.py @@ -8,12 +8,9 @@ def pytest_generate_tests(metafunc): version=6, suite_dir='JSON-Schema-Test-Suite/tests/draft6', ignored_suite_files=[ - 'bignum.json', 'ecmascript-regex.json', 'zeroTerminatedFloats.json', 'boolean_schema.json', - 'contains.json', - 'const.json', ], ignore_tests=[ 'invalid definition', @@ -22,8 +19,6 @@ def pytest_generate_tests(metafunc): 'remote ref, containing refs itself', 'dependencies with boolean subschemas', 'dependencies with empty array', - 'exclusiveMaximum validation', - 'exclusiveMinimum validation', 'format: uri-template', 'validation of URI References', 'items with boolean schema (true)', @@ -33,8 +28,6 @@ def pytest_generate_tests(metafunc): 'not with boolean schema true', 'not with boolean schema false', 'properties with boolean schema', - 'propertyNames with boolean schema false', - 'propertyNames validation', 'base URI change - change folder', 'base URI change - change folder in subschema', 'base URI change', diff --git a/tests/json_schema/test_draft07.py b/tests/json_schema/test_draft07.py index 120b639..f578d87 100644 --- a/tests/json_schema/test_draft07.py +++ b/tests/json_schema/test_draft07.py @@ -8,20 +8,15 @@ def pytest_generate_tests(metafunc): version=7, suite_dir='JSON-Schema-Test-Suite/tests/draft7', ignored_suite_files=[ - 'bignum.json', 'ecmascript-regex.json', 'zeroTerminatedFloats.json', 'boolean_schema.json', - 'contains.json', - 'content.json', - 'if-then-else.json', 'idn-email.json', 'idn-hostname.json', 'iri-reference.json', 'iri.json', 'relative-json-pointer.json', 'time.json', - 'const.json', ], ignore_tests=[ 'invalid definition', @@ -30,8 +25,6 @@ def pytest_generate_tests(metafunc): 'remote ref, containing refs itself', 'dependencies with boolean subschemas', 'dependencies with empty array', - 'exclusiveMaximum validation', - 'exclusiveMinimum validation', 'format: uri-template', 'validation of URI References', 'items with boolean schema (true)', @@ -41,8 +34,6 @@ def pytest_generate_tests(metafunc): 'not with boolean schema true', 'not with boolean schema false', 'properties with boolean schema', - 'propertyNames with boolean schema false', - 'propertyNames validation', 'base URI change - change folder', 'base URI change - change folder in subschema', 'base URI change', From 7d1aa131fed50d7a1b7fcaf56562862a91a424ea Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 14 Aug 2018 10:27:14 +0200 Subject: [PATCH 088/201] A float without fractional part is an integer --- fastjsonschema/draft04.py | 13 ++++++++++- fastjsonschema/draft06.py | 37 +++++++++++++++++++++++++++++++ fastjsonschema/generator.py | 10 --------- tests/json_schema/test_draft06.py | 1 - tests/json_schema/test_draft07.py | 1 - 5 files changed, 49 insertions(+), 13 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index cd75213..0b0a8df 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -3,6 +3,17 @@ from .generator import CodeGenerator, enforce_list +JSON_TYPE_TO_PYTHON_TYPE = { + 'null': 'NoneType', + 'boolean': 'bool', + 'number': 'int, float', + 'integer': 'int', + 'string': 'str', + 'array': 'list', + 'object': 'dict', +} + + # pylint: disable=line-too-long FORMAT_REGEXS = { 'date-time': r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+(?:[+-][0-2]\d:[0-5]\d|Z)?$', @@ -55,7 +66,7 @@ def generate_type(self): {'type': ['string', 'number']} """ types = enforce_list(self._definition['type']) - python_types = ', '.join(self.JSON_TYPE_TO_PYTHON_TYPE.get(t) for t in types) + python_types = ', '.join(JSON_TYPE_TO_PYTHON_TYPE.get(t) for t in types) extra = '' if ('number' in types or 'integer' in types) and 'boolean' not in types: diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py index 7118537..3a38862 100644 --- a/fastjsonschema/draft06.py +++ b/fastjsonschema/draft06.py @@ -1,4 +1,16 @@ from .draft04 import CodeGeneratorDraft04 +from .generator import enforce_list + + +JSON_TYPE_TO_PYTHON_TYPE = { + 'null': 'NoneType', + 'boolean': 'bool', + 'number': 'int, float', + 'integer': 'int', + 'string': 'str', + 'array': 'list', + 'object': 'dict', +} class CodeGeneratorDraft06(CodeGeneratorDraft04): @@ -12,6 +24,31 @@ def __init__(self, definition, resolver=None): ('const', self.generate_const), )) + def generate_type(self): + """ + Validation of type. Can be one type or list of types. + + Since draft 06 a float without fractional part is an integer. + + .. code-block:: python + + {'type': 'string'} + {'type': ['string', 'number']} + """ + types = enforce_list(self._definition['type']) + python_types = ', '.join(JSON_TYPE_TO_PYTHON_TYPE.get(t) for t in types) + + extra = '' + + if 'integer' in types: + extra += ' and not (isinstance({variable}, float) and {variable}.is_integer())'.format(variable=self._variable) + + if ('number' in types or 'integer' in types) and 'boolean' not in types: + extra += ' or isinstance({variable}, bool)'.format(variable=self._variable) + + with self.l('if not isinstance({variable}, ({})){}:', python_types, extra): + self.l('raise JsonSchemaException("{name} must be {}")', ' or '.join(types)) + def generate_exclusive_minimum(self): with self.l('if isinstance({variable}, (int, float)):'): with self.l('if {variable} <= {exclusiveMinimum}:'): diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 221ff9e..9ddd6b9 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -28,16 +28,6 @@ class CodeGenerator: INDENT = 4 # spaces - JSON_TYPE_TO_PYTHON_TYPE = { - 'null': 'NoneType', - 'boolean': 'bool', - 'number': 'int, float', - 'integer': 'int', - 'string': 'str', - 'array': 'list', - 'object': 'dict', - } - def __init__(self, definition, resolver=None): self._code = [] self._compile_regexps = {} diff --git a/tests/json_schema/test_draft06.py b/tests/json_schema/test_draft06.py index 6488119..0262b73 100644 --- a/tests/json_schema/test_draft06.py +++ b/tests/json_schema/test_draft06.py @@ -9,7 +9,6 @@ def pytest_generate_tests(metafunc): suite_dir='JSON-Schema-Test-Suite/tests/draft6', ignored_suite_files=[ 'ecmascript-regex.json', - 'zeroTerminatedFloats.json', 'boolean_schema.json', ], ignore_tests=[ diff --git a/tests/json_schema/test_draft07.py b/tests/json_schema/test_draft07.py index f578d87..92eb486 100644 --- a/tests/json_schema/test_draft07.py +++ b/tests/json_schema/test_draft07.py @@ -9,7 +9,6 @@ def pytest_generate_tests(metafunc): suite_dir='JSON-Schema-Test-Suite/tests/draft7', ignored_suite_files=[ 'ecmascript-regex.json', - 'zeroTerminatedFloats.json', 'boolean_schema.json', 'idn-email.json', 'idn-hostname.json', From e83cca3e51c9727fe97adfc9523f917918ad1e9c Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 14 Aug 2018 10:39:48 +0200 Subject: [PATCH 089/201] Boolean or empty array in dependencies --- fastjsonschema/draft04.py | 23 ++++++++++++++++++++++- tests/json_schema/test_draft06.py | 2 -- tests/json_schema/test_draft07.py | 2 -- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 0b0a8df..df8422d 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -387,12 +387,33 @@ def generate_additional_properties(self): self.l('raise JsonSchemaException("{name} must contain only specified properties")') def generate_dependencies(self): + """ + Means when object has property, it needs to have also other property. + + .. code-block:: python + + { + 'dependencies': { + 'bar': ['foo'], + }, + } + + Valid object is containing only foo, both bar and foo or none of them, but not + object with only bar. + + Since draft 06 definition can be boolean or empty array. True and empty array + means nothing, False means that key cannot be there at all. + """ self.create_variable_is_dict() with self.l('if {variable}_is_dict:'): self.create_variable_keys() for key, values in self._definition["dependencies"].items(): + if values == [] or values is True: + continue with self.l('if "{}" in {variable}_keys:', key): - if isinstance(values, list): + if values is False: + self.l('raise JsonSchemaException("{} in {name} must not be there")', key) + elif isinstance(values, list): for value in values: with self.l('if "{}" not in {variable}_keys:', value): self.l('raise JsonSchemaException("{name} missing dependency {} for {}")', value, key) diff --git a/tests/json_schema/test_draft06.py b/tests/json_schema/test_draft06.py index 0262b73..7711950 100644 --- a/tests/json_schema/test_draft06.py +++ b/tests/json_schema/test_draft06.py @@ -16,8 +16,6 @@ def pytest_generate_tests(metafunc): 'valid definition', 'Recursive references between schemas', 'remote ref, containing refs itself', - 'dependencies with boolean subschemas', - 'dependencies with empty array', 'format: uri-template', 'validation of URI References', 'items with boolean schema (true)', diff --git a/tests/json_schema/test_draft07.py b/tests/json_schema/test_draft07.py index 92eb486..ebdb8bf 100644 --- a/tests/json_schema/test_draft07.py +++ b/tests/json_schema/test_draft07.py @@ -22,8 +22,6 @@ def pytest_generate_tests(metafunc): 'valid definition', 'Recursive references between schemas', 'remote ref, containing refs itself', - 'dependencies with boolean subschemas', - 'dependencies with empty array', 'format: uri-template', 'validation of URI References', 'items with boolean schema (true)', From fc5467d79f6ef36bdd332c88649c9a5f4bd570d4 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 14 Aug 2018 10:50:48 +0200 Subject: [PATCH 090/201] Boolean in items --- fastjsonschema/draft04.py | 36 +++++++++++++++++++++++++------ tests/json_schema/test_draft06.py | 3 --- tests/json_schema/test_draft07.py | 3 --- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index df8422d..2a61b81 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -278,11 +278,35 @@ def generate_unique_items(self): self.l('raise JsonSchemaException("{name} must contain unique items")') def generate_items(self): + """ + Means array is valid only when all items are valid by this definition. + + .. code-block:: python + + { + 'items': [ + {'type': 'integer'}, + {'type': 'string'}, + ], + } + + Valid arrays are those with integers or strings, nothing else. + + Since draft 06 definition can be also boolean. True means nothing, False + means everything is invalid. + """ + items_definition = self._definition['items'] + if items_definition is True: + return + self.create_variable_is_list() with self.l('if {variable}_is_list:'): self.create_variable_with_length() - if isinstance(self._definition['items'], list): - for idx, item_definition in enumerate(self._definition['items']): + if items_definition is False: + with self.l('if {variable}:'): + self.l('raise JsonSchemaException("{name} must not be there")') + elif isinstance(items_definition, list): + for idx, item_definition in enumerate(items_definition): with self.l('if {variable}_len > {}:', idx): self.l('{variable}_{0} = {variable}[{0}]', idx) self.generate_func_code_block( @@ -295,19 +319,19 @@ def generate_items(self): if 'additionalItems' in self._definition: if self._definition['additionalItems'] is False: - self.l('if {variable}_len > {}: raise JsonSchemaException("{name} must contain only specified items")', len(self._definition['items'])) + self.l('if {variable}_len > {}: raise JsonSchemaException("{name} must contain only specified items")', len(items_definition)) else: - with self.l('for {variable}_x, {variable}_item in enumerate({variable}[{0}:], {0}):', len(self._definition['items'])): + with self.l('for {variable}_x, {variable}_item in enumerate({variable}[{0}:], {0}):', len(items_definition)): self.generate_func_code_block( self._definition['additionalItems'], '{}_item'.format(self._variable), '{}[{{{}_x}}]'.format(self._variable_name, self._variable), ) else: - if self._definition['items']: + if items_definition: with self.l('for {variable}_x, {variable}_item in enumerate({variable}):'): self.generate_func_code_block( - self._definition['items'], + items_definition, '{}_item'.format(self._variable), '{}[{{{}_x}}]'.format(self._variable_name, self._variable), ) diff --git a/tests/json_schema/test_draft06.py b/tests/json_schema/test_draft06.py index 7711950..71e1421 100644 --- a/tests/json_schema/test_draft06.py +++ b/tests/json_schema/test_draft06.py @@ -18,9 +18,6 @@ def pytest_generate_tests(metafunc): 'remote ref, containing refs itself', 'format: uri-template', 'validation of URI References', - 'items with boolean schema (true)', - 'items with boolean schema (false)', - 'items with boolean schema', 'items with boolean schemas', 'not with boolean schema true', 'not with boolean schema false', diff --git a/tests/json_schema/test_draft07.py b/tests/json_schema/test_draft07.py index ebdb8bf..8b73e70 100644 --- a/tests/json_schema/test_draft07.py +++ b/tests/json_schema/test_draft07.py @@ -24,9 +24,6 @@ def pytest_generate_tests(metafunc): 'remote ref, containing refs itself', 'format: uri-template', 'validation of URI References', - 'items with boolean schema (true)', - 'items with boolean schema (false)', - 'items with boolean schema', 'items with boolean schemas', 'not with boolean schema true', 'not with boolean schema false', From 696ffb13953e67c2ce8a0c722ce6bee8449ab208 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 14 Aug 2018 10:58:14 +0200 Subject: [PATCH 091/201] Boolean schemas --- fastjsonschema/draft06.py | 17 +++++++++++++++++ fastjsonschema/generator.py | 11 +++++++---- tests/json_schema/test_draft06.py | 15 --------------- tests/json_schema/test_draft07.py | 15 --------------- 4 files changed, 24 insertions(+), 34 deletions(-) diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py index 3a38862..b30c099 100644 --- a/fastjsonschema/draft06.py +++ b/fastjsonschema/draft06.py @@ -24,6 +24,23 @@ def __init__(self, definition, resolver=None): ('const', self.generate_const), )) + def _generate_func_code_block(self, definition): + if isinstance(definition, bool): + self.generate_boolean_schema() + elif '$ref' in definition: + # needed because ref overrides any sibling keywords + self.generate_ref() + else: + self.run_generate_functions(definition) + + def generate_boolean_schema(self): + """ + Means that schema can be specified by boolean. + True means everything is valid, False everything is invalid. + """ + if self._definition is False: + self.l('raise JsonSchemaException("{name} must not be there")') + def generate_type(self): """ Validation of type. Can be one type or list of types. diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 9ddd6b9..b0c73f5 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -147,16 +147,19 @@ def generate_func_code_block(self, definition, variable, variable_name, clear_va backup_variables = self._variables self._variables = set() + self._generate_func_code_block(definition) + + self._definition, self._variable, self._variable_name = backup + if clear_variables: + self._variables = backup_variables + + def _generate_func_code_block(self, definition): if '$ref' in definition: # needed because ref overrides any sibling keywords self.generate_ref() else: self.run_generate_functions(definition) - self._definition, self._variable, self._variable_name = backup - if clear_variables: - self._variables = backup_variables - def run_generate_functions(self, definition): for key, func in self._json_keywords_to_function.items(): if key in definition: diff --git a/tests/json_schema/test_draft06.py b/tests/json_schema/test_draft06.py index 71e1421..d5b6b93 100644 --- a/tests/json_schema/test_draft06.py +++ b/tests/json_schema/test_draft06.py @@ -26,22 +26,7 @@ def pytest_generate_tests(metafunc): 'base URI change - change folder in subschema', 'base URI change', 'root ref in remote ref', - 'allOf with boolean schemas, all true', - 'allOf with boolean schemas, some false', - 'allOf with boolean schemas, all false', - 'anyOf with boolean schemas, all true', - 'anyOf with boolean schemas, some false', - 'anyOf with boolean schemas, all false', - 'anyOf with boolean schemas, some true', - 'oneOf with boolean schemas, all true', - 'oneOf with boolean schemas, some false', - 'oneOf with boolean schemas, all false', - 'oneOf with boolean schemas, one true', - 'oneOf with boolean schemas, more than one true', 'validation of JSON-pointers (JSON String Representation)', - 'patternProperties with boolean schemas', - '$ref to boolean schema true', - '$ref to boolean schema false', ], ) metafunc.parametrize(['version', 'schema', 'data', 'is_valid'], param_values, ids=param_ids) diff --git a/tests/json_schema/test_draft07.py b/tests/json_schema/test_draft07.py index 8b73e70..3dd4692 100644 --- a/tests/json_schema/test_draft07.py +++ b/tests/json_schema/test_draft07.py @@ -33,22 +33,7 @@ def pytest_generate_tests(metafunc): 'base URI change', 'root ref in remote ref', 'validation of date strings', - 'allOf with boolean schemas, all true', - 'allOf with boolean schemas, some false', - 'allOf with boolean schemas, all false', - 'anyOf with boolean schemas, all true', - 'anyOf with boolean schemas, some false', - 'anyOf with boolean schemas, all false', - 'anyOf with boolean schemas, some true', - 'oneOf with boolean schemas, all true', - 'oneOf with boolean schemas, some false', - 'oneOf with boolean schemas, all false', - 'oneOf with boolean schemas, one true', - 'oneOf with boolean schemas, more than one true', 'validation of JSON-pointers (JSON String Representation)', - 'patternProperties with boolean schemas', - '$ref to boolean schema true', - '$ref to boolean schema false', ], ) metafunc.parametrize(['version', 'schema', 'data', 'is_valid'], param_values, ids=param_ids) From fcc6b62b9cb09d4695d9e00473aa13c9c04f0f64 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 14 Aug 2018 13:36:04 +0200 Subject: [PATCH 092/201] New formats --- Makefile | 2 +- fastjsonschema/draft04.py | 28 +++++++++++++++------------- fastjsonschema/draft06.py | 23 +++++++++++------------ fastjsonschema/draft07.py | 9 +++++++++ tests/json_schema/test_draft06.py | 1 - tests/json_schema/test_draft07.py | 4 ---- 6 files changed, 36 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index a74376e..15266cf 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ jsonschemasuitcases: git submodule update test: venv jsonschemasuitcases - ${PYTHON} -m pytest --benchmark-skip + ${PYTHON} -m pytest --benchmark-skip -v test-lf: venv jsonschemasuitcases ${PYTHON} -m pytest --benchmark-skip --last-failed diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 2a61b81..2282a10 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -14,19 +14,21 @@ } -# pylint: disable=line-too-long -FORMAT_REGEXS = { - 'date-time': r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+(?:[+-][0-2]\d:[0-5]\d|Z)?$', - 'uri': r'^\w+:(\/?\/?)[^\s]+$', - 'email': r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$', - 'ipv4': r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$', - 'ipv6': r'^(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|::(?:[0-9A-Fa-f]{1,4}:){5}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){4}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){3}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,2}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){2}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,3}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}:(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,4}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,5}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}|(?:(?:[0-9A-Fa-f]{1,4}:){,6}[0-9A-Fa-f]{1,4})?::)$', - 'hostname': r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{1,62}[A-Za-z0-9])$', -} - - # pylint: disable=too-many-instance-attributes,too-many-public-methods class CodeGeneratorDraft04(CodeGenerator): + # pylint: disable=line-too-long + FORMAT_REGEXS = { + 'date-time': r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+(?:[+-][0-2]\d:[0-5]\d|Z)?$', + 'email': r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$', + 'hostname': ( + r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*' + r'([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{1,62}[A-Za-z0-9])$' + ), + 'ipv4': r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$', + 'ipv6': r'^(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|::(?:[0-9A-Fa-f]{1,4}:){5}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){4}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){3}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,2}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){2}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,3}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}:(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,4}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,5}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}|(?:(?:[0-9A-Fa-f]{1,4}:){,6}[0-9A-Fa-f]{1,4})?::)$', + 'uri': r'^\w+:(\/?\/?)[^\s]+$', + } + def __init__(self, definition, resolver=None): super().__init__(definition, resolver) self._json_keywords_to_function.update(( @@ -203,8 +205,8 @@ def generate_pattern(self): def generate_format(self): with self.l('if isinstance({variable}, str):'): format_ = self._definition['format'] - if format_ in FORMAT_REGEXS: - format_regex = FORMAT_REGEXS[format_] + if format_ in self.FORMAT_REGEXS: + format_regex = self.FORMAT_REGEXS[format_] self._generate_format(format_, format_ + '_re_pattern', format_regex) # format regex is used only in meta schemas elif format_ == 'regex': diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py index b30c099..ece3356 100644 --- a/fastjsonschema/draft06.py +++ b/fastjsonschema/draft06.py @@ -1,19 +1,18 @@ -from .draft04 import CodeGeneratorDraft04 +from .draft04 import CodeGeneratorDraft04, JSON_TYPE_TO_PYTHON_TYPE from .generator import enforce_list -JSON_TYPE_TO_PYTHON_TYPE = { - 'null': 'NoneType', - 'boolean': 'bool', - 'number': 'int, float', - 'integer': 'int', - 'string': 'str', - 'array': 'list', - 'object': 'dict', -} - - class CodeGeneratorDraft06(CodeGeneratorDraft04): + FORMAT_REGEXS = dict(CodeGeneratorDraft04.FORMAT_REGEXS, **{ + 'relative-json-pointer': r'^(?:0|[1-9][0-9]*)(?:#|(?:\/(?:[^~/]|~0|~1)*)*)$', + 'uri-template': ( + r'^(?:(?:[^\x00-\x20\"\'<>%\\^`{|}]|%[0-9a-f]{2})|' + r'\{[+#./;?&=,!@|]?(?:[a-z0-9_]|%[0-9a-f]{2})+' + r'(?::[1-9][0-9]{0,3}|\*)?(?:,(?:[a-z0-9_]|%[0-9a-f]{2})+' + r'(?::[1-9][0-9]{0,3}|\*)?)*\})*$' + ), + }) + def __init__(self, definition, resolver=None): super().__init__(definition, resolver) self._json_keywords_to_function.update(( diff --git a/fastjsonschema/draft07.py b/fastjsonschema/draft07.py index 9db483e..3df2066 100644 --- a/fastjsonschema/draft07.py +++ b/fastjsonschema/draft07.py @@ -2,6 +2,15 @@ class CodeGeneratorDraft07(CodeGeneratorDraft06): + FORMAT_REGEXS = dict(CodeGeneratorDraft06.FORMAT_REGEXS, **{ + 'date': r'^(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})$', + 'time': ( + r'^(?P\d{1,2}):(?P\d{1,2})' + r'(?::(?P\d{1,2})(?:\.(?P\d{1,6}))?' + r'([zZ]|[+-]\d\d:\d\d)?)?$' + ), + }) + def __init__(self, definition, resolver=None): super().__init__(definition, resolver) self._json_keywords_to_function.update(( diff --git a/tests/json_schema/test_draft06.py b/tests/json_schema/test_draft06.py index d5b6b93..618f8b8 100644 --- a/tests/json_schema/test_draft06.py +++ b/tests/json_schema/test_draft06.py @@ -16,7 +16,6 @@ def pytest_generate_tests(metafunc): 'valid definition', 'Recursive references between schemas', 'remote ref, containing refs itself', - 'format: uri-template', 'validation of URI References', 'items with boolean schemas', 'not with boolean schema true', diff --git a/tests/json_schema/test_draft07.py b/tests/json_schema/test_draft07.py index 3dd4692..ae2b26c 100644 --- a/tests/json_schema/test_draft07.py +++ b/tests/json_schema/test_draft07.py @@ -14,15 +14,12 @@ def pytest_generate_tests(metafunc): 'idn-hostname.json', 'iri-reference.json', 'iri.json', - 'relative-json-pointer.json', - 'time.json', ], ignore_tests=[ 'invalid definition', 'valid definition', 'Recursive references between schemas', 'remote ref, containing refs itself', - 'format: uri-template', 'validation of URI References', 'items with boolean schemas', 'not with boolean schema true', @@ -32,7 +29,6 @@ def pytest_generate_tests(metafunc): 'base URI change - change folder in subschema', 'base URI change', 'root ref in remote ref', - 'validation of date strings', 'validation of JSON-pointers (JSON String Representation)', ], ) From dd20246cf74567cb7d16adb22385b2c3c5c46009 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 14 Aug 2018 14:00:17 +0200 Subject: [PATCH 093/201] Finished support of boolean schemas --- Makefile | 2 +- fastjsonschema/draft04.py | 16 ++++++++++++---- fastjsonschema/ref_resolver.py | 11 +++++++++-- tests/json_schema/test_draft06.py | 5 ----- tests/json_schema/test_draft07.py | 5 ----- 5 files changed, 22 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 15266cf..a74376e 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ jsonschemasuitcases: git submodule update test: venv jsonschemasuitcases - ${PYTHON} -m pytest --benchmark-skip -v + ${PYTHON} -m pytest --benchmark-skip test-lf: venv jsonschemasuitcases ${PYTHON} -m pytest --benchmark-skip --last-failed diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 2282a10..e34280c 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -174,13 +174,21 @@ def generate_not(self): {'not': {'type': 'null'}} Valid values for this definitions are 'hello', 42, {} ... but not None. + + Since draft 06 definition can be boolean. False means nothing, True + means everything is invalid. """ - if not self._definition['not']: + not_definition = self._definition['not'] + if not_definition is True: + self.l('raise JsonSchemaException("{name} must not be there")') + elif not_definition is False: + return + elif not not_definition: with self.l('if {}:', self._variable): self.l('raise JsonSchemaException("{name} must not be valid by not definition")') else: with self.l('try:'): - self.generate_func_code_block(self._definition['not'], self._variable, self._variable_name) + self.generate_func_code_block(not_definition, self._variable, self._variable_name) self.l('except JsonSchemaException: pass') self.l('else: raise JsonSchemaException("{name} must not be valid by not definition")') @@ -316,7 +324,7 @@ def generate_items(self): '{}_{}'.format(self._variable, idx), '{}[{}]'.format(self._variable_name, idx), ) - if 'default' in item_definition: + if isinstance(item_definition, dict) and 'default' in item_definition: self.l('else: {variable}.append({})', repr(item_definition['default'])) if 'additionalItems' in self._definition: @@ -373,7 +381,7 @@ def generate_properties(self): '{}_{}'.format(self._variable, key_name), '{}.{}'.format(self._variable_name, key), ) - if 'default' in prop_definition: + if isinstance(prop_definition, dict) and 'default' in prop_definition: self.l('else: {variable}["{}"] = {}', key, repr(prop_definition['default'])) def generate_pattern_properties(self): diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index 8bd1ead..60a1bda 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -106,7 +106,12 @@ def from_schema(cls, schema, handlers={}, **kwargs): :argument schema schema: the referring schema :rtype: :class:`RefResolver` """ - return cls(schema.get('id', ''), schema, handlers=handlers, **kwargs) + return cls( + schema.get('id', '') if isinstance(schema, dict) else '', + schema, + handlers=handlers, + **kwargs + ) @contextlib.contextmanager def in_scope(self, scope): @@ -158,7 +163,9 @@ def walk(self, node: dict): """ Walk thru schema and dereferencing ``id`` and ``$ref`` instances """ - if '$ref' in node and isinstance(node['$ref'], str): + if isinstance(node, bool): + pass + elif '$ref' in node and isinstance(node['$ref'], str): ref = node['$ref'] node['$ref'] = urlparse.urljoin(self.resolution_scope, ref) elif 'id' in node and isinstance(node['id'], str): diff --git a/tests/json_schema/test_draft06.py b/tests/json_schema/test_draft06.py index 618f8b8..984ce0d 100644 --- a/tests/json_schema/test_draft06.py +++ b/tests/json_schema/test_draft06.py @@ -9,7 +9,6 @@ def pytest_generate_tests(metafunc): suite_dir='JSON-Schema-Test-Suite/tests/draft6', ignored_suite_files=[ 'ecmascript-regex.json', - 'boolean_schema.json', ], ignore_tests=[ 'invalid definition', @@ -17,10 +16,6 @@ def pytest_generate_tests(metafunc): 'Recursive references between schemas', 'remote ref, containing refs itself', 'validation of URI References', - 'items with boolean schemas', - 'not with boolean schema true', - 'not with boolean schema false', - 'properties with boolean schema', 'base URI change - change folder', 'base URI change - change folder in subschema', 'base URI change', diff --git a/tests/json_schema/test_draft07.py b/tests/json_schema/test_draft07.py index ae2b26c..6e47996 100644 --- a/tests/json_schema/test_draft07.py +++ b/tests/json_schema/test_draft07.py @@ -9,7 +9,6 @@ def pytest_generate_tests(metafunc): suite_dir='JSON-Schema-Test-Suite/tests/draft7', ignored_suite_files=[ 'ecmascript-regex.json', - 'boolean_schema.json', 'idn-email.json', 'idn-hostname.json', 'iri-reference.json', @@ -21,10 +20,6 @@ def pytest_generate_tests(metafunc): 'Recursive references between schemas', 'remote ref, containing refs itself', 'validation of URI References', - 'items with boolean schemas', - 'not with boolean schema true', - 'not with boolean schema false', - 'properties with boolean schema', 'base URI change - change folder', 'base URI change - change folder in subschema', 'base URI change', From 5a469b7aa13a82814c3004ecae89062d03604f3a Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 14 Aug 2018 14:38:10 +0200 Subject: [PATCH 094/201] Validation of json pointer format --- fastjsonschema/draft06.py | 3 ++- fastjsonschema/draft07.py | 6 ++++++ tests/json_schema/test_draft06.py | 1 - tests/json_schema/test_draft07.py | 1 - 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py index ece3356..b776148 100644 --- a/fastjsonschema/draft06.py +++ b/fastjsonschema/draft06.py @@ -4,7 +4,8 @@ class CodeGeneratorDraft06(CodeGeneratorDraft04): FORMAT_REGEXS = dict(CodeGeneratorDraft04.FORMAT_REGEXS, **{ - 'relative-json-pointer': r'^(?:0|[1-9][0-9]*)(?:#|(?:\/(?:[^~/]|~0|~1)*)*)$', + 'json-pointer': r'^(/(([^/~])|(~[01]))*)*$', + #'uri-reference': r'', 'uri-template': ( r'^(?:(?:[^\x00-\x20\"\'<>%\\^`{|}]|%[0-9a-f]{2})|' r'\{[+#./;?&=,!@|]?(?:[a-z0-9_]|%[0-9a-f]{2})+' diff --git a/fastjsonschema/draft07.py b/fastjsonschema/draft07.py index 3df2066..7c82a75 100644 --- a/fastjsonschema/draft07.py +++ b/fastjsonschema/draft07.py @@ -4,6 +4,12 @@ class CodeGeneratorDraft07(CodeGeneratorDraft06): FORMAT_REGEXS = dict(CodeGeneratorDraft06.FORMAT_REGEXS, **{ 'date': r'^(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})$', + #'iri': r'', + #'iri-reference': r'', + #'idn-email': r'', + #'idn-hostname': r'', + 'relative-json-pointer': r'^(?:0|[1-9][0-9]*)(?:#|(?:\/(?:[^~/]|~0|~1)*)*)$', + #'regex': r'', 'time': ( r'^(?P\d{1,2}):(?P\d{1,2})' r'(?::(?P\d{1,2})(?:\.(?P\d{1,6}))?' diff --git a/tests/json_schema/test_draft06.py b/tests/json_schema/test_draft06.py index 984ce0d..22744b0 100644 --- a/tests/json_schema/test_draft06.py +++ b/tests/json_schema/test_draft06.py @@ -20,7 +20,6 @@ def pytest_generate_tests(metafunc): 'base URI change - change folder in subschema', 'base URI change', 'root ref in remote ref', - 'validation of JSON-pointers (JSON String Representation)', ], ) metafunc.parametrize(['version', 'schema', 'data', 'is_valid'], param_values, ids=param_ids) diff --git a/tests/json_schema/test_draft07.py b/tests/json_schema/test_draft07.py index 6e47996..18367a1 100644 --- a/tests/json_schema/test_draft07.py +++ b/tests/json_schema/test_draft07.py @@ -24,7 +24,6 @@ def pytest_generate_tests(metafunc): 'base URI change - change folder in subschema', 'base URI change', 'root ref in remote ref', - 'validation of JSON-pointers (JSON String Representation)', ], ) metafunc.parametrize(['version', 'schema', 'data', 'is_valid'], param_values, ids=param_ids) From 133cd8b338d629daf63575455434b92b68ef25a7 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 14 Aug 2018 14:42:41 +0200 Subject: [PATCH 095/201] Comment which tests are optional --- tests/json_schema/test_draft04.py | 1 + tests/json_schema/test_draft06.py | 1 + tests/json_schema/test_draft07.py | 1 + 3 files changed, 3 insertions(+) diff --git a/tests/json_schema/test_draft04.py b/tests/json_schema/test_draft04.py index 5c00861..35af321 100644 --- a/tests/json_schema/test_draft04.py +++ b/tests/json_schema/test_draft04.py @@ -8,6 +8,7 @@ def pytest_generate_tests(metafunc): version=4, suite_dir='JSON-Schema-Test-Suite/tests/draft4', ignored_suite_files=[ + # Optional. 'ecmascript-regex.json', ], ignore_tests=[], diff --git a/tests/json_schema/test_draft06.py b/tests/json_schema/test_draft06.py index 22744b0..58eca5d 100644 --- a/tests/json_schema/test_draft06.py +++ b/tests/json_schema/test_draft06.py @@ -8,6 +8,7 @@ def pytest_generate_tests(metafunc): version=6, suite_dir='JSON-Schema-Test-Suite/tests/draft6', ignored_suite_files=[ + # Optional. 'ecmascript-regex.json', ], ignore_tests=[ diff --git a/tests/json_schema/test_draft07.py b/tests/json_schema/test_draft07.py index 18367a1..9edddc2 100644 --- a/tests/json_schema/test_draft07.py +++ b/tests/json_schema/test_draft07.py @@ -8,6 +8,7 @@ def pytest_generate_tests(metafunc): version=7, suite_dir='JSON-Schema-Test-Suite/tests/draft7', ignored_suite_files=[ + # Optional. 'ecmascript-regex.json', 'idn-email.json', 'idn-hostname.json', From c8ac899de1753b8f42ec2d1e161a1c6bb8385749 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 14 Aug 2018 14:46:11 +0200 Subject: [PATCH 096/201] Lint --- fastjsonschema/draft06.py | 4 +++- fastjsonschema/draft07.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py index b776148..c998535 100644 --- a/fastjsonschema/draft06.py +++ b/fastjsonschema/draft06.py @@ -58,7 +58,9 @@ def generate_type(self): extra = '' if 'integer' in types: - extra += ' and not (isinstance({variable}, float) and {variable}.is_integer())'.format(variable=self._variable) + extra += ' and not (isinstance({variable}, float) and {variable}.is_integer())'.format( + variable=self._variable, + ) if ('number' in types or 'integer' in types) and 'boolean' not in types: extra += ' or isinstance({variable}, bool)'.format(variable=self._variable) diff --git a/fastjsonschema/draft07.py b/fastjsonschema/draft07.py index 7c82a75..e7458aa 100644 --- a/fastjsonschema/draft07.py +++ b/fastjsonschema/draft07.py @@ -19,6 +19,7 @@ class CodeGeneratorDraft07(CodeGeneratorDraft06): def __init__(self, definition, resolver=None): super().__init__(definition, resolver) + # pylint: disable=duplicate-code self._json_keywords_to_function.update(( ('if', self.generate_if_then_else), ('contentEncoding', self.generate_content_encoding), From dd651fcf47351c1ae7045fef1117bcbd32ace238 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Sat, 25 Aug 2018 15:36:48 +0000 Subject: [PATCH 097/201] Doc strings --- fastjsonschema/draft04.py | 54 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index e34280c..1fd8caa 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -151,7 +151,7 @@ def generate_one_of(self): ], } - Valid values for this definitions are 3, 5, 6, ... but not 15 for example. + Valid values for this definition are 3, 5, 6, ... but not 15 for example. """ self.l('{variable}_one_of_count = 0') for definition_item in self._definition['oneOf']: @@ -173,7 +173,7 @@ def generate_not(self): {'not': {'type': 'null'}} - Valid values for this definitions are 'hello', 42, {} ... but not None. + Valid values for this definition are 'hello', 42, {} ... but not None. Since draft 06 definition can be boolean. False means nothing, True means everything is invalid. @@ -211,6 +211,15 @@ def generate_pattern(self): self.l('raise JsonSchemaException("{name} must match pattern {pattern}")') def generate_format(self): + """ + Means that value have to be in specified format. For example date, email or other. + + .. code-block:: python + + {'format': 'email'} + + Valid value for this definition is user@example.com but not @username + """ with self.l('if isinstance({variable}, str):'): format_ = self._definition['format'] if format_ in self.FORMAT_REGEXS: @@ -368,6 +377,19 @@ def generate_required(self): self.l('raise JsonSchemaException("{name} must contain {required} properties")') def generate_properties(self): + """ + Means object with defined keys. + + .. code-block:: python + + { + 'properties': { + 'key': {'type': 'number'}, + }, + } + + Valid object is containing key called 'key' and value any number. + """ self.create_variable_is_dict() with self.l('if {variable}_is_dict:'): self.create_variable_keys() @@ -385,6 +407,19 @@ def generate_properties(self): self.l('else: {variable}["{}"] = {}', key, repr(prop_definition['default'])) def generate_pattern_properties(self): + """ + Means object with defined keys as patterns. + + .. code-block:: python + + { + 'patternProperties': { + '^x': {'type': 'number'}, + }, + } + + Valid object is containing key starting with a 'x' and value any number. + """ self.create_variable_is_dict() with self.l('if {variable}_is_dict:'): self.create_variable_keys() @@ -402,6 +437,21 @@ def generate_pattern_properties(self): ) def generate_additional_properties(self): + """ + Means object with keys with values defined by definition. + + .. code-block:: python + + { + 'properties': { + 'key': {'type': 'number'}, + } + 'additionalProperties': {'type': 'string'}, + } + + Valid object is containing key called 'key' and it's value any number and + any other key with any string. + """ self.create_variable_is_dict() with self.l('if {variable}_is_dict:'): self.create_variable_keys() From 7129c47ce1c0d4be52779d6667a2ab5599a83cb6 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Sat, 25 Aug 2018 15:50:46 +0000 Subject: [PATCH 098/201] Fix generating missing formats --- fastjsonschema/__init__.py | 4 ++-- fastjsonschema/draft04.py | 2 ++ tests/json_schema/test_draft06.py | 3 --- tests/json_schema/test_draft07.py | 3 --- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 71b83da..fc05b0b 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -121,8 +121,8 @@ def compile_to_code(definition, version=7, handlers={}): .. code-block:: bash - echo "{'type': 'string'}" | pytohn3 -m fastjsonschema > your_file.py - pytohn3 -m fastjsonschema "{'type': 'string'}" > your_file.py + echo "{'type': 'string'}" | python3 -m fastjsonschema > your_file.py + python3 -m fastjsonschema "{'type': 'string'}" > your_file.py Exception :any:`JsonSchemaException` is thrown when validation fails. """ diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 1fd8caa..3ee257b 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -231,6 +231,8 @@ def generate_format(self): self.l('re.compile({variable})') with self.l('except Exception:'): self.l('raise JsonSchemaException("{name} must be a valid regex")') + else: + self.l('pass') def _generate_format(self, format_name, regexp_name, regexp): if self._definition['format'] == format_name: diff --git a/tests/json_schema/test_draft06.py b/tests/json_schema/test_draft06.py index 58eca5d..88e772c 100644 --- a/tests/json_schema/test_draft06.py +++ b/tests/json_schema/test_draft06.py @@ -12,10 +12,7 @@ def pytest_generate_tests(metafunc): 'ecmascript-regex.json', ], ignore_tests=[ - 'invalid definition', - 'valid definition', 'Recursive references between schemas', - 'remote ref, containing refs itself', 'validation of URI References', 'base URI change - change folder', 'base URI change - change folder in subschema', diff --git a/tests/json_schema/test_draft07.py b/tests/json_schema/test_draft07.py index 9edddc2..123f7ac 100644 --- a/tests/json_schema/test_draft07.py +++ b/tests/json_schema/test_draft07.py @@ -16,10 +16,7 @@ def pytest_generate_tests(metafunc): 'iri.json', ], ignore_tests=[ - 'invalid definition', - 'valid definition', 'Recursive references between schemas', - 'remote ref, containing refs itself', 'validation of URI References', 'base URI change - change folder', 'base URI change - change folder in subschema', From 69aa83505f357d7917be4e78b64b457551f38856 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Fri, 7 Sep 2018 09:59:12 +0200 Subject: [PATCH 099/201] Support of in ref resolver --- fastjsonschema/draft06.py | 2 +- fastjsonschema/draft07.py | 6 ++--- fastjsonschema/ref_resolver.py | 38 ++++++++++--------------------- tests/json_schema/test_draft06.py | 8 ------- tests/json_schema/test_draft07.py | 11 --------- tests/json_schema/utils.py | 2 +- 6 files changed, 17 insertions(+), 50 deletions(-) diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py index c998535..13f85d6 100644 --- a/fastjsonschema/draft06.py +++ b/fastjsonschema/draft06.py @@ -5,7 +5,7 @@ class CodeGeneratorDraft06(CodeGeneratorDraft04): FORMAT_REGEXS = dict(CodeGeneratorDraft04.FORMAT_REGEXS, **{ 'json-pointer': r'^(/(([^/~])|(~[01]))*)*$', - #'uri-reference': r'', + 'uri-reference': r'^(\w+:(\/?\/?))?[^#\\\s]*(#[^\\\s]*)?$', 'uri-template': ( r'^(?:(?:[^\x00-\x20\"\'<>%\\^`{|}]|%[0-9a-f]{2})|' r'\{[+#./;?&=,!@|]?(?:[a-z0-9_]|%[0-9a-f]{2})+' diff --git a/fastjsonschema/draft07.py b/fastjsonschema/draft07.py index e7458aa..44d576a 100644 --- a/fastjsonschema/draft07.py +++ b/fastjsonschema/draft07.py @@ -4,9 +4,9 @@ class CodeGeneratorDraft07(CodeGeneratorDraft06): FORMAT_REGEXS = dict(CodeGeneratorDraft06.FORMAT_REGEXS, **{ 'date': r'^(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})$', - #'iri': r'', - #'iri-reference': r'', - #'idn-email': r'', + 'iri': r'^\w+:(\/?\/?)[^\s]+$', + 'iri-reference': r'^(\w+:(\/?\/?))?[^#\\\s]*(#[^\\\s]*)?$', + 'idn-email': r'^\w+@\w+\.\w+$', #'idn-hostname': r'', 'relative-json-pointer': r'^(?:0|[1-9][0-9]*)(?:#|(?:\/(?:[^~/]|~0|~1)*)*)$', #'regex': r'', diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index 60a1bda..fb3d663 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -23,11 +23,6 @@ def resolve_path(schema, fragment): Return definition from path. Path is unescaped according https://tools.ietf.org/html/rfc6901 - - :argument schema: the referrant schema document - :argument str fragment: a URI fragment to resolve within it - :returns: the retrieved schema definition - """ fragment = fragment.lstrip('/') parts = unquote(fragment).split('/') if fragment else [] @@ -59,11 +54,6 @@ def resolve_remote(uri, handlers): For unknown schemes urlib is used with UTF-8 encoding. .. _Requests: http://pypi.python.org/pypi/requests/ - - :argument str uri: the URI to resolve - :argument dict handlers: the URI resolver functions for each scheme - :returns: the retrieved schema document - """ scheme = urlparse.urlsplit(uri).scheme if scheme in handlers: @@ -78,18 +68,13 @@ def resolve_remote(uri, handlers): class RefResolver: """ Resolve JSON References. - - :argument str base_uri: URI of the referring document - :argument schema: the actual referring schema document - :argument dict store: a mapping from URIs to documents to cache - :argument bool cache: whether remote refs should be cached after - first resolution - :argument dict handlers: a mapping from URI schemes to functions that - should be used to retrieve them """ # pylint: disable=dangerous-default-value,too-many-arguments def __init__(self, base_uri, schema, store={}, cache=True, handlers={}): + """ + `base_uri` is URI of the referring document from the `schema`. + """ self.base_uri = base_uri self.resolution_scope = base_uri self.schema = schema @@ -102,19 +87,19 @@ def __init__(self, base_uri, schema, store={}, cache=True, handlers={}): def from_schema(cls, schema, handlers={}, **kwargs): """ Construct a resolver from a JSON schema object. - - :argument schema schema: the referring schema - :rtype: :class:`RefResolver` """ return cls( - schema.get('id', '') if isinstance(schema, dict) else '', + schema.get('$id', schema.get('id', '')) if isinstance(schema, dict) else '', schema, handlers=handlers, **kwargs ) @contextlib.contextmanager - def in_scope(self, scope): + def in_scope(self, scope: str): + """ + Context manager to handle current scope. + """ old_scope = self.resolution_scope self.resolution_scope = urlparse.urljoin(old_scope, scope) try: @@ -123,12 +108,10 @@ def in_scope(self, scope): self.resolution_scope = old_scope @contextlib.contextmanager - def resolving(self, ref): + def resolving(self, ref: str): """ Context manager which resolves a JSON ``ref`` and enters the resolution scope of this ref. - - :argument str ref: reference to resolve """ new_uri = urlparse.urljoin(self.resolution_scope, ref) uri, fragment = urlparse.urldefrag(new_uri) @@ -154,6 +137,9 @@ def get_uri(self): return normalize(self.resolution_scope) def get_scope_name(self): + """ + Get current scope and return it as a valid function name. + """ name = 'validate_' + unquote(self.resolution_scope).replace('~1', '_').replace('~0', '_') name = re.sub(r'[:/#\.\-\%]', '_', name) name = name.lower().rstrip('_') diff --git a/tests/json_schema/test_draft06.py b/tests/json_schema/test_draft06.py index 88e772c..70c21c0 100644 --- a/tests/json_schema/test_draft06.py +++ b/tests/json_schema/test_draft06.py @@ -11,14 +11,6 @@ def pytest_generate_tests(metafunc): # Optional. 'ecmascript-regex.json', ], - ignore_tests=[ - 'Recursive references between schemas', - 'validation of URI References', - 'base URI change - change folder', - 'base URI change - change folder in subschema', - 'base URI change', - 'root ref in remote ref', - ], ) metafunc.parametrize(['version', 'schema', 'data', 'is_valid'], param_values, ids=param_ids) diff --git a/tests/json_schema/test_draft07.py b/tests/json_schema/test_draft07.py index 123f7ac..70904b6 100644 --- a/tests/json_schema/test_draft07.py +++ b/tests/json_schema/test_draft07.py @@ -10,18 +10,7 @@ def pytest_generate_tests(metafunc): ignored_suite_files=[ # Optional. 'ecmascript-regex.json', - 'idn-email.json', 'idn-hostname.json', - 'iri-reference.json', - 'iri.json', - ], - ignore_tests=[ - 'Recursive references between schemas', - 'validation of URI References', - 'base URI change - change folder', - 'base URI change - change folder in subschema', - 'base URI change', - 'root ref in remote ref', ], ) metafunc.parametrize(['version', 'schema', 'data', 'is_valid'], param_values, ids=param_ids) diff --git a/tests/json_schema/utils.py b/tests/json_schema/utils.py index c1375e2..7af3d1a 100644 --- a/tests/json_schema/utils.py +++ b/tests/json_schema/utils.py @@ -29,7 +29,7 @@ def remotes_handler(uri): return requests.get(uri).json() -def resolve_param_values_and_ids(version, suite_dir, ignored_suite_files, ignore_tests): +def resolve_param_values_and_ids(version, suite_dir, ignored_suite_files=[], ignore_tests=[]): suite_dir_path = Path(suite_dir).resolve() test_file_paths = sorted(set(suite_dir_path.glob("**/*.json"))) From 1f756ee93be7e790658633a201437853260a6cbd Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Fri, 7 Sep 2018 10:42:20 +0200 Subject: [PATCH 100/201] Detect version from definition --- Makefile | 2 +- docs/conf.py | 2 +- fastjsonschema/__init__.py | 85 +++++++++++++++++-------------- fastjsonschema/version.py | 2 +- performance.py | 1 + tests/conftest.py | 5 +- tests/json_schema/test_draft04.py | 5 +- tests/json_schema/test_draft06.py | 4 +- tests/json_schema/test_draft07.py | 4 +- tests/json_schema/utils.py | 15 ++++-- 10 files changed, 71 insertions(+), 54 deletions(-) diff --git a/Makefile b/Makefile index a74376e..9994bce 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all venv lint test test-lf benchmark benchmark-save performance doc upload deb clean +.PHONY: all venv lint jsonschemasuitcases test test-lf benchmark benchmark-save performance printcode doc upload deb clean SHELL=/bin/bash VENV_NAME?=venv diff --git a/docs/conf.py b/docs/conf.py index 8a8f693..a83394f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -23,7 +23,7 @@ # General information about the project. project = u'fastjsonschema' -copyright = u'2016-{}, Seznam.cz'.format(time.strftime("%Y")) +copyright = u'2016-{}, Michal Horejsek'.format(time.strftime("%Y")) # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index fc05b0b..434f944 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -23,40 +23,39 @@ .. code-block:: bash $ make performance - fast_compiled valid ==> 0.026877017982769758 - fast_compiled invalid ==> 0.0015628149849362671 - fast_file valid ==> 0.025493122986517847 - fast_file invalid ==> 0.0012430319911800325 - fast_not_compiled valid ==> 4.790547857992351 - fast_not_compiled invalid ==> 1.2642899919883348 - jsonschema valid ==> 5.036152001994196 - jsonschema invalid ==> 1.1929481109953485 - jsonspec valid ==> 7.196442283981014 - jsonspec invalid ==> 1.7245555499684997 - validictory valid ==> 0.36818933801259845 - validictory invalid ==> 0.022672351042274386 - -This library follows and implements `JSON schema draft-04 `_. Sometimes -it's not perfectly clear so I recommend also check out this `understaning json schema -`_. + fast_compiled valid ==> 0.030474655970465392 + fast_compiled invalid ==> 0.0017561429995112121 + fast_file valid ==> 0.028758891974575818 + fast_file invalid ==> 0.0017655809642747045 + fast_not_compiled valid ==> 4.597834145999514 + fast_not_compiled invalid ==> 1.139162228035275 + jsonschema valid ==> 5.014410221017897 + jsonschema invalid ==> 1.1362981660058722 + jsonspec valid ==> 8.1144932230236 + jsonspec invalid ==> 2.0143173419637606 + validictory valid ==> 0.4084212710149586 + validictory invalid ==> 0.026061681972350925 + +This library follows and implements `JSON schema draft-04, draft-06 and draft-07 +`_. Sometimes it's not perfectly clear so I recommend also +check out this `understaning json schema `_. Note that there are some differences compared to JSON schema standard: * Regular expressions are full Python ones, not only what JSON schema allows. It's easier to allow everything and also it's faster to compile without limits. So keep in mind that when - you will use more advanced regular expression, it may not work with other library. + you will use more advanced regular expression, it may not work with other library or in + other language. * JSON schema says you can use keyword ``default`` for providing default values. This implementation uses that and always returns transformed input data. Support only for Python 3.3 and higher. """ -from os.path import exists - -from .exceptions import JsonSchemaException from .draft04 import CodeGeneratorDraft04 from .draft06 import CodeGeneratorDraft06 from .draft07 import CodeGeneratorDraft07 +from .exceptions import JsonSchemaException from .ref_resolver import RefResolver from .version import VERSION @@ -64,7 +63,7 @@ # pylint: disable=redefined-builtin,dangerous-default-value,exec-used -def compile(definition, version=7, handlers={}): +def compile(definition, handlers={}): """ Generates validation function for validating JSON schema by ``definition``. Example: @@ -89,14 +88,23 @@ def compile(definition, version=7, handlers={}): data = validate({}) assert data == {'a': 42} - Args: - definition (dict): Json schema definition - handlers (dict): A mapping from URI schemes to functions - that should be used to retrieve them. + Supported implementations are draft-04, draft-06 and draft-07. Which version + should be used is determined by `$draft` in your ``definition``. When not + specified, the latest implementation is used (draft-07). + + .. code-block:: python + + validate = fastjsonschema.compile({ + '$schema': 'http://json-schema.org/draft-04/schema', + 'type': 'number', + }) + + You can pass mapping from URI to function that should be used to retrieve + remote schemes used in your ``definition`` in parameter ``handlers``. Exception :any:`JsonSchemaException` is thrown when validation fails. """ - resolver, code_generator = _factory(definition, version, handlers) + resolver, code_generator = _factory(definition, handlers) global_state = code_generator.global_state # Do not pass local state so it can recursively call itself. exec(code_generator.func_code, global_state) @@ -104,7 +112,7 @@ def compile(definition, version=7, handlers={}): # pylint: disable=dangerous-default-value -def compile_to_code(definition, version=7, handlers={}): +def compile_to_code(definition, handlers={}): """ Generates validation function for validating JSON schema by ``definition`` and returns compiled code. Example: @@ -126,7 +134,7 @@ def compile_to_code(definition, version=7, handlers={}): Exception :any:`JsonSchemaException` is thrown when validation fails. """ - _, code_generator = _factory(definition, version, handlers) + _, code_generator = _factory(definition, handlers) return ( 'VERSION = "' + VERSION + '"\n' + code_generator.global_state_code + '\n' + @@ -134,17 +142,18 @@ def compile_to_code(definition, version=7, handlers={}): ) -def _factory(definition, version, handlers): +def _factory(definition, handlers): resolver = RefResolver.from_schema(definition, handlers=handlers) - code_generator = _get_code_generator_class(version)(definition, resolver=resolver) + code_generator = _get_code_generator_class(definition)(definition, resolver=resolver) return resolver, code_generator -def _get_code_generator_class(version): - if version == 4: - return CodeGeneratorDraft04 - if version == 6: - return CodeGeneratorDraft06 - if version == 7: - return CodeGeneratorDraft07 - raise JsonSchemaException('Unsupported JSON schema version. Supported are 4, 6 and 7.') +def _get_code_generator_class(schema): + # Schema in from draft-06 can be just the boolean value. + if isinstance(schema, dict): + schema_version = schema.get('$schema', '') + if 'draft-04' in schema_version: + return CodeGeneratorDraft04 + if 'draft-06' in schema_version: + return CodeGeneratorDraft06 + return CodeGeneratorDraft07 diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index 1dae207..aadc97c 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '1.6' +VERSION = '2.0dev' diff --git a/performance.py b/performance.py index 6d77ede..8669acd 100644 --- a/performance.py +++ b/performance.py @@ -11,6 +11,7 @@ NUMBER = 1000 JSON_SCHEMA = { + '$schema': 'http://json-schema.org/draft-04/schema#', 'type': 'array', 'items': [ { diff --git a/tests/conftest.py b/tests/conftest.py index 68ad5c7..aa745bb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,7 +21,10 @@ def f(definition, value, expected): print(code_generator.func_code) pprint(code_generator.global_state) - validator = compile(definition, version=4) + # By default old tests are written for draft-04. + definition.setdefault('$schema', 'http://json-schema.org/draft-04/schema') + + validator = compile(definition) if isinstance(expected, JsonSchemaException): with pytest.raises(JsonSchemaException) as exc: validator(value) diff --git a/tests/json_schema/test_draft04.py b/tests/json_schema/test_draft04.py index 35af321..7b980b8 100644 --- a/tests/json_schema/test_draft04.py +++ b/tests/json_schema/test_draft04.py @@ -5,15 +5,14 @@ def pytest_generate_tests(metafunc): param_values, param_ids = resolve_param_values_and_ids( - version=4, + schema_version='http://json-schema.org/draft-04/schema', suite_dir='JSON-Schema-Test-Suite/tests/draft4', ignored_suite_files=[ # Optional. 'ecmascript-regex.json', ], - ignore_tests=[], ) - metafunc.parametrize(['version', 'schema', 'data', 'is_valid'], param_values, ids=param_ids) + metafunc.parametrize(['schema_version', 'schema', 'data', 'is_valid'], param_values, ids=param_ids) # Real test function to be used with parametrization by previous hook function. diff --git a/tests/json_schema/test_draft06.py b/tests/json_schema/test_draft06.py index 70c21c0..a62d75d 100644 --- a/tests/json_schema/test_draft06.py +++ b/tests/json_schema/test_draft06.py @@ -5,14 +5,14 @@ def pytest_generate_tests(metafunc): param_values, param_ids = resolve_param_values_and_ids( - version=6, + schema_version='http://json-schema.org/draft-06/schema', suite_dir='JSON-Schema-Test-Suite/tests/draft6', ignored_suite_files=[ # Optional. 'ecmascript-regex.json', ], ) - metafunc.parametrize(['version', 'schema', 'data', 'is_valid'], param_values, ids=param_ids) + metafunc.parametrize(['schema_version', 'schema', 'data', 'is_valid'], param_values, ids=param_ids) # Real test function to be used with parametrization by previous hook function. diff --git a/tests/json_schema/test_draft07.py b/tests/json_schema/test_draft07.py index 70904b6..b720caf 100644 --- a/tests/json_schema/test_draft07.py +++ b/tests/json_schema/test_draft07.py @@ -5,7 +5,7 @@ def pytest_generate_tests(metafunc): param_values, param_ids = resolve_param_values_and_ids( - version=7, + schema_version='http://json-schema.org/draft-07/schema', suite_dir='JSON-Schema-Test-Suite/tests/draft7', ignored_suite_files=[ # Optional. @@ -13,7 +13,7 @@ def pytest_generate_tests(metafunc): 'idn-hostname.json', ], ) - metafunc.parametrize(['version', 'schema', 'data', 'is_valid'], param_values, ids=param_ids) + metafunc.parametrize(['schema_version', 'schema', 'data', 'is_valid'], param_values, ids=param_ids) # Real test function to be used with parametrization by previous hook function. diff --git a/tests/json_schema/utils.py b/tests/json_schema/utils.py index 7af3d1a..6c1b686 100644 --- a/tests/json_schema/utils.py +++ b/tests/json_schema/utils.py @@ -29,7 +29,7 @@ def remotes_handler(uri): return requests.get(uri).json() -def resolve_param_values_and_ids(version, suite_dir, ignored_suite_files=[], ignore_tests=[]): +def resolve_param_values_and_ids(schema_version, suite_dir, ignored_suite_files=[], ignore_tests=[]): suite_dir_path = Path(suite_dir).resolve() test_file_paths = sorted(set(suite_dir_path.glob("**/*.json"))) @@ -41,7 +41,7 @@ def resolve_param_values_and_ids(version, suite_dir, ignored_suite_files=[], ign for test_case in test_cases: for test_data in test_case['tests']: param_values.append(pytest.param( - version, + schema_version, test_case['schema'], test_data['data'], test_data['valid'], @@ -58,16 +58,21 @@ def resolve_param_values_and_ids(version, suite_dir, ignored_suite_files=[], ign return param_values, param_ids -def template_test(version, schema, data, is_valid): +def template_test(schema_version, schema, data, is_valid): """ Test function to be used (imported) in final test file to run the tests which are generated by `pytest_generate_tests` hook. """ # For debug purposes. When test fails, it will print stdout. resolver = RefResolver.from_schema(schema, handlers={'http': remotes_handler}) - print(_get_code_generator_class(version)(schema, resolver=resolver).func_code) + print(_get_code_generator_class(schema_version)(schema, resolver=resolver).func_code) - validate = compile(schema, version=version, handlers={'http': remotes_handler}) + # JSON schema test suits do not contain schema version. + # Our library needs to know that or it would use always the latest implementation. + if isinstance(schema, dict): + schema.setdefault('$schema', schema_version) + + validate = compile(schema, handlers={'http': remotes_handler}) try: result = validate(data) print('Validate result:', result) From 831d48437d4bc06854d725d2ec5d9b98a6f5ee6d Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Fri, 7 Sep 2018 10:45:54 +0200 Subject: [PATCH 101/201] Version 2.0 --- README.rst | 20 ++++++++++---------- fastjsonschema/version.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index d6e0515..5f77591 100644 --- a/README.rst +++ b/README.rst @@ -17,16 +17,16 @@ least an order of magnitude faster than other Python implemantaions. See `documentation `_ for performance test details. -Current version is implementation of `json-schema `_ draft-04. -Note that there are some differences compared to JSON schema standard: +This library follows and implements `JSON schema draft-04, draft-06 and draft-07 +`_. Note that there are some differences compared to JSON +schema standard: -* Regular expressions are full Python ones, not only what JSON schema - allows. It's easier to allow everything and also it's faster to - compile without limits. So keep in mind that when you will use more - advanced regular expression, it may not work with other library. -* JSON schema says you can use keyword ``default`` for providing default - values. This implementation uses that and always returns transformed - input data. + * Regular expressions are full Python ones, not only what JSON schema allows. It's easier + to allow everything and also it's faster to compile without limits. So keep in mind that when + you will use more advanced regular expression, it may not work with other library or in + other language. + * JSON schema says you can use keyword ``default`` for providing default values. This implementation + uses that and always returns transformed input data. Install ------- @@ -35,7 +35,7 @@ Install pip install fastjsonschema -Support for Python 3.3 and higher. +Support only for Python 3.3 and higher. Documentation ------------- diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index aadc97c..db7986c 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.0dev' +VERSION = '2.0' From 88a39415794964539ae76deaf0b6004acefef807 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Fri, 7 Sep 2018 10:55:31 +0200 Subject: [PATCH 102/201] Changelog --- CHANGELOG.txt | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 CHANGELOG.txt diff --git a/CHANGELOG.txt b/CHANGELOG.txt new file mode 100644 index 0000000..5fd5531 --- /dev/null +++ b/CHANGELOG.txt @@ -0,0 +1,43 @@ +=== 2.0 (2018-09-07) === + +* Support of draft-06 +* Support of draft-07 +* Code generation to a file + + +=== 1.6 (2018-06-21) === + +* Bugfixing + + +=== 1.5 (2018-06-20) === + +* Support of definitions +* Support of referencies + + +=== 1.4 (2018-06-11) === + +* Better date-time regex +* Support of dependencies + + +=== 1.3 (2018-04-25) === + +* Fix patter inside of anyOf + + +=== 1.2 (2018-04-24) === + +* Support of formats +* Support of properties + + +=== 1.1 (2017-01-03) === + +* Support of float numbers + + +=== 1.0 (2016-09-23) === + +* First version From 58cba3784555f06e6cb27e18b210b2defce391d4 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 12 Sep 2018 12:43:02 +0200 Subject: [PATCH 103/201] #21 Fix code generation with regex patterns --- CHANGELOG.txt | 5 +++++ fastjsonschema/generator.py | 2 +- fastjsonschema/version.py | 2 +- tests/test_compile_to_code.py | 11 ++++++++++- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 5fd5531..0c5369e 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +=== 2.1 (2018-09-12) === + +* Fix code generation with regex patterns + + === 2.0 (2018-09-07) === * Support of draft-06 diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index b0c73f5..11358d2 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -85,7 +85,7 @@ def global_state_code(self): """ self._generate_func_code() - if self._compile_regexps: + if not self._compile_regexps: return '\n'.join( [ 'from fastjsonschema import JsonSchemaException', diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index db7986c..8e4933f 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.0' +VERSION = '2.1' diff --git a/tests/test_compile_to_code.py b/tests/test_compile_to_code.py index 617e812..394e648 100644 --- a/tests/test_compile_to_code.py +++ b/tests/test_compile_to_code.py @@ -9,6 +9,7 @@ def test_compile_to_code(): 'properties': { 'a': {'type': 'string'}, 'b': {'type': 'integer'}, + 'c': {'format': 'hostname'}, } }) if not os.path.isdir('temp'): @@ -16,4 +17,12 @@ def test_compile_to_code(): with open('temp/schema.py', 'w') as f: f.write(code) from temp.schema import validate - assert validate({'a': 'a', 'b': 1}) == {'a': 'a', 'b': 1} + assert validate({ + 'a': 'a', + 'b': 1, + 'c': 'example.com', + }) == { + 'a': 'a', + 'b': 1, + 'c': 'example.com', + } From dc3ae94332ce901181120db8a2f9f8b6d5339622 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 12 Sep 2018 12:46:58 +0200 Subject: [PATCH 104/201] Fixed link in README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5f77591..0ee9e58 100644 --- a/README.rst +++ b/README.rst @@ -40,4 +40,4 @@ Support only for Python 3.3 and higher. Documentation ------------- -Documentation: `https://horejsek.github.io/python-fastjsonschema`_ +Documentation: `https://horejsek.github.io/python-fastjsonschema `_ From e9033693637b78e88327df6691f5004fb7e48224 Mon Sep 17 00:00:00 2001 From: Frederik Petersen Date: Wed, 12 Sep 2018 14:02:09 +0200 Subject: [PATCH 105/201] Added failing test case for #23 --- tests/test_datetime.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/test_datetime.py diff --git a/tests/test_datetime.py b/tests/test_datetime.py new file mode 100644 index 0000000..2263e9f --- /dev/null +++ b/tests/test_datetime.py @@ -0,0 +1,15 @@ + +import pytest + +from fastjsonschema import JsonSchemaException + + +exc = JsonSchemaException('data must be date-time') +@pytest.mark.parametrize('value, expected', [ + ('', exc), + ('bla', exc), + ('2018-02-05T14:17:10.00Z', '2018-02-05T14:17:10.00Z'), + ('2018-02-05T14:17:10Z', '2018-02-05T14:17:10Z'), +]) +def test_datetime(asserter, value, expected): + asserter({'type': 'string', 'format': 'date-time'}, value, expected) From 193838a5822f9db591e6ac5d9705de3a42848599 Mon Sep 17 00:00:00 2001 From: Frederik Petersen Date: Wed, 12 Sep 2018 14:04:11 +0200 Subject: [PATCH 106/201] Allow date-time strings without milliseconds. Fixes #23 --- fastjsonschema/draft04.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 3ee257b..6d79a1e 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -18,7 +18,7 @@ class CodeGeneratorDraft04(CodeGenerator): # pylint: disable=line-too-long FORMAT_REGEXS = { - 'date-time': r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+(?:[+-][0-2]\d:[0-5]\d|Z)?$', + 'date-time': r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d(?:\.\d+)?(?:[+-][0-2]\d:[0-5]\d|Z)?$', 'email': r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$', 'hostname': ( r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*' From 08f29dce2cf601178eec7978226ba54c38147244 Mon Sep 17 00:00:00 2001 From: Frederik Petersen Date: Wed, 12 Sep 2018 14:25:21 +0200 Subject: [PATCH 107/201] Added failing test case for #22 --- tests/test_compile_to_code.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/tests/test_compile_to_code.py b/tests/test_compile_to_code.py index 394e648..7312f38 100644 --- a/tests/test_compile_to_code.py +++ b/tests/test_compile_to_code.py @@ -1,8 +1,19 @@ import os import pytest +import shutil from fastjsonschema import JsonSchemaException, compile_to_code +@pytest.yield_fixture(autouse=True) +def run_around_tests(): + temp_dir = 'temp' + # Code that will run before your test, for example: + if not os.path.isdir(temp_dir): + os.makedirs(temp_dir) + # A test function will be run at this point + yield + # Code that will run after your test, for example: + shutil.rmtree(temp_dir) def test_compile_to_code(): code = compile_to_code({ @@ -12,11 +23,9 @@ def test_compile_to_code(): 'c': {'format': 'hostname'}, } }) - if not os.path.isdir('temp'): - os.makedirs('temp') - with open('temp/schema.py', 'w') as f: + with open('temp/schema_1.py', 'w') as f: f.write(code) - from temp.schema import validate + from temp.schema_1 import validate assert validate({ 'a': 'a', 'b': 1, @@ -26,3 +35,18 @@ def test_compile_to_code(): 'b': 1, 'c': 'example.com', } + +def test_compile_to_code_ipv6_regex(): + code = compile_to_code({ + 'properties': { + 'ip': {'format': 'ipv6'}, + } + }) + with open('temp/schema_2.py', 'w') as f: + f.write(code) + from temp.schema_2 import validate + assert validate({ + 'ip': '2001:0db8:85a3:0000:0000:8a2e:0370:7334' + }) == { + 'ip': '2001:0db8:85a3:0000:0000:8a2e:0370:7334' + } \ No newline at end of file From 8ff317179ed60aaab36a5a5d1f28ebfd7c9f6672 Mon Sep 17 00:00:00 2001 From: Frederik Petersen Date: Wed, 12 Sep 2018 14:36:25 +0200 Subject: [PATCH 108/201] Fixed #22 --- fastjsonschema/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 11358d2..28b9480 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -93,7 +93,7 @@ def global_state_code(self): '', ] ) - regexs = ['"{}": {}'.format(key, value) for key, value in self._compile_regexps.items()] + regexs = ['"{}": re.compile(r"{}")'.format(key, value.pattern) for key, value in self._compile_regexps.items()] return '\n'.join( [ 'import re', From 85cb8ab080d579bfafea10cb3fadd3e73f5cddbf Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 12 Sep 2018 14:41:02 +0200 Subject: [PATCH 109/201] Comment --- tests/test_compile_to_code.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_compile_to_code.py b/tests/test_compile_to_code.py index 7312f38..079f358 100644 --- a/tests/test_compile_to_code.py +++ b/tests/test_compile_to_code.py @@ -15,12 +15,13 @@ def run_around_tests(): # Code that will run after your test, for example: shutil.rmtree(temp_dir) + def test_compile_to_code(): code = compile_to_code({ 'properties': { 'a': {'type': 'string'}, 'b': {'type': 'integer'}, - 'c': {'format': 'hostname'}, + 'c': {'format': 'hostname'}, # Test generation of regex patterns to the file. } }) with open('temp/schema_1.py', 'w') as f: From a6f7bf077dde54aba03b413770b7716d032a3475 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 12 Sep 2018 14:42:00 +0200 Subject: [PATCH 110/201] Updated AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index f0d8503..0cf8116 100644 --- a/AUTHORS +++ b/AUTHORS @@ -4,5 +4,6 @@ Michal Hořejšek CONTRIBUTORS anentropic Antti Jokipii +Frederik Petersen Guillaume Desvé Kris Molendyke From 6c07d2300c5018628dab9487bd5a89b85acb3084 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 12 Sep 2018 14:43:33 +0200 Subject: [PATCH 111/201] Version 2.2 --- CHANGELOG.txt | 6 ++++++ fastjsonschema/version.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 0c5369e..b7f02ff 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,9 @@ +=== 2.2 (2018-09-12) === + +* Fix code generation with long regex patterns +* Fix regex of date-time (allow time without miliseconds) + + === 2.1 (2018-09-12) === * Fix code generation with regex patterns diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index 8e4933f..99233b7 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.1' +VERSION = '2.2' From ad5d89be86d2c43b865f3d64d2de8f795057f6a3 Mon Sep 17 00:00:00 2001 From: Frederik Petersen Date: Wed, 12 Sep 2018 17:06:47 +0200 Subject: [PATCH 112/201] Created failing test case for #26 --- tests/test_datetime.py | 1 - tests/test_hostname.py | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 tests/test_hostname.py diff --git a/tests/test_datetime.py b/tests/test_datetime.py index 2263e9f..b9e460b 100644 --- a/tests/test_datetime.py +++ b/tests/test_datetime.py @@ -1,4 +1,3 @@ - import pytest from fastjsonschema import JsonSchemaException diff --git a/tests/test_hostname.py b/tests/test_hostname.py new file mode 100644 index 0000000..19e3b5b --- /dev/null +++ b/tests/test_hostname.py @@ -0,0 +1,14 @@ +import pytest + +from fastjsonschema import JsonSchemaException + + +exc = JsonSchemaException('data must be hostname') +@pytest.mark.parametrize('value, expected', [ + ('', exc), + ('localhost', 'localhost'), + ('example.com', 'example.com'), + ('example.de', 'example.de'), +]) +def test_datetime(asserter, value, expected): + asserter({'type': 'string', 'format': 'hostname'}, value, expected) From aca2aaa2a3f81f6e2517fac1df81d280065c896d Mon Sep 17 00:00:00 2001 From: Frederik Petersen Date: Wed, 12 Sep 2018 17:24:16 +0200 Subject: [PATCH 113/201] Switched to different hostname regex as this one did not allow for 2 char tlds. Fixes #26 --- fastjsonschema/draft04.py | 5 +---- tests/test_hostname.py | 5 +++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 6d79a1e..949e0d8 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -20,10 +20,7 @@ class CodeGeneratorDraft04(CodeGenerator): FORMAT_REGEXS = { 'date-time': r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d(?:\.\d+)?(?:[+-][0-2]\d:[0-5]\d|Z)?$', 'email': r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$', - 'hostname': ( - r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*' - r'([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{1,62}[A-Za-z0-9])$' - ), + 'hostname': r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$', 'ipv4': r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$', 'ipv6': r'^(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|::(?:[0-9A-Fa-f]{1,4}:){5}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){4}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){3}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,2}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){2}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,3}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}:(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,4}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,5}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}|(?:(?:[0-9A-Fa-f]{1,4}:){,6}[0-9A-Fa-f]{1,4})?::)$', 'uri': r'^\w+:(\/?\/?)[^\s]+$', diff --git a/tests/test_hostname.py b/tests/test_hostname.py index 19e3b5b..f78a250 100644 --- a/tests/test_hostname.py +++ b/tests/test_hostname.py @@ -6,9 +6,14 @@ exc = JsonSchemaException('data must be hostname') @pytest.mark.parametrize('value, expected', [ ('', exc), + ('LDhsjf878&d', exc), + ('bla.bla-', exc), + ('example.example.com-', exc), ('localhost', 'localhost'), ('example.com', 'example.com'), ('example.de', 'example.de'), + ('example.fr', 'example.fr'), + ('example.example.com', 'example.example.com'), ]) def test_datetime(asserter, value, expected): asserter({'type': 'string', 'format': 'hostname'}, value, expected) From eb346dc11610bf57cf8826b41662de2d212f155c Mon Sep 17 00:00:00 2001 From: Frederik Petersen Date: Fri, 14 Sep 2018 09:48:29 +0200 Subject: [PATCH 114/201] Added length limit to hostname labels in regex and fixed test name method --- fastjsonschema/draft04.py | 2 +- tests/test_hostname.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 949e0d8..57b2a85 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -20,7 +20,7 @@ class CodeGeneratorDraft04(CodeGenerator): FORMAT_REGEXS = { 'date-time': r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d(?:\.\d+)?(?:[+-][0-2]\d:[0-5]\d|Z)?$', 'email': r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$', - 'hostname': r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$', + 'hostname': r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{0,61}[A-Za-z0-9])$', 'ipv4': r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$', 'ipv6': r'^(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|::(?:[0-9A-Fa-f]{1,4}:){5}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){4}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){3}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,2}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){2}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,3}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}:(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,4}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,5}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}|(?:(?:[0-9A-Fa-f]{1,4}:){,6}[0-9A-Fa-f]{1,4})?::)$', 'uri': r'^\w+:(\/?\/?)[^\s]+$', diff --git a/tests/test_hostname.py b/tests/test_hostname.py index f78a250..8620ed1 100644 --- a/tests/test_hostname.py +++ b/tests/test_hostname.py @@ -15,5 +15,5 @@ ('example.fr', 'example.fr'), ('example.example.com', 'example.example.com'), ]) -def test_datetime(asserter, value, expected): +def test_hostname(asserter, value, expected): asserter({'type': 'string', 'format': 'hostname'}, value, expected) From b6c4850b9b54fd8fd4cfbcd37222c2396cec81b3 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Fri, 14 Sep 2018 15:01:39 +0000 Subject: [PATCH 115/201] Merged format tests into one file --- tests/test_datetime.py | 14 -------------- tests/{test_hostname.py => test_format.py} | 11 +++++++++++ 2 files changed, 11 insertions(+), 14 deletions(-) delete mode 100644 tests/test_datetime.py rename tests/{test_hostname.py => test_format.py} (60%) diff --git a/tests/test_datetime.py b/tests/test_datetime.py deleted file mode 100644 index b9e460b..0000000 --- a/tests/test_datetime.py +++ /dev/null @@ -1,14 +0,0 @@ -import pytest - -from fastjsonschema import JsonSchemaException - - -exc = JsonSchemaException('data must be date-time') -@pytest.mark.parametrize('value, expected', [ - ('', exc), - ('bla', exc), - ('2018-02-05T14:17:10.00Z', '2018-02-05T14:17:10.00Z'), - ('2018-02-05T14:17:10Z', '2018-02-05T14:17:10Z'), -]) -def test_datetime(asserter, value, expected): - asserter({'type': 'string', 'format': 'date-time'}, value, expected) diff --git a/tests/test_hostname.py b/tests/test_format.py similarity index 60% rename from tests/test_hostname.py rename to tests/test_format.py index 8620ed1..7302694 100644 --- a/tests/test_hostname.py +++ b/tests/test_format.py @@ -3,6 +3,17 @@ from fastjsonschema import JsonSchemaException +exc = JsonSchemaException('data must be date-time') +@pytest.mark.parametrize('value, expected', [ + ('', exc), + ('bla', exc), + ('2018-02-05T14:17:10.00Z', '2018-02-05T14:17:10.00Z'), + ('2018-02-05T14:17:10Z', '2018-02-05T14:17:10Z'), +]) +def test_datetime(asserter, value, expected): + asserter({'type': 'string', 'format': 'date-time'}, value, expected) + + exc = JsonSchemaException('data must be hostname') @pytest.mark.parametrize('value, expected', [ ('', exc), From 3eb5b3a3beedc7d2f731871d48526644e4c1beb8 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Fri, 14 Sep 2018 15:01:57 +0000 Subject: [PATCH 116/201] Version 2.3 --- CHANGELOG.txt | 5 +++++ fastjsonschema/version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index b7f02ff..292ada9 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +=== 2.3 (2018-09-14) === + +* Fix regex of hostname + + === 2.2 (2018-09-12) === * Fix code generation with long regex patterns diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index 99233b7..f4d1515 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.2' +VERSION = '2.3' From c07c95c63cc4add3634ea3589ec95176a10d0325 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 20 Sep 2018 13:04:53 +0200 Subject: [PATCH 117/201] Exception message --- fastjsonschema/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastjsonschema/exceptions.py b/fastjsonschema/exceptions.py index ac12512..48f7370 100644 --- a/fastjsonschema/exceptions.py +++ b/fastjsonschema/exceptions.py @@ -5,5 +5,5 @@ class JsonSchemaException(ValueError): """ def __init__(self, message): - super().__init__() + super().__init__(message) self.message = message From 90e34b65cdc7ae664d54d0f70c2280b7d268f36b Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 26 Sep 2018 14:52:55 +0200 Subject: [PATCH 118/201] #28 Fix overriding variables in pattern properties --- fastjsonschema/draft04.py | 12 ++++++------ tests/test_pattern_properties.py | 26 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 tests/test_pattern_properties.py diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 57b2a85..dea1299 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -424,15 +424,15 @@ def generate_pattern_properties(self): self.create_variable_keys() for pattern, definition in self._definition['patternProperties'].items(): self._compile_regexps['{}'.format(pattern)] = re.compile(pattern) - with self.l('for key, val in {variable}.items():'): + with self.l('for {variable}_key, {variable}_val in {variable}.items():'): for pattern, definition in self._definition['patternProperties'].items(): - with self.l('if REGEX_PATTERNS["{}"].search(key):', pattern): - with self.l('if key in {variable}_keys:'): - self.l('{variable}_keys.remove(key)') + with self.l('if REGEX_PATTERNS["{}"].search({variable}_key):', pattern): + with self.l('if {variable}_key in {variable}_keys:'): + self.l('{variable}_keys.remove({variable}_key)') self.generate_func_code_block( definition, - 'val', - '{}.{{key}}'.format(self._variable_name), + '{}_val'.format(self._variable), + '{}.{{{}_key}}'.format(self._variable_name, self._variable), ) def generate_additional_properties(self): diff --git a/tests/test_pattern_properties.py b/tests/test_pattern_properties.py new file mode 100644 index 0000000..826b190 --- /dev/null +++ b/tests/test_pattern_properties.py @@ -0,0 +1,26 @@ +def test_dont_override_variable_names(asserter): + value = { + 'foo:bar': { + 'baz': { + 'bat': {}, + }, + 'bit': {}, + }, + } + asserter({ + 'type': 'object', + 'patternProperties': { + '^foo:': { + 'type': 'object', + 'properties': { + 'baz': { + 'type': 'object', + 'patternProperties': { + '^b': {'type': 'object'}, + }, + }, + 'bit': {'type': 'object'}, + }, + }, + }, + }, value, value) From e18a0d8147054c95b35d191fa21138c0d72bdea3 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 26 Sep 2018 15:03:10 +0200 Subject: [PATCH 119/201] Fix of overriding variables in general --- CHANGELOG.txt | 5 +++++ fastjsonschema/draft04.py | 2 +- fastjsonschema/draft06.py | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 292ada9..34182f4 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +=== 2.4 (unreleased) + +* Fix overriding variables (in pattern properties, property names, unique items and contains) + + === 2.3 (2018-09-14) === * Fix regex of hostname diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index dea1299..3a33305 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -292,7 +292,7 @@ def generate_unique_items(self): 2.1439831256866455 """ self.create_variable_with_length() - with self.l('if {variable}_len > len(set(str(x) for x in {variable})):'): + with self.l('if {variable}_len > len(set(str({variable}_x) for {variable}_x in {variable})):'): self.l('raise JsonSchemaException("{name} must contain unique items")') def generate_items(self): diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py index 13f85d6..f22a46c 100644 --- a/fastjsonschema/draft06.py +++ b/fastjsonschema/draft06.py @@ -105,11 +105,11 @@ def generate_property_names(self): self.create_variable_with_length() with self.l('if {variable}_len != 0:'): self.l('{variable}_property_names = True') - with self.l('for key in {variable}:'): + with self.l('for {variable}_key in {variable}:'): with self.l('try:'): self.generate_func_code_block( property_names_definition, - 'key', + '{}_key'.format(self._variable), self._variable_name, clear_variables=True, ) @@ -143,11 +143,11 @@ def generate_contains(self): self.l('raise JsonSchemaException("{name} must not be empty")') else: self.l('{variable}_contains = False') - with self.l('for key in {variable}:'): + with self.l('for {variable}_key in {variable}:'): with self.l('try:'): self.generate_func_code_block( contains_definition, - 'key', + '{}_key'.format(self._variable), self._variable_name, clear_variables=True, ) From acec3cb4ce34908b77ba1f0701ff13988ddd86a0 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 26 Sep 2018 15:10:59 +0200 Subject: [PATCH 120/201] Improved documentation --- fastjsonschema/__init__.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 434f944..63b34cd 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -7,18 +7,18 @@ """ This project was made to come up with fast JSON validations. Just let's see some numbers first: - * Probalby most popular ``jsonschema`` can take in tests up to 5 seconds for valid inputs + * Probably most popular ``jsonschema`` can take up to 5 seconds for valid inputs and 1.2 seconds for invalid inputs. - * Secondly most popular ``json-spec`` is even worse with up to 7.2 and 1.7 seconds. - * Lastly ``validictory`` is much better with 370 or 23 miliseconds, but it does not + * Second most popular ``json-spec`` is even worse with up to 7.2 and 1.7 seconds. + * Last ``validictory`` is much better with 370 or 23 milliseconds, but it does not follow all standards and it can be still slow for some purposes. That's why this project exists. It compiles definition into Python most stupid code -which people would had hard time to write by themselfs because of not-written-rule DRY -(don't repeat yourself). When you compile definition, then times are 25 miliseconds for -valid inputs and less than 2 miliseconds for invalid inputs. Pretty amazing, right? :-) +which people would have hard time to write by themselves because of not-written-rule DRY +(don't repeat yourself). When you compile definition, then times are 25 milliseconds for +valid inputs and less than 2 milliseconds for invalid inputs. Pretty amazing, right? :-) -You can try it for yourself with included script: +You can try it for yourself with an included script: .. code-block:: bash @@ -36,16 +36,16 @@ validictory valid ==> 0.4084212710149586 validictory invalid ==> 0.026061681972350925 -This library follows and implements `JSON schema draft-04, draft-06 and draft-07 +This library follows and implements `JSON schema draft-04, draft-06, and draft-07 `_. Sometimes it's not perfectly clear so I recommend also -check out this `understaning json schema `_. +check out this `understanding json schema `_. Note that there are some differences compared to JSON schema standard: * Regular expressions are full Python ones, not only what JSON schema allows. It's easier to allow everything and also it's faster to compile without limits. So keep in mind that when - you will use more advanced regular expression, it may not work with other library or in - other language. + you will use a more advanced regular expression, it may not work with other library or in + other languages. * JSON schema says you can use keyword ``default`` for providing default values. This implementation uses that and always returns transformed input data. @@ -65,7 +65,8 @@ # pylint: disable=redefined-builtin,dangerous-default-value,exec-used def compile(definition, handlers={}): """ - Generates validation function for validating JSON schema by ``definition``. Example: + Generates validation function for validating JSON schema passed in ``definition``. + Example: .. code-block:: python @@ -102,7 +103,8 @@ def compile(definition, handlers={}): You can pass mapping from URI to function that should be used to retrieve remote schemes used in your ``definition`` in parameter ``handlers``. - Exception :any:`JsonSchemaException` is thrown when validation fails. + Exception :any:`JsonSchemaException` is thrown when generation code fail + (wrong definition) or validation fails (data does not follow definition). """ resolver, code_generator = _factory(definition, handlers) global_state = code_generator.global_state @@ -114,8 +116,8 @@ def compile(definition, handlers={}): # pylint: disable=dangerous-default-value def compile_to_code(definition, handlers={}): """ - Generates validation function for validating JSON schema by ``definition`` - and returns compiled code. Example: + Generates validation code for validating JSON schema passed in ``definition``. + Example: .. code-block:: python @@ -131,8 +133,6 @@ def compile_to_code(definition, handlers={}): echo "{'type': 'string'}" | python3 -m fastjsonschema > your_file.py python3 -m fastjsonschema "{'type': 'string'}" > your_file.py - - Exception :any:`JsonSchemaException` is thrown when validation fails. """ _, code_generator = _factory(definition, handlers) return ( From 26469cdfc261c8842a27a8cfe9cdc4d35995aa2d Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 27 Sep 2018 07:47:58 +0000 Subject: [PATCH 121/201] #29 Fix string in const --- CHANGELOG.txt | 1 + fastjsonschema/draft06.py | 5 ++++- tests/test_const.py | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 tests/test_const.py diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 34182f4..77b240d 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,7 @@ === 2.4 (unreleased) * Fix overriding variables (in pattern properties, property names, unique items and contains) +* Fix string in const === 2.3 (2018-09-14) === diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py index f22a46c..a8bd6be 100644 --- a/fastjsonschema/draft06.py +++ b/fastjsonschema/draft06.py @@ -170,5 +170,8 @@ def generate_const(self): Only valid value is 42 in this example. """ - with self.l('if {variable} != {}:', self._definition['const']): + const = self._definition['const'] + if isinstance(const, str): + const = '"{}"'.format(const) + with self.l('if {variable} != {}:', const): self.l('raise JsonSchemaException("{name} must be same as const definition")') diff --git a/tests/test_const.py b/tests/test_const.py new file mode 100644 index 0000000..352df07 --- /dev/null +++ b/tests/test_const.py @@ -0,0 +1,14 @@ +import pytest + + +@pytest.mark.parametrize('value', ( + 'foo', + 42, + False, + [1, 2, 3] +)) +def test_const(asserter, value): + asserter({ + '$schema': 'http://json-schema.org/draft-06/schema', + 'const': value, + }, value, value) From e88b686bf84a1163a8a6a7841799d15af0c2f2a8 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 27 Sep 2018 09:53:54 +0000 Subject: [PATCH 122/201] Improved security --- CHANGELOG.txt | 1 + fastjsonschema/draft04.py | 35 +++++++++++++++++++++++++---- fastjsonschema/draft06.py | 10 ++++++++- fastjsonschema/exceptions.py | 6 +++++ tests/conftest.py | 4 ++-- tests/test_security.py | 43 ++++++++++++++++++++++++++++++++++++ 6 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 tests/test_security.py diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 77b240d..2f3f379 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,6 +2,7 @@ * Fix overriding variables (in pattern properties, property names, unique items and contains) * Fix string in const +* Improve security: not generating code from any definition === 2.3 (2018-09-14) === diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 3a33305..8f0bc00 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -1,5 +1,6 @@ import re +from .exceptions import JsonSchemaDefinitionException from .generator import CodeGenerator, enforce_list @@ -65,7 +66,10 @@ def generate_type(self): {'type': ['string', 'number']} """ types = enforce_list(self._definition['type']) - python_types = ', '.join(JSON_TYPE_TO_PYTHON_TYPE.get(t) for t in types) + try: + python_types = ', '.join(JSON_TYPE_TO_PYTHON_TYPE[t] for t in types) + except KeyError as exc: + raise JsonSchemaDefinitionException('Unknown type: {}'.format(exc)) extra = '' if ('number' in types or 'integer' in types) and 'boolean' not in types: @@ -84,6 +88,8 @@ def generate_enum(self): 'enum': ['a', 'b'], } """ + if not isinstance(self._definition['enum'], (list, tuple)): + raise JsonSchemaDefinitionException('enum must be an array') with self.l('if {variable} not in {enum}:'): self.l('raise JsonSchemaException("{name} must be one of {enum}")') @@ -192,20 +198,25 @@ def generate_not(self): def generate_min_length(self): with self.l('if isinstance({variable}, str):'): self.create_variable_with_length() + if not isinstance(self._definition['minLength'], int): + raise JsonSchemaDefinitionException('minLength must be a number') with self.l('if {variable}_len < {minLength}:'): self.l('raise JsonSchemaException("{name} must be longer than or equal to {minLength} characters")') def generate_max_length(self): with self.l('if isinstance({variable}, str):'): self.create_variable_with_length() + if not isinstance(self._definition['maxLength'], int): + raise JsonSchemaDefinitionException('maxLength must be a number') with self.l('if {variable}_len > {maxLength}:'): self.l('raise JsonSchemaException("{name} must be shorter than or equal to {maxLength} characters")') def generate_pattern(self): with self.l('if isinstance({variable}, str):'): - self._compile_regexps['{}'.format(self._definition['pattern'])] = re.compile(self._definition['pattern']) - with self.l('if not REGEX_PATTERNS["{}"].search({variable}):', self._definition['pattern']): - self.l('raise JsonSchemaException("{name} must match pattern {pattern}")') + safe_pattern = self._definition['pattern'].replace('"', '\\"') + self._compile_regexps[self._definition['pattern']] = re.compile(self._definition['pattern']) + with self.l('if not REGEX_PATTERNS["{}"].search({variable}):', safe_pattern): + self.l('raise JsonSchemaException("{name} must match pattern {}")', safe_pattern) def generate_format(self): """ @@ -240,6 +251,8 @@ def _generate_format(self, format_name, regexp_name, regexp): def generate_minimum(self): with self.l('if isinstance({variable}, (int, float)):'): + if not isinstance(self._definition['minimum'], (int, float)): + raise JsonSchemaDefinitionException('minimum must be a number') if self._definition.get('exclusiveMinimum', False): with self.l('if {variable} <= {minimum}:'): self.l('raise JsonSchemaException("{name} must be bigger than {minimum}")') @@ -249,6 +262,8 @@ def generate_minimum(self): def generate_maximum(self): with self.l('if isinstance({variable}, (int, float)):'): + if not isinstance(self._definition['maximum'], (int, float)): + raise JsonSchemaDefinitionException('maximum must be a number') if self._definition.get('exclusiveMaximum', False): with self.l('if {variable} >= {maximum}:'): self.l('raise JsonSchemaException("{name} must be smaller than {maximum}")') @@ -258,6 +273,8 @@ def generate_maximum(self): def generate_multiple_of(self): with self.l('if isinstance({variable}, (int, float)):'): + if not isinstance(self._definition['multipleOf'], (int, float)): + raise JsonSchemaDefinitionException('multipleOf must be a number') self.l('quotient = {variable} / {multipleOf}') with self.l('if int(quotient) != quotient:'): self.l('raise JsonSchemaException("{name} must be multiple of {multipleOf}")') @@ -265,6 +282,8 @@ def generate_multiple_of(self): def generate_min_items(self): self.create_variable_is_list() with self.l('if {variable}_is_list:'): + if not isinstance(self._definition['minItems'], int): + raise JsonSchemaDefinitionException('minItems must be a number') self.create_variable_with_length() with self.l('if {variable}_len < {minItems}:'): self.l('raise JsonSchemaException("{name} must contain at least {minItems} items")') @@ -272,6 +291,8 @@ def generate_min_items(self): def generate_max_items(self): self.create_variable_is_list() with self.l('if {variable}_is_list:'): + if not isinstance(self._definition['maxItems'], int): + raise JsonSchemaDefinitionException('maxItems must be a number') self.create_variable_with_length() with self.l('if {variable}_len > {maxItems}:'): self.l('raise JsonSchemaException("{name} must contain less than or equal to {maxItems} items")') @@ -357,6 +378,8 @@ def generate_items(self): def generate_min_properties(self): self.create_variable_is_dict() with self.l('if {variable}_is_dict:'): + if not isinstance(self._definition['minProperties'], int): + raise JsonSchemaDefinitionException('minProperties must be a number') self.create_variable_with_length() with self.l('if {variable}_len < {minProperties}:'): self.l('raise JsonSchemaException("{name} must contain at least {minProperties} properties")') @@ -364,6 +387,8 @@ def generate_min_properties(self): def generate_max_properties(self): self.create_variable_is_dict() with self.l('if {variable}_is_dict:'): + if not isinstance(self._definition['maxProperties'], int): + raise JsonSchemaDefinitionException('maxProperties must be a number') self.create_variable_with_length() with self.l('if {variable}_len > {maxProperties}:'): self.l('raise JsonSchemaException("{name} must contain less than or equal to {maxProperties} properties")') @@ -371,6 +396,8 @@ def generate_max_properties(self): def generate_required(self): self.create_variable_is_dict() with self.l('if {variable}_is_dict:'): + if not isinstance(self._definition['required'], (list, tuple)): + raise JsonSchemaDefinitionException('required must be an array') self.create_variable_with_length() with self.l('if not all(prop in {variable} for prop in {required}):'): self.l('raise JsonSchemaException("{name} must contain {required} properties")') diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py index a8bd6be..4dfa60a 100644 --- a/fastjsonschema/draft06.py +++ b/fastjsonschema/draft06.py @@ -1,4 +1,5 @@ from .draft04 import CodeGeneratorDraft04, JSON_TYPE_TO_PYTHON_TYPE +from .exceptions import JsonSchemaDefinitionException from .generator import enforce_list @@ -53,7 +54,10 @@ def generate_type(self): {'type': ['string', 'number']} """ types = enforce_list(self._definition['type']) - python_types = ', '.join(JSON_TYPE_TO_PYTHON_TYPE.get(t) for t in types) + try: + python_types = ', '.join(JSON_TYPE_TO_PYTHON_TYPE[t] for t in types) + except KeyError as exc: + raise JsonSchemaDefinitionException('Unknown type: {}'.format(exc)) extra = '' @@ -70,11 +74,15 @@ def generate_type(self): def generate_exclusive_minimum(self): with self.l('if isinstance({variable}, (int, float)):'): + if not isinstance(self._definition['exclusiveMinimum'], (int, float)): + raise JsonSchemaDefinitionException('exclusiveMinimum must be an integer or a float') with self.l('if {variable} <= {exclusiveMinimum}:'): self.l('raise JsonSchemaException("{name} must be bigger than {exclusiveMinimum}")') def generate_exclusive_maximum(self): with self.l('if isinstance({variable}, (int, float)):'): + if not isinstance(self._definition['exclusiveMaximum'], (int, float)): + raise JsonSchemaDefinitionException('exclusiveMaximum must be an integer or a float') with self.l('if {variable} >= {exclusiveMaximum}:'): self.l('raise JsonSchemaException("{name} must be smaller than {exclusiveMaximum}")') diff --git a/fastjsonschema/exceptions.py b/fastjsonschema/exceptions.py index 48f7370..912bebe 100644 --- a/fastjsonschema/exceptions.py +++ b/fastjsonschema/exceptions.py @@ -7,3 +7,9 @@ class JsonSchemaException(ValueError): def __init__(self, message): super().__init__(message) self.message = message + + +class JsonSchemaDefinitionException(JsonSchemaException): + """ + Exception raised by generator of validation function. + """ diff --git a/tests/conftest.py b/tests/conftest.py index aa745bb..8f805cf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,14 +10,14 @@ import pytest from fastjsonschema import JsonSchemaException, compile -from fastjsonschema.draft04 import CodeGeneratorDraft04 +from fastjsonschema.draft07 import CodeGeneratorDraft07 @pytest.fixture def asserter(): def f(definition, value, expected): # When test fails, it will show up code. - code_generator = CodeGeneratorDraft04(definition) + code_generator = CodeGeneratorDraft07(definition) print(code_generator.func_code) pprint(code_generator.global_state) diff --git a/tests/test_security.py b/tests/test_security.py new file mode 100644 index 0000000..1d325de --- /dev/null +++ b/tests/test_security.py @@ -0,0 +1,43 @@ +import pytest + +from fastjsonschema import JsonSchemaDefinitionException, compile + + +@pytest.mark.parametrize('schema', [ + {'type': 'validate(10)'}, + {'enum': 'validate(10)'}, + {'minLength': 'validate(10)'}, + {'maxLength': 'validate(10)'}, + {'minimum': 'validate(10)'}, + {'maximum': 'validate(10)'}, + {'multipleOf': 'validate(10)'}, + {'minItems': 'validate(10)'}, + {'maxItems': 'validate(10)'}, + {'minProperties': 'validate(10)'}, + {'maxProperties': 'validate(10)'}, + {'required': 'validate(10)'}, + {'exclusiveMinimum': 'validate(10)'}, + {'exclusiveMaximum': 'validate(10)'}, +]) +def test_not_generate_code_from_definition(schema): + with pytest.raises(JsonSchemaDefinitionException): + compile({ + '$schema': 'http://json-schema.org/draft-07/schema', + **schema + }) + + +@pytest.mark.parametrize('schema,value', [ + ({'const': 'validate(10)'}, 'validate(10)'), + ({'pattern': '" + validate("10") + "'}, '" validate"10" "'), + ({'pattern': "' + validate('10') + '"}, '\' validate\'10\' \''), + ({'pattern': "' + validate(\"10\") + '"}, '\' validate"10" \''), + ({'properties': { + 'validate(10)': {'type': 'string'}, + }}, {'validate(10)': '10'}), +]) +def test_generate_code_with_proper_variable_names(asserter, schema, value): + asserter({ + '$schema': 'http://json-schema.org/draft-07/schema', + **schema + }, value, value) From 8432be5b610808112d952eeff55cbb02533dc8b7 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 27 Sep 2018 09:55:35 +0000 Subject: [PATCH 123/201] Added validate function --- CHANGELOG.txt | 1 + fastjsonschema/__init__.py | 33 +++++++++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 2f3f379..26df46f 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -3,6 +3,7 @@ * Fix overriding variables (in pattern properties, property names, unique items and contains) * Fix string in const * Improve security: not generating code from any definition +* Added validate function for lazy programmers === 2.3 (2018-09-14) === diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 63b34cd..e0a103d 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -55,11 +55,30 @@ from .draft04 import CodeGeneratorDraft04 from .draft06 import CodeGeneratorDraft06 from .draft07 import CodeGeneratorDraft07 -from .exceptions import JsonSchemaException +from .exceptions import JsonSchemaException, JsonSchemaDefinitionException from .ref_resolver import RefResolver from .version import VERSION -__all__ = ('VERSION', 'JsonSchemaException', 'compile', 'compile_to_code') +__all__ = ('VERSION', 'JsonSchemaException', 'validate', 'compile', 'compile_to_code') + + +def validate(definition, data): + """ + Validation function for lazy programmers or for use cases, when you need + to call validation only once, so you do not have to compile it first. + Use it only when you do not care about performance (even thought it will + be still faster than alternative implementations). + + .. code-block:: python + + import fastjsonschema + + validate({'type': 'string'}, 'hello') + # same as: compile({'type': 'string'})('hello') + + Preffered is to use :any:`compile` function. + """ + return compile(definition)(data) # pylint: disable=redefined-builtin,dangerous-default-value,exec-used @@ -103,8 +122,11 @@ def compile(definition, handlers={}): You can pass mapping from URI to function that should be used to retrieve remote schemes used in your ``definition`` in parameter ``handlers``. - Exception :any:`JsonSchemaException` is thrown when generation code fail - (wrong definition) or validation fails (data does not follow definition). + Exception :any:`JsonSchemaDefinitionException` is raised when generating the + code fails (bad definition). + + Exception :any:`JsonSchemaException` is raised from generated funtion when + validation fails (data do not follow the definition). """ resolver, code_generator = _factory(definition, handlers) global_state = code_generator.global_state @@ -133,6 +155,9 @@ def compile_to_code(definition, handlers={}): echo "{'type': 'string'}" | python3 -m fastjsonschema > your_file.py python3 -m fastjsonschema "{'type': 'string'}" > your_file.py + + Exception :any:`JsonSchemaDefinitionException` is raised when generating the + code fails (bad definition). """ _, code_generator = _factory(definition, handlers) return ( From a9c37948e1ede8b72dcabb59b68a9be20d1b0417 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 27 Sep 2018 13:53:10 +0000 Subject: [PATCH 124/201] Comment about performance --- fastjsonschema/draft04.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 8f0bc00..c5fb625 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -18,6 +18,10 @@ # pylint: disable=too-many-instance-attributes,too-many-public-methods class CodeGeneratorDraft04(CodeGenerator): # pylint: disable=line-too-long + # I was thinking about using ipaddress module instead of regexps for example, but it's big + # difference in performance. With a module I got this difference: over 100 ms with a module + # vs. 9 ms with a regex! Other modules are also unefective or not available in standard + # library. Some regexps are not 100% precise but good enough, fast and without dependencies. FORMAT_REGEXS = { 'date-time': r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d(?:\.\d+)?(?:[+-][0-2]\d:[0-5]\d|Z)?$', 'email': r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$', From 90c3d5202973aec77af57198092f0a34e4426310 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 27 Sep 2018 13:53:54 +0000 Subject: [PATCH 125/201] Version 2.4 --- CHANGELOG.txt | 2 +- fastjsonschema/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 26df46f..c09c7a1 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,4 +1,4 @@ -=== 2.4 (unreleased) +=== 2.4 (2018-09-27) * Fix overriding variables (in pattern properties, property names, unique items and contains) * Fix string in const diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index f4d1515..4c0bedb 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.3' +VERSION = '2.4' From 6373c4b7c0b1fba03c9d170e7e1aa2c9e7064bd0 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 1 Oct 2018 07:50:40 +0000 Subject: [PATCH 126/201] Doc fix --- fastjsonschema/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index e0a103d..31ce632 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -59,7 +59,7 @@ from .ref_resolver import RefResolver from .version import VERSION -__all__ = ('VERSION', 'JsonSchemaException', 'validate', 'compile', 'compile_to_code') +__all__ = ('VERSION', 'JsonSchemaException', 'JsonSchemaDefinitionException', 'validate', 'compile', 'compile_to_code') def validate(definition, data): From ddb4813a40b927d09142478cdf002b1789a44387 Mon Sep 17 00:00:00 2001 From: Ben Donohue Date: Tue, 2 Oct 2018 17:49:55 -0400 Subject: [PATCH 127/201] typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🚀 --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 0ee9e58..d0e91da 100644 --- a/README.rst +++ b/README.rst @@ -13,7 +13,7 @@ Fast JSON schema for Python :target: https://pypi.python.org/pypi/fastjsonschema This project was made to come up with fast JSON validations. It is at -least an order of magnitude faster than other Python implemantaions. +least an order of magnitude faster than other Python implementations. See `documentation `_ for performance test details. From c0f196463d517b45cf55f717b1468e4d9a5b68e8 Mon Sep 17 00:00:00 2001 From: "joao.alves" Date: Thu, 4 Oct 2018 15:57:24 +0100 Subject: [PATCH 128/201] Be more flexible in email validation There is no magic regex that works 100% of the time, therefore the validation should lean towards allowing most strings rather than blocking potentially valid email addresses. jsonschema is using "if @ in email" whereas json-spec uses the regex I suggest. I think fastjsonschema should follow the same principle, because most people will migrate from these. For a list of valid/invalid emails see (for example): https://gist.github.com/cjaoude/fd9910626629b53c4d25 --- fastjsonschema/draft04.py | 2 +- fastjsonschema/draft07.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index c5fb625..3e5bee3 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -24,7 +24,7 @@ class CodeGeneratorDraft04(CodeGenerator): # library. Some regexps are not 100% precise but good enough, fast and without dependencies. FORMAT_REGEXS = { 'date-time': r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d(?:\.\d+)?(?:[+-][0-2]\d:[0-5]\d|Z)?$', - 'email': r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$', + 'email': r'^[^@]+@[^@]+\.[^@]+$', 'hostname': r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{0,61}[A-Za-z0-9])$', 'ipv4': r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$', 'ipv6': r'^(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|::(?:[0-9A-Fa-f]{1,4}:){5}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){4}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){3}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,2}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){2}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,3}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}:(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,4}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,5}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}|(?:(?:[0-9A-Fa-f]{1,4}:){,6}[0-9A-Fa-f]{1,4})?::)$', diff --git a/fastjsonschema/draft07.py b/fastjsonschema/draft07.py index 44d576a..cc3978c 100644 --- a/fastjsonschema/draft07.py +++ b/fastjsonschema/draft07.py @@ -6,7 +6,7 @@ class CodeGeneratorDraft07(CodeGeneratorDraft06): 'date': r'^(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})$', 'iri': r'^\w+:(\/?\/?)[^\s]+$', 'iri-reference': r'^(\w+:(\/?\/?))?[^#\\\s]*(#[^\\\s]*)?$', - 'idn-email': r'^\w+@\w+\.\w+$', + 'idn-email': r'^[^@]+@[^@]+\.[^@]+$', #'idn-hostname': r'', 'relative-json-pointer': r'^(?:0|[1-9][0-9]*)(?:#|(?:\/(?:[^~/]|~0|~1)*)*)$', #'regex': r'', From 7d4fde5a9e432d45120fd9ddc5649cb1bad75a21 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 4 Oct 2018 17:18:32 +0000 Subject: [PATCH 129/201] Docs --- README.rst | 30 +----------------------------- docs/index.rst | 2 ++ fastjsonschema/__init__.py | 26 +++++++++++++++----------- 3 files changed, 18 insertions(+), 40 deletions(-) diff --git a/README.rst b/README.rst index d0e91da..428a9dc 100644 --- a/README.rst +++ b/README.rst @@ -12,32 +12,4 @@ Fast JSON schema for Python :alt: Supported Python versions :target: https://pypi.python.org/pypi/fastjsonschema -This project was made to come up with fast JSON validations. It is at -least an order of magnitude faster than other Python implementations. -See `documentation `_ for -performance test details. - -This library follows and implements `JSON schema draft-04, draft-06 and draft-07 -`_. Note that there are some differences compared to JSON -schema standard: - - * Regular expressions are full Python ones, not only what JSON schema allows. It's easier - to allow everything and also it's faster to compile without limits. So keep in mind that when - you will use more advanced regular expression, it may not work with other library or in - other language. - * JSON schema says you can use keyword ``default`` for providing default values. This implementation - uses that and always returns transformed input data. - -Install -------- - -.. code-block:: bash - - pip install fastjsonschema - -Support only for Python 3.3 and higher. - -Documentation -------------- - -Documentation: `https://horejsek.github.io/python-fastjsonschema `_ +See `documentation `_. diff --git a/docs/index.rst b/docs/index.rst index 884c322..fb68d32 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,6 +8,8 @@ Installation pip install fastjsonschema +Support only for Python 3.3 and higher. + Documentation ************* diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 31ce632..cb8e03e 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -5,20 +5,26 @@ # """ -This project was made to come up with fast JSON validations. Just let's see some numbers first: +``fastjsonschema`` implements validation of JSON documents by JSON schema. +The library implements JSON schema drafts 04, 06 and 07. The main purpose is +to have a really fast implementation. See some numbers: * Probably most popular ``jsonschema`` can take up to 5 seconds for valid inputs and 1.2 seconds for invalid inputs. * Second most popular ``json-spec`` is even worse with up to 7.2 and 1.7 seconds. - * Last ``validictory`` is much better with 370 or 23 milliseconds, but it does not - follow all standards and it can be still slow for some purposes. + * Last ``validictory``, now deprecated, is much better with 370 or 23 milliseconds, + but it does not follow all standards and it can be still slow for some purposes. -That's why this project exists. It compiles definition into Python most stupid code -which people would have hard time to write by themselves because of not-written-rule DRY -(don't repeat yourself). When you compile definition, then times are 25 milliseconds for -valid inputs and less than 2 milliseconds for invalid inputs. Pretty amazing, right? :-) +With this library you can gain big improvements as ``fastjsonschema`` takes +only about 25 milliseconds for valid inputs and 2 milliseconds for invalid ones. +Pretty amazing, right? :-) -You can try it for yourself with an included script: +Technically it works by generating the most stupid code on the fly which is fast but +is hard to write by hand. The best efficiency is achieved when compiled once and used +many times, of course. It works similarly like regular expressions. But you can also +generate the code to the file which is even slightly faster. + +You can do the performance on your computer or server with an included script: .. code-block:: bash @@ -38,7 +44,7 @@ This library follows and implements `JSON schema draft-04, draft-06, and draft-07 `_. Sometimes it's not perfectly clear so I recommend also -check out this `understanding json schema `_. +check out this `understanding JSON schema `_. Note that there are some differences compared to JSON schema standard: @@ -48,8 +54,6 @@ other languages. * JSON schema says you can use keyword ``default`` for providing default values. This implementation uses that and always returns transformed input data. - -Support only for Python 3.3 and higher. """ from .draft04 import CodeGeneratorDraft04 From 80d8c923fee814beed5496cd95218015c127d149 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 4 Oct 2018 17:23:12 +0000 Subject: [PATCH 130/201] Docs --- docs/index.rst | 16 ++-------------- fastjsonschema/__init__.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index fb68d32..40c5af9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,17 +1,5 @@ -fastjsonschema documentation -############################ - -Installation -************ - -.. code-block:: bash - - pip install fastjsonschema - -Support only for Python 3.3 and higher. - -Documentation -************* +Fast JSON schema for Python +########################### .. automodule:: fastjsonschema :members: diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index cb8e03e..db42de4 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -5,6 +5,18 @@ # """ +Installation +************ + +.. code-block:: bash + + pip install fastjsonschema + +Support only for Python 3.3 and higher. + +About +***** + ``fastjsonschema`` implements validation of JSON documents by JSON schema. The library implements JSON schema drafts 04, 06 and 07. The main purpose is to have a really fast implementation. See some numbers: @@ -54,6 +66,9 @@ other languages. * JSON schema says you can use keyword ``default`` for providing default values. This implementation uses that and always returns transformed input data. + +API +*** """ from .draft04 import CodeGeneratorDraft04 From 5a26a8a4631a80949b1b3fd35bcabbab1cf390b9 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 22 Oct 2018 15:15:14 +0200 Subject: [PATCH 131/201] Version 2.5 --- CHANGELOG.txt | 5 +++++ fastjsonschema/version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index c09c7a1..3b982b8 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +=== 2.5 (2018-10-22) + +* E-mail regex allows any e-mail with @. + + === 2.4 (2018-09-27) * Fix overriding variables (in pattern properties, property names, unique items and contains) diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index 4c0bedb..07d5da9 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.4' +VERSION = '2.5' From 80b772fed9b95025c47c076f2a19aae80adc49dc Mon Sep 17 00:00:00 2001 From: bcaller Date: Fri, 26 Oct 2018 15:53:16 +0100 Subject: [PATCH 132/201] Use ECMA 262 definition of $ (end of string like \Z) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We often set patterns on strings e.g. {"type": "string", "pattern": "^[abc]+$"} This matches e.g. `aaa`, `b`. It also surprisingly also matches these strings with a final \n e.g. `aaa\n`, `b\n` The python $ character matches end of string. But it also matches if a \n is the last character even when multiline is False (default). Instead in python re \Z matches the end of the string only. This is what we want but it is very python specific and so wouldn't be friendly in documentation. In this commit, $ is swapped with \Z inside the pattern handling code. A breaking change with my substitution here is that if you have used $ inside a set of characters [$] you now would need to escape it like [\$]. Python docs: "Matches the end of the string or just before the newline at the end of the string, and in MULTILINE mode also matches before a newline. foo matches both ‘foo’ and ‘foobar’, while the regular expression foo$ matches only ‘foo’. More interestingly, searching for foo.$ in 'foo1\nfoo2\n' matches ‘foo2’ normally, but ‘foo1’ in MULTILINE mode; searching for a single $ in 'foo\n' will find two (empty) matches: one just before the newline, and one at the end of the string." https://docs.python.org/3/library/re.html The json schema draft 3 states: "Regular expressions SHOULD follow the regular expression specification from ECMA 262/Perl 5" https://tools.ietf.org/html/draft-zyp-json-schema-03 In that regular expression definition: If multiline is FALSE, $ must match the end of the string. If multiline is TRUE, $ may match \n. https://www.ecma-international.org/ecma-262/5.1/#sec-15.10.2.6 --- fastjsonschema/draft04.py | 8 ++++++-- tests/test_string.py | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 3e5bee3..27703c4 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -14,6 +14,8 @@ 'object': 'dict', } +DOLLAR_FINDER = re.compile(r"(? Date: Fri, 26 Oct 2018 16:06:48 +0100 Subject: [PATCH 133/201] Forbid \n at end of email, datetime, etc The python $ allows a \n as the final character. The email address `abc@def.com\n` should fail validation. Instead of $ we actually want \Z. --- fastjsonschema/draft04.py | 12 ++++++------ fastjsonschema/draft06.py | 6 +++--- fastjsonschema/draft07.py | 12 ++++++------ tests/test_format.py | 2 ++ 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 27703c4..d8ea35c 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -25,12 +25,12 @@ class CodeGeneratorDraft04(CodeGenerator): # vs. 9 ms with a regex! Other modules are also unefective or not available in standard # library. Some regexps are not 100% precise but good enough, fast and without dependencies. FORMAT_REGEXS = { - 'date-time': r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d(?:\.\d+)?(?:[+-][0-2]\d:[0-5]\d|Z)?$', - 'email': r'^[^@]+@[^@]+\.[^@]+$', - 'hostname': r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{0,61}[A-Za-z0-9])$', - 'ipv4': r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$', - 'ipv6': r'^(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|::(?:[0-9A-Fa-f]{1,4}:){5}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){4}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){3}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,2}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){2}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,3}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}:(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,4}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,5}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}|(?:(?:[0-9A-Fa-f]{1,4}:){,6}[0-9A-Fa-f]{1,4})?::)$', - 'uri': r'^\w+:(\/?\/?)[^\s]+$', + 'date-time': r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d(?:\.\d+)?(?:[+-][0-2]\d:[0-5]\d|Z)?\Z', + 'email': r'^[^@]+@[^@]+\.[^@]+\Z', + 'hostname': r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{0,61}[A-Za-z0-9])\Z', + 'ipv4': r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\Z', + 'ipv6': r'^(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|::(?:[0-9A-Fa-f]{1,4}:){5}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){4}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){3}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,2}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){2}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,3}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}:(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,4}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,5}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}|(?:(?:[0-9A-Fa-f]{1,4}:){,6}[0-9A-Fa-f]{1,4})?::)\Z', + 'uri': r'^\w+:(\/?\/?)[^\s]+\Z', } def __init__(self, definition, resolver=None): diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py index 4dfa60a..5ab86f2 100644 --- a/fastjsonschema/draft06.py +++ b/fastjsonschema/draft06.py @@ -5,13 +5,13 @@ class CodeGeneratorDraft06(CodeGeneratorDraft04): FORMAT_REGEXS = dict(CodeGeneratorDraft04.FORMAT_REGEXS, **{ - 'json-pointer': r'^(/(([^/~])|(~[01]))*)*$', - 'uri-reference': r'^(\w+:(\/?\/?))?[^#\\\s]*(#[^\\\s]*)?$', + 'json-pointer': r'^(/(([^/~])|(~[01]))*)*\Z', + 'uri-reference': r'^(\w+:(\/?\/?))?[^#\\\s]*(#[^\\\s]*)?\Z', 'uri-template': ( r'^(?:(?:[^\x00-\x20\"\'<>%\\^`{|}]|%[0-9a-f]{2})|' r'\{[+#./;?&=,!@|]?(?:[a-z0-9_]|%[0-9a-f]{2})+' r'(?::[1-9][0-9]{0,3}|\*)?(?:,(?:[a-z0-9_]|%[0-9a-f]{2})+' - r'(?::[1-9][0-9]{0,3}|\*)?)*\})*$' + r'(?::[1-9][0-9]{0,3}|\*)?)*\})*\Z' ), }) diff --git a/fastjsonschema/draft07.py b/fastjsonschema/draft07.py index cc3978c..d553d3b 100644 --- a/fastjsonschema/draft07.py +++ b/fastjsonschema/draft07.py @@ -3,17 +3,17 @@ class CodeGeneratorDraft07(CodeGeneratorDraft06): FORMAT_REGEXS = dict(CodeGeneratorDraft06.FORMAT_REGEXS, **{ - 'date': r'^(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})$', - 'iri': r'^\w+:(\/?\/?)[^\s]+$', - 'iri-reference': r'^(\w+:(\/?\/?))?[^#\\\s]*(#[^\\\s]*)?$', - 'idn-email': r'^[^@]+@[^@]+\.[^@]+$', + 'date': r'^(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})\Z', + 'iri': r'^\w+:(\/?\/?)[^\s]+\Z', + 'iri-reference': r'^(\w+:(\/?\/?))?[^#\\\s]*(#[^\\\s]*)?\Z', + 'idn-email': r'^[^@]+@[^@]+\.[^@]+\Z', #'idn-hostname': r'', - 'relative-json-pointer': r'^(?:0|[1-9][0-9]*)(?:#|(?:\/(?:[^~/]|~0|~1)*)*)$', + 'relative-json-pointer': r'^(?:0|[1-9][0-9]*)(?:#|(?:\/(?:[^~/]|~0|~1)*)*)\Z', #'regex': r'', 'time': ( r'^(?P\d{1,2}):(?P\d{1,2})' r'(?::(?P\d{1,2})(?:\.(?P\d{1,6}))?' - r'([zZ]|[+-]\d\d:\d\d)?)?$' + r'([zZ]|[+-]\d\d:\d\d)?)?\Z' ), }) diff --git a/tests/test_format.py b/tests/test_format.py index 7302694..b7171e4 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -7,6 +7,7 @@ @pytest.mark.parametrize('value, expected', [ ('', exc), ('bla', exc), + ('2018-02-05T14:17:10.00Z\n', exc), ('2018-02-05T14:17:10.00Z', '2018-02-05T14:17:10.00Z'), ('2018-02-05T14:17:10Z', '2018-02-05T14:17:10Z'), ]) @@ -20,6 +21,7 @@ def test_datetime(asserter, value, expected): ('LDhsjf878&d', exc), ('bla.bla-', exc), ('example.example.com-', exc), + ('example.example.com\n', exc), ('localhost', 'localhost'), ('example.com', 'example.com'), ('example.de', 'example.de'), From 7b738b713fbc126ace5796927cfdfa918bddd79c Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 1 Nov 2018 14:16:33 +0100 Subject: [PATCH 134/201] Version 2.6 --- AUTHORS | 1 + CHANGELOG.txt | 5 +++++ fastjsonschema/__init__.py | 3 +++ fastjsonschema/version.py | 2 +- 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 0cf8116..15a1ef2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -4,6 +4,7 @@ Michal Hořejšek CONTRIBUTORS anentropic Antti Jokipii +bcaller Frederik Petersen Guillaume Desvé Kris Molendyke diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 3b982b8..9258d5f 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +=== 2.6 (2018-11-01) + +* Swap $ in regexps to \Z to follow ECMA 262 ($ matches really the end of the string, not the end or new line and the end). Because of that your regular expressions have to escape dollar when used as regular character. + + === 2.5 (2018-10-22) * E-mail regex allows any e-mail with @. diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index db42de4..65b2119 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -64,6 +64,9 @@ to allow everything and also it's faster to compile without limits. So keep in mind that when you will use a more advanced regular expression, it may not work with other library or in other languages. + * Because Python matches new line for a dollar in regular expressions (`a$` matches `a` and `a\n`), + instead of `$` is used `\Z` and all dollars in your regular expression are changed to `\Z`. + When you want to use dollar as regular character, you have to escape it (`\$`). * JSON schema says you can use keyword ``default`` for providing default values. This implementation uses that and always returns transformed input data. diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index 07d5da9..9cacf7c 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.5' +VERSION = '2.6' From bcfb834be3dc680c0f69fb0097b6a44c3c97ce8c Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 1 Nov 2018 14:21:20 +0100 Subject: [PATCH 135/201] Docs --- fastjsonschema/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 65b2119..4aa4c46 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -64,9 +64,9 @@ to allow everything and also it's faster to compile without limits. So keep in mind that when you will use a more advanced regular expression, it may not work with other library or in other languages. - * Because Python matches new line for a dollar in regular expressions (`a$` matches `a` and `a\n`), - instead of `$` is used `\Z` and all dollars in your regular expression are changed to `\Z`. - When you want to use dollar as regular character, you have to escape it (`\$`). + * Because Python matches new line for a dollar in regular expressions (``a$`` matches ``a`` and ``a\\n``), + instead of ``$`` is used ``\Z`` and all dollars in your regular expression are changed to ``\\Z`` + as well. When you want to use dollar as regular character, you have to escape it (``\$``). * JSON schema says you can use keyword ``default`` for providing default values. This implementation uses that and always returns transformed input data. From dd79d0fbe1122cfe04ad550767682348742008a9 Mon Sep 17 00:00:00 2001 From: bcaller Date: Mon, 5 Nov 2018 16:14:21 +0000 Subject: [PATCH 136/201] Python 3.7 regex bug fixed Added py37 to tox environment list. It finds an error with my previous PR: The docs show a change in `Pattern.sub`. ``` Changed in version 3.7: Unknown escapes in repl consisting of '\' and an ASCII letter now are errors. ``` ``` >>> import re >>> DOLLAR_FINDER = re.compile(r"(?>> DOLLAR_FINDER.sub(r'\Z', 'hello$') Traceback (most recent call last): File "/usr/local/lib/python3.7/sre_parse.py", line 1021, in parse_template this = chr(ESCAPES[this][1]) KeyError: '\\Z' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "", line 1, in File "/usr/local/lib/python3.7/re.py", line 309, in _subx template = _compile_repl(template, pattern) File "/usr/local/lib/python3.7/re.py", line 300, in _compile_repl return sre_parse.parse_template(repl, pattern) File "/usr/local/lib/python3.7/sre_parse.py", line 1024, in parse_template raise s.error('bad escape %s' % this, len(this)) re.error: bad escape \Z at position 0 >>> DOLLAR_FINDER.sub(r'\\Z', 'hello$') 'hello\\Z' ``` --- fastjsonschema/draft04.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index d8ea35c..5e3c2d6 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -221,7 +221,7 @@ def generate_pattern(self): with self.l('if isinstance({variable}, str):'): pattern = self._definition['pattern'] safe_pattern = pattern.replace('"', '\\"') - end_of_string_fixed_pattern = DOLLAR_FINDER.sub(r'\Z', pattern) + end_of_string_fixed_pattern = DOLLAR_FINDER.sub(r'\\Z', pattern) self._compile_regexps[pattern] = re.compile(end_of_string_fixed_pattern) with self.l('if not REGEX_PATTERNS["{}"].search({variable}):', safe_pattern): self.l('raise JsonSchemaException("{name} must match pattern {}")', safe_pattern) diff --git a/tox.ini b/tox.ini index ff5d2d5..3b1287e 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py{34,35,36} +envlist = py{34,35,36,37} [testenv] whitelist_externals = From 80b1522dfb4589b369649ca41a4ba23b04ff9e62 Mon Sep 17 00:00:00 2001 From: bcaller Date: Mon, 5 Nov 2018 16:38:06 +0000 Subject: [PATCH 137/201] Fix tox & linting, add linting to tox.ini --- fastjsonschema/__init__.py | 2 +- pylintrc | 4 ++-- tox.ini | 12 +++++++++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 4aa4c46..3c15e44 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -4,7 +4,7 @@ # \/ \/ If you look at it, you might die. # -""" +r""" Installation ************ diff --git a/pylintrc b/pylintrc index a5abbfe..c64248d 100644 --- a/pylintrc +++ b/pylintrc @@ -3,8 +3,8 @@ ignore=tests [MESSAGES CONTROL] -# missing-docstring only for now, remove after this issue is deployed https://github.com/PyCQA/pylint/issues/1164 -disable=missing-docstring +# missing-docstring can be removed after this issue is deployed https://github.com/PyCQA/pylint/issues/1164 +disable=duplicate-code,missing-docstring [REPORTS] output-format=colorized diff --git a/tox.ini b/tox.ini index 3b1287e..cd81793 100644 --- a/tox.ini +++ b/tox.ini @@ -4,10 +4,16 @@ # and then run "tox" from this directory. [tox] -envlist = py{34,35,36,37} +envlist = py{34,35,36,37},lint [testenv] -whitelist_externals = +deps = pytest commands = - pytest + pytest -m "not benchmark" + +[testenv:lint] +deps = + pylint +commands = + pylint fastjsonschema From 05b0ee2cd6a497e711947787c1db358ba894266b Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Fri, 16 Nov 2018 12:24:27 +0100 Subject: [PATCH 138/201] Version 2.7 --- CHANGELOG.txt | 7 ++++++- fastjsonschema/version.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 9258d5f..ab76434 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +=== 2.7 (2018-11-16) + +* Fix regexps for Python 3.7 + + === 2.6 (2018-11-01) * Swap $ in regexps to \Z to follow ECMA 262 ($ matches really the end of the string, not the end or new line and the end). Because of that your regular expressions have to escape dollar when used as regular character. @@ -5,7 +10,7 @@ === 2.5 (2018-10-22) -* E-mail regex allows any e-mail with @. +* E-mail regex allows any e-mail with @ === 2.4 (2018-09-27) diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index 9cacf7c..22afd2b 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.6' +VERSION = '2.7' From c0ff67be79919bcf510c79962205fa9000d4ff28 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Sat, 5 Jan 2019 12:34:26 +0100 Subject: [PATCH 139/201] #40 Fix invalid code with quotes in enum --- fastjsonschema/draft04.py | 6 ++++-- tests/test_common.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 5e3c2d6..52e83ec 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -94,10 +94,12 @@ def generate_enum(self): 'enum': ['a', 'b'], } """ - if not isinstance(self._definition['enum'], (list, tuple)): + enum = self._definition['enum'] + if not isinstance(enum, (list, tuple)): raise JsonSchemaDefinitionException('enum must be an array') with self.l('if {variable} not in {enum}:'): - self.l('raise JsonSchemaException("{name} must be one of {enum}")') + enum = str(enum).replace('"', '\\"') + self.l('raise JsonSchemaException("{name} must be one of {}")', enum) def generate_all_of(self): """ diff --git a/tests/test_common.py b/tests/test_common.py index 6168a61..3eeb747 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -4,7 +4,7 @@ from fastjsonschema import JsonSchemaException -exc = JsonSchemaException('data must be one of [1, 2, \'a\']') +exc = JsonSchemaException('data must be one of [1, 2, \'a\', "b\'c"]') @pytest.mark.parametrize('value, expected', [ (1, 1), (2, 2), @@ -13,7 +13,7 @@ ('aa', exc), ]) def test_enum(asserter, value, expected): - asserter({'enum': [1, 2, 'a']}, value, expected) + asserter({'enum': [1, 2, 'a', "b'c"]}, value, expected) exc = JsonSchemaException('data must be string or number') From a185faafbfa4d102176f802ef03ab42db8fdf5db Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Sat, 5 Jan 2019 12:35:50 +0100 Subject: [PATCH 140/201] Version 2.8 --- CHANGELOG.txt | 5 +++++ fastjsonschema/version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index ab76434..ad9bc7c 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +=== 2.8 (2019-01-05) + +* Fix quotes in enum generating invalid code + + === 2.7 (2018-11-16) * Fix regexps for Python 3.7 diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index 22afd2b..72efc6b 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.7' +VERSION = '2.8' From aa3644ae05d74c628eb92aeb6cd4499bf73961cb Mon Sep 17 00:00:00 2001 From: remorses Date: Sun, 3 Mar 2019 12:36:14 +0100 Subject: [PATCH 141/201] substituted requests package with urllib --- fastjsonschema/ref_resolver.py | 14 +++++--------- setup.py | 4 ---- tests/json_schema/utils.py | 6 ++++-- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index fb3d663..803c525 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -13,7 +13,6 @@ from urllib.parse import unquote from urllib.request import urlopen -import requests from .exceptions import JsonSchemaException @@ -47,21 +46,18 @@ def resolve_remote(uri, handlers): .. note:: - Requests_ library is used to fetch ``http`` or ``https`` - requests from the remote ``uri``, if handlers does not - define otherwise. + urllib library is used to fetch requests from the remote ``uri`` + if handlers does notdefine otherwise. - For unknown schemes urlib is used with UTF-8 encoding. - .. _Requests: http://pypi.python.org/pypi/requests/ """ scheme = urlparse.urlsplit(uri).scheme if scheme in handlers: result = handlers[scheme](uri) - elif scheme in ['http', 'https']: - result = requests.get(uri).json() else: - result = json.loads(urlopen(uri).read().decode('utf-8')) + req = urlopen(uri) + encoding = req.info().get_content_charset() or 'utf-8' + result = json.loads(req.read().decode(encoding),) return result diff --git a/setup.py b/setup.py index 5a9c478..b9d1c39 100644 --- a/setup.py +++ b/setup.py @@ -20,10 +20,6 @@ name='fastjsonschema', version=VERSION, packages=['fastjsonschema'], - - install_requires=[ - 'requests', - ], extras_require={ 'devel': [ 'colorama', diff --git a/tests/json_schema/utils.py b/tests/json_schema/utils.py index 6c1b686..d6a91a5 100644 --- a/tests/json_schema/utils.py +++ b/tests/json_schema/utils.py @@ -2,7 +2,7 @@ from pathlib import Path import pytest -import requests +from urllib.request import urlopen from fastjsonschema import RefResolver, JsonSchemaException, compile, _get_code_generator_class @@ -26,7 +26,9 @@ def remotes_handler(uri): if uri in REMOTES: return REMOTES[uri] - return requests.get(uri).json() + req = urlopen(uri) + encoding = req.info().get_content_charset() or 'utf-8' + return json.loads(req.read().decode(encoding),) def resolve_param_values_and_ids(schema_version, suite_dir, ignored_suite_files=[], ignore_tests=[]): From a94b8324999d63336c0d7f24583cd7191ec409c1 Mon Sep 17 00:00:00 2001 From: remorses Date: Sun, 3 Mar 2019 12:37:44 +0100 Subject: [PATCH 142/201] removed space --- fastjsonschema/ref_resolver.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index 803c525..b3367cc 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -48,8 +48,6 @@ def resolve_remote(uri, handlers): urllib library is used to fetch requests from the remote ``uri`` if handlers does notdefine otherwise. - - """ scheme = urlparse.urlsplit(uri).scheme if scheme in handlers: From 0f7bd0e6d901e634e208295ec1bed315bf8ed9df Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 4 Mar 2019 10:30:45 +0100 Subject: [PATCH 143/201] Version 2.9 --- CHANGELOG.txt | 5 +++++ fastjsonschema/version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index ad9bc7c..8f2f3bd 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +=== 2.9 (2019-03-04) + +* Use of urllib instead of requests for smaller memory usage. + + === 2.8 (2019-01-05) * Fix quotes in enum generating invalid code diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index 72efc6b..4a45c12 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.8' +VERSION = '2.9' From b335a40745a9955fbf41c5c4778ea5274a42569d Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 15 Apr 2019 14:02:00 +0200 Subject: [PATCH 144/201] Fixed pattern regexps with a space --- CHANGELOG.txt | 5 +++++ fastjsonschema/draft04.py | 4 ++-- fastjsonschema/version.py | 2 +- tests/test_security.py | 3 +++ tests/test_string.py | 12 ++++++++++++ 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 8f2f3bd..0a9c9a9 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +=== 2.10 (2019-04-15) + +* Fix pattern regexps with a space. + + === 2.9 (2019-03-04) * Use of urllib instead of requests for smaller memory usage. diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 52e83ec..3fc59f9 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -225,7 +225,7 @@ def generate_pattern(self): safe_pattern = pattern.replace('"', '\\"') end_of_string_fixed_pattern = DOLLAR_FINDER.sub(r'\\Z', pattern) self._compile_regexps[pattern] = re.compile(end_of_string_fixed_pattern) - with self.l('if not REGEX_PATTERNS["{}"].search({variable}):', safe_pattern): + with self.l('if not REGEX_PATTERNS[{}].search({variable}):', repr(pattern)): self.l('raise JsonSchemaException("{name} must match pattern {}")', safe_pattern) def generate_format(self): @@ -460,7 +460,7 @@ def generate_pattern_properties(self): with self.l('if {variable}_is_dict:'): self.create_variable_keys() for pattern, definition in self._definition['patternProperties'].items(): - self._compile_regexps['{}'.format(pattern)] = re.compile(pattern) + self._compile_regexps[pattern] = re.compile(pattern) with self.l('for {variable}_key, {variable}_val in {variable}.items():'): for pattern, definition in self._definition['patternProperties'].items(): with self.l('if REGEX_PATTERNS["{}"].search({variable}_key):', pattern): diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index 4a45c12..18e2aef 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.9' +VERSION = '2.10' diff --git a/tests/test_security.py b/tests/test_security.py index 1d325de..335ac3b 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -35,6 +35,9 @@ def test_not_generate_code_from_definition(schema): ({'properties': { 'validate(10)': {'type': 'string'}, }}, {'validate(10)': '10'}), + ({'patternProperties': { + 'validate(10)': {'type': 'string'}, + }}, {'validate(10)': '10'}), ]) def test_generate_code_with_proper_variable_names(asserter, schema, value): asserter({ diff --git a/tests/test_string.py b/tests/test_string.py index 849a1f6..31f457d 100644 --- a/tests/test_string.py +++ b/tests/test_string.py @@ -62,6 +62,18 @@ def test_pattern(asserter, value, expected): 'pattern': '^[ab]*[^ab]+(c{2}|d)$', }, value, expected) + +@pytest.mark.parametrize('pattern', [ + ' ', + '\\x20', +]) +def test_pattern_with_space(asserter, pattern): + asserter({ + 'type': 'string', + 'pattern': pattern, + }, ' ', ' ') + + exc = JsonSchemaException('data must be a valid regex') @pytest.mark.parametrize('value, expected', [ ('[a-z]', '[a-z]'), From 8c38d0f91fa5d928ff629080cdb75ab23f96590f Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 16 Apr 2019 14:44:37 +0200 Subject: [PATCH 145/201] Fixed colliding variables --- CHANGELOG.txt | 5 +++++ fastjsonschema/draft04.py | 8 ++++---- fastjsonschema/version.py | 2 +- tests/test_security.py | 17 +++++++++++++++++ 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 0a9c9a9..8649c0b 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +=== 2.11 (2019-04-16) + +* Fix of additionalProperties (colliding variable names). + + === 2.10 (2019-04-15) * Fix pattern regexps with a space. diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 3fc59f9..344e4b8 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -357,10 +357,10 @@ def generate_items(self): elif isinstance(items_definition, list): for idx, item_definition in enumerate(items_definition): with self.l('if {variable}_len > {}:', idx): - self.l('{variable}_{0} = {variable}[{0}]', idx) + self.l('{variable}__{0} = {variable}[{0}]', idx) self.generate_func_code_block( item_definition, - '{}_{}'.format(self._variable, idx), + '{}__{}'.format(self._variable, idx), '{}[{}]'.format(self._variable_name, idx), ) if isinstance(item_definition, dict) and 'default' in item_definition: @@ -433,10 +433,10 @@ def generate_properties(self): key_name = re.sub(r'($[^a-zA-Z]|[^a-zA-Z0-9])', '', key) with self.l('if "{}" in {variable}_keys:', key): self.l('{variable}_keys.remove("{}")', key) - self.l('{variable}_{0} = {variable}["{1}"]', key_name, key) + self.l('{variable}__{0} = {variable}["{1}"]', key_name, key) self.generate_func_code_block( prop_definition, - '{}_{}'.format(self._variable, key_name), + '{}__{}'.format(self._variable, key_name), '{}.{}'.format(self._variable_name, key), ) if isinstance(prop_definition, dict) and 'default' in prop_definition: diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index 18e2aef..052819d 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.10' +VERSION = '2.11' diff --git a/tests/test_security.py b/tests/test_security.py index 335ac3b..7ee39f1 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -44,3 +44,20 @@ def test_generate_code_with_proper_variable_names(asserter, schema, value): '$schema': 'http://json-schema.org/draft-07/schema', **schema }, value, value) + + +def test_generate_code_without_overriding_variables(asserter): + # We use variable name by property name. In the code is automatically generated + # FOO_keys which could colide with keys parameter. Then the variable is reused and + # for example additionalProperties feature is not working well. We need to make + # sure the name not colide. + value = { + 'keys': [1, 2, 3], + } + asserter({ + 'type': 'object', + 'properties': { + 'keys': {'type': 'array'}, + }, + 'additionalProperties': False, + }, value, value) From 8e8133eeb7cc5382a2f020ca5c3d3e3d08f71c3b Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Fri, 24 May 2019 12:17:38 +0200 Subject: [PATCH 146/201] Fixed clearing variables for properties blocks --- CHANGELOG.txt | 5 +++++ fastjsonschema/draft04.py | 2 ++ fastjsonschema/version.py | 2 +- tests/test_pattern_properties.py | 13 +++++++++++++ 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 8649c0b..233107e 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +=== 2.12 (2019-05-24) + +* Fix of properties (local variable referenced before assignment). + + === 2.11 (2019-04-16) * Fix of additionalProperties (colliding variable names). diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 344e4b8..42e15b2 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -438,6 +438,7 @@ def generate_properties(self): prop_definition, '{}__{}'.format(self._variable, key_name), '{}.{}'.format(self._variable_name, key), + clear_variables=True, ) if isinstance(prop_definition, dict) and 'default' in prop_definition: self.l('else: {variable}["{}"] = {}', key, repr(prop_definition['default'])) @@ -470,6 +471,7 @@ def generate_pattern_properties(self): definition, '{}_val'.format(self._variable), '{}.{{{}_key}}'.format(self._variable_name, self._variable), + clear_variables=True, ) def generate_additional_properties(self): diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index 052819d..7ea7455 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.11' +VERSION = '2.12' diff --git a/tests/test_pattern_properties.py b/tests/test_pattern_properties.py index 826b190..450d86d 100644 --- a/tests/test_pattern_properties.py +++ b/tests/test_pattern_properties.py @@ -24,3 +24,16 @@ def test_dont_override_variable_names(asserter): }, }, }, value, value) + + +def test_clear_variables(asserter): + value = { + 'bar': {'baz': 'foo'} + } + asserter({ + 'type': 'object', + 'patternProperties': { + 'foo': {'type': 'object', 'required': ['baz']}, + 'bar': {'type': 'object', 'required': ['baz']} + } + }, value, value) From d98ea82a2a309d99d63b83b3666e19b6fc37299e Mon Sep 17 00:00:00 2001 From: David Majda Date: Thu, 6 Jun 2019 16:50:49 +0200 Subject: [PATCH 147/201] Makefile: Run pytest with "-W default" The "-W default" option activates some warnings that are ignored by default. See the documentation: https://docs.python.org/3/library/warnings.html#default-warning-filter --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9994bce..e04874f 100644 --- a/Makefile +++ b/Makefile @@ -30,13 +30,14 @@ jsonschemasuitcases: git submodule update test: venv jsonschemasuitcases - ${PYTHON} -m pytest --benchmark-skip + ${PYTHON} -m pytest -W default --benchmark-skip test-lf: venv jsonschemasuitcases - ${PYTHON} -m pytest --benchmark-skip --last-failed + ${PYTHON} -m pytest -W default --benchmark-skip --last-failed # Call make benchmark-save before change and then make benchmark to compare results. benchmark: venv jsonschemasuitcases ${PYTHON} -m pytest \ + -W default \ --benchmark-only \ --benchmark-sort=name \ --benchmark-group-by=fullfunc \ @@ -45,6 +46,7 @@ benchmark: venv jsonschemasuitcases --benchmark-compare-fail='min:5%' benchmark-save: venv jsonschemasuitcases ${PYTHON} -m pytest \ + -W default \ --benchmark-only \ --benchmark-sort=name \ --benchmark-group-by=fullfunc \ From 1f5685a38d3915532d866dd99ca474f76254294d Mon Sep 17 00:00:00 2001 From: David Majda Date: Thu, 6 Jun 2019 17:13:09 +0200 Subject: [PATCH 148/201] Add escaping in exception message for "pattern" Missing escaping caused warnings like this in Python 3.7: :6: DeprecationWarning: invalid escape sequence \d --- fastjsonschema/draft04.py | 2 +- tests/test_string.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 42e15b2..252bbee 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -222,7 +222,7 @@ def generate_max_length(self): def generate_pattern(self): with self.l('if isinstance({variable}, str):'): pattern = self._definition['pattern'] - safe_pattern = pattern.replace('"', '\\"') + safe_pattern = pattern.replace('\\', '\\\\').replace('"', '\\"') end_of_string_fixed_pattern = DOLLAR_FINDER.sub(r'\\Z', pattern) self._compile_regexps[pattern] = re.compile(end_of_string_fixed_pattern) with self.l('if not REGEX_PATTERNS[{}].search({variable}):', repr(pattern)): diff --git a/tests/test_string.py b/tests/test_string.py index 31f457d..7f2e5cd 100644 --- a/tests/test_string.py +++ b/tests/test_string.py @@ -74,6 +74,16 @@ def test_pattern_with_space(asserter, pattern): }, ' ', ' ') +def test_pattern_with_escape_no_warnings(asserter): + with pytest.warns(None) as record: + asserter({ + 'type': 'string', + 'pattern': '\\s' + }, ' ', ' ') + + assert len(record) == 0 + + exc = JsonSchemaException('data must be a valid regex') @pytest.mark.parametrize('value, expected', [ ('[a-z]', '[a-z]'), From e84fa021f93e9ea69a28b5c7c5c1647630c54ebf Mon Sep 17 00:00:00 2001 From: David Majda Date: Thu, 6 Jun 2019 17:30:47 +0200 Subject: [PATCH 149/201] Add escaping in generated code for "patternProperties" Missing escaping caused incorrect matching and also caused warnings like this in Python 3.7: :8: DeprecationWarning: invalid escape sequence \d --- fastjsonschema/draft04.py | 2 +- tests/test_pattern_properties.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 252bbee..64ee719 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -464,7 +464,7 @@ def generate_pattern_properties(self): self._compile_regexps[pattern] = re.compile(pattern) with self.l('for {variable}_key, {variable}_val in {variable}.items():'): for pattern, definition in self._definition['patternProperties'].items(): - with self.l('if REGEX_PATTERNS["{}"].search({variable}_key):', pattern): + with self.l('if REGEX_PATTERNS[{}].search({variable}_key):', repr(pattern)): with self.l('if {variable}_key in {variable}_keys:'): self.l('{variable}_keys.remove({variable}_key)') self.generate_func_code_block( diff --git a/tests/test_pattern_properties.py b/tests/test_pattern_properties.py index 450d86d..ae86946 100644 --- a/tests/test_pattern_properties.py +++ b/tests/test_pattern_properties.py @@ -1,3 +1,6 @@ +import pytest + + def test_dont_override_variable_names(asserter): value = { 'foo:bar': { @@ -37,3 +40,31 @@ def test_clear_variables(asserter): 'bar': {'type': 'object', 'required': ['baz']} } }, value, value) + + +def test_pattern_with_escape(asserter): + value = { + '\\n': {} + } + asserter({ + 'type': 'object', + 'patternProperties': { + '\\\\n': {'type': 'object'} + } + }, value, value) + + +def test_pattern_with_escape_no_warnings(asserter): + value = { + 'bar': {} + } + + with pytest.warns(None) as record: + asserter({ + 'type': 'object', + 'patternProperties': { + '\\w+': {'type': 'object'} + } + }, value, value) + + assert len(record) == 0 From afbd9e93f4767266d5ff2c21b012fb162945854e Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 10 Jun 2019 10:57:48 +0200 Subject: [PATCH 150/201] Updated JSON test suites --- JSON-Schema-Test-Suite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JSON-Schema-Test-Suite b/JSON-Schema-Test-Suite index 71843cb..1bd999a 160000 --- a/JSON-Schema-Test-Suite +++ b/JSON-Schema-Test-Suite @@ -1 +1 @@ -Subproject commit 71843cbbd4be3194ee83253b402656550c8c0522 +Subproject commit 1bd999ac16bd8d3fdb5c44ef13a0759aefb4ab73 From c4b6338e4829dade9f675788ba8ad4629b61c189 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 10 Jun 2019 10:59:13 +0200 Subject: [PATCH 151/201] Fix escaping --- fastjsonschema/draft04.py | 21 ++++++++++----------- fastjsonschema/generator.py | 14 +++++++++++++- fastjsonschema/ref_resolver.py | 2 +- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 64ee719..c3d2101 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -98,8 +98,7 @@ def generate_enum(self): if not isinstance(enum, (list, tuple)): raise JsonSchemaDefinitionException('enum must be an array') with self.l('if {variable} not in {enum}:'): - enum = str(enum).replace('"', '\\"') - self.l('raise JsonSchemaException("{name} must be one of {}")', enum) + self.l('raise JsonSchemaException("{name} must be one of {}")', self.e(enum)) def generate_all_of(self): """ @@ -410,7 +409,7 @@ def generate_required(self): raise JsonSchemaDefinitionException('required must be an array') self.create_variable_with_length() with self.l('if not all(prop in {variable} for prop in {required}):'): - self.l('raise JsonSchemaException("{name} must contain {required} properties")') + self.l('raise JsonSchemaException("{name} must contain {} properties")', self.e(self._definition['required'])) def generate_properties(self): """ @@ -431,17 +430,17 @@ def generate_properties(self): self.create_variable_keys() for key, prop_definition in self._definition['properties'].items(): key_name = re.sub(r'($[^a-zA-Z]|[^a-zA-Z0-9])', '', key) - with self.l('if "{}" in {variable}_keys:', key): - self.l('{variable}_keys.remove("{}")', key) - self.l('{variable}__{0} = {variable}["{1}"]', key_name, key) + with self.l('if "{}" in {variable}_keys:', self.e(key)): + self.l('{variable}_keys.remove("{}")', self.e(key)) + self.l('{variable}__{0} = {variable}["{1}"]', key_name, self.e(key)) self.generate_func_code_block( prop_definition, '{}__{}'.format(self._variable, key_name), - '{}.{}'.format(self._variable_name, key), + '{}.{}'.format(self._variable_name, self.e(key)), clear_variables=True, ) if isinstance(prop_definition, dict) and 'default' in prop_definition: - self.l('else: {variable}["{}"] = {}', key, repr(prop_definition['default'])) + self.l('else: {variable}["{}"] = {}', self.e(key), repr(prop_definition['default'])) def generate_pattern_properties(self): """ @@ -532,12 +531,12 @@ def generate_dependencies(self): for key, values in self._definition["dependencies"].items(): if values == [] or values is True: continue - with self.l('if "{}" in {variable}_keys:', key): + with self.l('if "{}" in {variable}_keys:', self.e(key)): if values is False: self.l('raise JsonSchemaException("{} in {name} must not be there")', key) elif isinstance(values, list): for value in values: - with self.l('if "{}" not in {variable}_keys:', value): - self.l('raise JsonSchemaException("{name} missing dependency {} for {}")', value, key) + with self.l('if "{}" not in {variable}_keys:', self.e(value)): + self.l('raise JsonSchemaException("{name} missing dependency {} for {}")', self.e(value), self.e(key)) else: self.generate_func_code_block(values, self._variable, self._variable_name, clear_variables=True) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 28b9480..2c2ccda 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -220,7 +220,19 @@ def l(self, line, *args, **kwds): name=name, **kwds ) - self._code.append(spaces + line.format(*args, **context)) + line = line.format(*args, **context) + line = line.replace('\n', '\\n').replace('\r', '\\r') + self._code.append(spaces + line) + + def e(self, string): + """ + Short-cut of escape. Used for inserting user values into a string message. + + .. code-block:: python + + self.l('raise JsonSchemaException("Variable: {}")', self.e(variable)) + """ + return str(string).replace('"', '\\"') def create_variable_with_length(self): """ diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index b3367cc..d82973a 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -134,7 +134,7 @@ def get_scope_name(self): """ Get current scope and return it as a valid function name. """ - name = 'validate_' + unquote(self.resolution_scope).replace('~1', '_').replace('~0', '_') + name = 'validate_' + unquote(self.resolution_scope).replace('~1', '_').replace('~0', '_').replace('"', '') name = re.sub(r'[:/#\.\-\%]', '_', name) name = name.lower().rstrip('_') return name From e73b14e4518c08e5bc304bfa142545d6bafaccc7 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 10 Jun 2019 10:59:26 +0200 Subject: [PATCH 152/201] Fix date-time regexp --- fastjsonschema/draft04.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index c3d2101..3cb43fe 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -25,7 +25,7 @@ class CodeGeneratorDraft04(CodeGenerator): # vs. 9 ms with a regex! Other modules are also unefective or not available in standard # library. Some regexps are not 100% precise but good enough, fast and without dependencies. FORMAT_REGEXS = { - 'date-time': r'^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d(?:\.\d+)?(?:[+-][0-2]\d:[0-5]\d|Z)?\Z', + 'date-time': r'^\d{4}-[01]\d-[0-3]\d(t|T)[0-2]\d:[0-5]\d:[0-5]\d(?:\.\d+)?(?:[+-][0-2]\d:[0-5]\d|z|Z)?\Z', 'email': r'^[^@]+@[^@]+\.[^@]+\Z', 'hostname': r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{0,61}[A-Za-z0-9])\Z', 'ipv4': r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\Z', From cbd017f2fd8b157dc9e1347a21a3fa5855aafb08 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 10 Jun 2019 10:59:45 +0200 Subject: [PATCH 153/201] Ignore IRI tests --- tests/json_schema/test_draft07.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/json_schema/test_draft07.py b/tests/json_schema/test_draft07.py index b720caf..bc7b291 100644 --- a/tests/json_schema/test_draft07.py +++ b/tests/json_schema/test_draft07.py @@ -11,6 +11,7 @@ def pytest_generate_tests(metafunc): # Optional. 'ecmascript-regex.json', 'idn-hostname.json', + 'iri.json', ], ) metafunc.parametrize(['schema_version', 'schema', 'data', 'is_valid'], param_values, ids=param_ids) From cc80a08d0ddd12a15b62a384c84d830fe9a37968 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 10 Jun 2019 11:00:30 +0200 Subject: [PATCH 154/201] Updated AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 15a1ef2..721bda3 100644 --- a/AUTHORS +++ b/AUTHORS @@ -8,3 +8,4 @@ bcaller Frederik Petersen Guillaume Desvé Kris Molendyke +David Majda From 26187efaace40268ea924da9f18552fb090960c3 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 10 Jun 2019 11:04:43 +0200 Subject: [PATCH 155/201] Version 2.13 --- CHANGELOG.txt | 8 ++++++++ fastjsonschema/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 233107e..281e850 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,11 @@ +=== 2.13 (2019-06-10) + +* Resolved Python 3.7 warnings +* Updated JSON Schema test suites + * Fix of date-time regexp (allow small T and Z). + * Fix escaping (proper handling of \n, \r or " everywhere). + + === 2.12 (2019-05-24) * Fix of properties (local variable referenced before assignment). diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index 7ea7455..dbf4c44 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.12' +VERSION = '2.13' From 2cd86e438e5a2fedcab408bde61aa3e3057ca46f Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 10 Jun 2019 11:08:25 +0200 Subject: [PATCH 156/201] Python 3.7 and 3.8 is supported --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index b9d1c39..e7720d1 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,8 @@ "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: Implementation :: CPython", 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', From b1ba8065272353c171d5374f2ee6e1b7c63a68b2 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 18 Jun 2019 18:04:19 +0200 Subject: [PATCH 157/201] Only RFC3339 is possible for date-time format (time zone is required) --- fastjsonschema/draft04.py | 2 +- tests/test_format.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 3cb43fe..360c409 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -25,7 +25,7 @@ class CodeGeneratorDraft04(CodeGenerator): # vs. 9 ms with a regex! Other modules are also unefective or not available in standard # library. Some regexps are not 100% precise but good enough, fast and without dependencies. FORMAT_REGEXS = { - 'date-time': r'^\d{4}-[01]\d-[0-3]\d(t|T)[0-2]\d:[0-5]\d:[0-5]\d(?:\.\d+)?(?:[+-][0-2]\d:[0-5]\d|z|Z)?\Z', + 'date-time': r'^\d{4}-[01]\d-[0-3]\d(t|T)[0-2]\d:[0-5]\d:[0-5]\d(?:\.\d+)?(?:[+-][0-2]\d:[0-5]\d|z|Z)\Z', 'email': r'^[^@]+@[^@]+\.[^@]+\Z', 'hostname': r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{0,61}[A-Za-z0-9])\Z', 'ipv4': r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\Z', diff --git a/tests/test_format.py b/tests/test_format.py index b7171e4..19b551b 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -7,6 +7,7 @@ @pytest.mark.parametrize('value, expected', [ ('', exc), ('bla', exc), + ('2018-02-05T14:17:10.00', exc), ('2018-02-05T14:17:10.00Z\n', exc), ('2018-02-05T14:17:10.00Z', '2018-02-05T14:17:10.00Z'), ('2018-02-05T14:17:10Z', '2018-02-05T14:17:10Z'), From 807d3c4519bf91f0b9cec82e144124ccf4c26bc5 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 18 Jun 2019 18:06:51 +0200 Subject: [PATCH 158/201] Changelog --- CHANGELOG.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 281e850..5cb01f1 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +=== 2.14 (unreleased) + +* Fix of date-time regexp (time zone is required by RFC 3339) + + === 2.13 (2019-06-10) * Resolved Python 3.7 warnings From 3eca153d50e54a3d344b594d5766f32fc68789eb Mon Sep 17 00:00:00 2001 From: Yohann Streibel Date: Fri, 26 Jul 2019 11:54:45 +0200 Subject: [PATCH 159/201] Add handlers to validate methode and introduce format_checkers --- fastjsonschema/__init__.py | 12 ++++++------ fastjsonschema/draft04.py | 11 +++++++++-- fastjsonschema/draft06.py | 4 ++-- fastjsonschema/draft07.py | 4 ++-- fastjsonschema/generator.py | 6 +++++- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 3c15e44..62b2e03 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -84,7 +84,7 @@ __all__ = ('VERSION', 'JsonSchemaException', 'JsonSchemaDefinitionException', 'validate', 'compile', 'compile_to_code') -def validate(definition, data): +def validate(definition, data, handlers={}, format_checkers={}): """ Validation function for lazy programmers or for use cases, when you need to call validation only once, so you do not have to compile it first. @@ -100,11 +100,11 @@ def validate(definition, data): Preffered is to use :any:`compile` function. """ - return compile(definition)(data) + return compile(definition, handlers, format_checkers)(data) # pylint: disable=redefined-builtin,dangerous-default-value,exec-used -def compile(definition, handlers={}): +def compile(definition, handlers={}, format_checkers={}): """ Generates validation function for validating JSON schema passed in ``definition``. Example: @@ -150,7 +150,7 @@ def compile(definition, handlers={}): Exception :any:`JsonSchemaException` is raised from generated funtion when validation fails (data do not follow the definition). """ - resolver, code_generator = _factory(definition, handlers) + resolver, code_generator = _factory(definition, handlers, format_checkers) global_state = code_generator.global_state # Do not pass local state so it can recursively call itself. exec(code_generator.func_code, global_state) @@ -189,9 +189,9 @@ def compile_to_code(definition, handlers={}): ) -def _factory(definition, handlers): +def _factory(definition, handlers, format_checkers={}): resolver = RefResolver.from_schema(definition, handlers=handlers) - code_generator = _get_code_generator_class(definition)(definition, resolver=resolver) + code_generator = _get_code_generator_class(definition)(definition, resolver=resolver, format_checkers=format_checkers) return resolver, code_generator diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 360c409..d294bd6 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -33,8 +33,8 @@ class CodeGeneratorDraft04(CodeGenerator): 'uri': r'^\w+:(\/?\/?)[^\s]+\Z', } - def __init__(self, definition, resolver=None): - super().__init__(definition, resolver) + def __init__(self, definition, resolver=None, format_checkers={}): + super().__init__(definition, resolver, format_checkers) self._json_keywords_to_function.update(( ('type', self.generate_type), ('enum', self.generate_enum), @@ -248,6 +248,13 @@ def generate_format(self): self.l('re.compile({variable})') with self.l('except Exception:'): self.l('raise JsonSchemaException("{name} must be a valid regex")') + + # format checking from format_checker + if format_ in self._format_checkers: + with self.l('try:'): + self.l('format_checkers[{}]({variable})', repr(format_)) + with self.l('except Exception as e:'): + self.l('raise JsonSchemaException("{name} is not a valid {variable}") from e') else: self.l('pass') diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py index 5ab86f2..bca39bb 100644 --- a/fastjsonschema/draft06.py +++ b/fastjsonschema/draft06.py @@ -15,8 +15,8 @@ class CodeGeneratorDraft06(CodeGeneratorDraft04): ), }) - def __init__(self, definition, resolver=None): - super().__init__(definition, resolver) + def __init__(self, definition, resolver=None, format_checkers={}): + super().__init__(definition, resolver, format_checkers) self._json_keywords_to_function.update(( ('exclusiveMinimum', self.generate_exclusive_minimum), ('exclusiveMaximum', self.generate_exclusive_maximum), diff --git a/fastjsonschema/draft07.py b/fastjsonschema/draft07.py index d553d3b..c0b968c 100644 --- a/fastjsonschema/draft07.py +++ b/fastjsonschema/draft07.py @@ -17,8 +17,8 @@ class CodeGeneratorDraft07(CodeGeneratorDraft06): ), }) - def __init__(self, definition, resolver=None): - super().__init__(definition, resolver) + def __init__(self, definition, resolver=None, format_checkers={}): + super().__init__(definition, resolver, format_checkers) # pylint: disable=duplicate-code self._json_keywords_to_function.update(( ('if', self.generate_if_then_else), diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 2c2ccda..85048e1 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -28,7 +28,7 @@ class CodeGenerator: INDENT = 4 # spaces - def __init__(self, definition, resolver=None): + def __init__(self, definition, resolver=None, format_checkers={}): self._code = [] self._compile_regexps = {} @@ -48,6 +48,9 @@ def __init__(self, definition, resolver=None): if resolver is None: resolver = RefResolver.from_schema(definition) self._resolver = resolver + + self._format_checkers = format_checkers + # add main function to `self._needed_validation_functions` self._needed_validation_functions[self._resolver.get_uri()] = self._resolver.get_scope_name() @@ -75,6 +78,7 @@ def global_state(self): REGEX_PATTERNS=self._compile_regexps, re=re, JsonSchemaException=JsonSchemaException, + format_checkers=self._format_checkers ) @property From 78dd60e6091f09daf60667c30d93cee8ee6b7cac Mon Sep 17 00:00:00 2001 From: Yohann Streibel Date: Tue, 20 Aug 2019 09:47:10 +0200 Subject: [PATCH 160/201] renaming format_chechers to formats and add str, re.pattern --- fastjsonschema/__init__.py | 12 ++++++------ fastjsonschema/draft04.py | 10 +++++----- fastjsonschema/draft06.py | 4 ++-- fastjsonschema/draft07.py | 4 ++-- fastjsonschema/generator.py | 12 +++++++++--- tests/conftest.py | 4 ++-- tests/test_format.py | 33 +++++++++++++++++++++++++++++++++ 7 files changed, 59 insertions(+), 20 deletions(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 62b2e03..d464a02 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -84,7 +84,7 @@ __all__ = ('VERSION', 'JsonSchemaException', 'JsonSchemaDefinitionException', 'validate', 'compile', 'compile_to_code') -def validate(definition, data, handlers={}, format_checkers={}): +def validate(definition, data, handlers={}, formats={}): """ Validation function for lazy programmers or for use cases, when you need to call validation only once, so you do not have to compile it first. @@ -100,11 +100,11 @@ def validate(definition, data, handlers={}, format_checkers={}): Preffered is to use :any:`compile` function. """ - return compile(definition, handlers, format_checkers)(data) + return compile(definition, handlers, formats)(data) # pylint: disable=redefined-builtin,dangerous-default-value,exec-used -def compile(definition, handlers={}, format_checkers={}): +def compile(definition, handlers={}, formats={}): """ Generates validation function for validating JSON schema passed in ``definition``. Example: @@ -150,7 +150,7 @@ def compile(definition, handlers={}, format_checkers={}): Exception :any:`JsonSchemaException` is raised from generated funtion when validation fails (data do not follow the definition). """ - resolver, code_generator = _factory(definition, handlers, format_checkers) + resolver, code_generator = _factory(definition, handlers, formats) global_state = code_generator.global_state # Do not pass local state so it can recursively call itself. exec(code_generator.func_code, global_state) @@ -189,9 +189,9 @@ def compile_to_code(definition, handlers={}): ) -def _factory(definition, handlers, format_checkers={}): +def _factory(definition, handlers, formats={}): resolver = RefResolver.from_schema(definition, handlers=handlers) - code_generator = _get_code_generator_class(definition)(definition, resolver=resolver, format_checkers=format_checkers) + code_generator = _get_code_generator_class(definition)(definition, resolver=resolver, formats=formats) return resolver, code_generator diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index d294bd6..9fc3aa1 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -33,8 +33,8 @@ class CodeGeneratorDraft04(CodeGenerator): 'uri': r'^\w+:(\/?\/?)[^\s]+\Z', } - def __init__(self, definition, resolver=None, format_checkers={}): - super().__init__(definition, resolver, format_checkers) + def __init__(self, definition, resolver=None, formats={}): + super().__init__(definition, resolver, formats) self._json_keywords_to_function.update(( ('type', self.generate_type), ('enum', self.generate_enum), @@ -249,10 +249,10 @@ def generate_format(self): with self.l('except Exception:'): self.l('raise JsonSchemaException("{name} must be a valid regex")') - # format checking from format_checker - if format_ in self._format_checkers: + # format checking from format callable + if format_ in self._formats: with self.l('try:'): - self.l('format_checkers[{}]({variable})', repr(format_)) + self.l('formats[{}]({variable})', repr(format_)) with self.l('except Exception as e:'): self.l('raise JsonSchemaException("{name} is not a valid {variable}") from e') else: diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py index bca39bb..e8521a6 100644 --- a/fastjsonschema/draft06.py +++ b/fastjsonschema/draft06.py @@ -15,8 +15,8 @@ class CodeGeneratorDraft06(CodeGeneratorDraft04): ), }) - def __init__(self, definition, resolver=None, format_checkers={}): - super().__init__(definition, resolver, format_checkers) + def __init__(self, definition, resolver=None, formats={}): + super().__init__(definition, resolver, formats) self._json_keywords_to_function.update(( ('exclusiveMinimum', self.generate_exclusive_minimum), ('exclusiveMaximum', self.generate_exclusive_maximum), diff --git a/fastjsonschema/draft07.py b/fastjsonschema/draft07.py index c0b968c..3101328 100644 --- a/fastjsonschema/draft07.py +++ b/fastjsonschema/draft07.py @@ -17,8 +17,8 @@ class CodeGeneratorDraft07(CodeGeneratorDraft06): ), }) - def __init__(self, definition, resolver=None, format_checkers={}): - super().__init__(definition, resolver, format_checkers) + def __init__(self, definition, resolver=None, formats={}): + super().__init__(definition, resolver, formats) # pylint: disable=duplicate-code self._json_keywords_to_function.update(( ('if', self.generate_if_then_else), diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 85048e1..b4ec546 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -28,7 +28,7 @@ class CodeGenerator: INDENT = 4 # spaces - def __init__(self, definition, resolver=None, format_checkers={}): + def __init__(self, definition, resolver=None, formats={}): self._code = [] self._compile_regexps = {} @@ -49,7 +49,13 @@ def __init__(self, definition, resolver=None, format_checkers={}): resolver = RefResolver.from_schema(definition) self._resolver = resolver - self._format_checkers = format_checkers + # Initialize formats values as callable + for key, value in formats.items(): + if isinstance(value, str): + formats[key] = lambda data: re.match(value, data) + elif isinstance(value, re.Pattern): + formats[key] = lambda data: value.match(data) + self._formats = formats # add main function to `self._needed_validation_functions` self._needed_validation_functions[self._resolver.get_uri()] = self._resolver.get_scope_name() @@ -78,7 +84,7 @@ def global_state(self): REGEX_PATTERNS=self._compile_regexps, re=re, JsonSchemaException=JsonSchemaException, - format_checkers=self._format_checkers + formats=self._formats ) @property diff --git a/tests/conftest.py b/tests/conftest.py index 8f805cf..0fb695a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,7 +15,7 @@ @pytest.fixture def asserter(): - def f(definition, value, expected): + def f(definition, value, expected, formats={}): # When test fails, it will show up code. code_generator = CodeGeneratorDraft07(definition) print(code_generator.func_code) @@ -24,7 +24,7 @@ def f(definition, value, expected): # By default old tests are written for draft-04. definition.setdefault('$schema', 'http://json-schema.org/draft-04/schema') - validator = compile(definition) + validator = compile(definition, formats=formats) if isinstance(expected, JsonSchemaException): with pytest.raises(JsonSchemaException) as exc: validator(value) diff --git a/tests/test_format.py b/tests/test_format.py index 19b551b..58a5cd2 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -1,5 +1,11 @@ +from builtins import ValueError + +import datetime + import pytest +import re + from fastjsonschema import JsonSchemaException @@ -31,3 +37,30 @@ def test_datetime(asserter, value, expected): ]) def test_hostname(asserter, value, expected): asserter({'type': 'string', 'format': 'hostname'}, value, expected) + + +def __special_timestamp_format_checker(date_string: str) -> bool: + dt = datetime.datetime.fromisoformat(date_string).replace(tzinfo=datetime.timezone.utc) + dt_now = datetime.datetime.now(datetime.timezone.utc) + if dt > dt_now: + raise ValueError(f"{date_string} is in the future") + return True + + +pattern = "^(19|20)[0-9][0-9]-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01]) (0[0-9]|1[0-9]|2[0-3]):([0-5][0-9]):([" \ + "0-5][0-9])\\.[0-9]{6}$ " + + +exc = JsonSchemaException('data is not a valid data') +@pytest.mark.parametrize('value, expected, formats', [ + ('', exc, {"special-timestamp": __special_timestamp_format_checker}), + ('bla', exc, {"special-timestamp": __special_timestamp_format_checker}), + ('2018-02-05T14:17:10.00', exc, {"special-timestamp": __special_timestamp_format_checker}), + ('2019-03-12 13:08:03.001000\n', exc, {"special-timestamp": __special_timestamp_format_checker}), + ('2999-03-12 13:08:03.001000', '2999-03-12 13:08:03.001000', exc, {"special-timestamp": __special_timestamp_format_checker}), + ('2019-03-12 13:08:03.001000', '2019-03-12 13:08:03.001000', {"special-timestamp": __special_timestamp_format_checker}), + ('2019-03-12 13:08:03.001000', '2019-03-12 13:08:03.001000', {"special-timestamp": pattern}), + ('2019-03-12 13:08:03.001000', '2019-03-12 13:08:03.001000', {"special-timestamp": re.compile(pattern)}), +]) +def test_special_datetime(asserter, value, expected, formats): + asserter({'type': 'string', 'format': 'special-timestamp'}, value, expected, formats=formats) From b21744f8a5eac00eaed983a944d9dbb19d26429b Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 24 Sep 2019 23:40:31 +0200 Subject: [PATCH 161/201] Better implementation of custom formats --- CHANGELOG.txt | 1 + fastjsonschema/__init__.py | 15 ++++++++++++-- fastjsonschema/draft04.py | 31 ++++++++++++++++++---------- fastjsonschema/generator.py | 11 +--------- tests/conftest.py | 2 +- tests/test_format.py | 40 ++++++++++++++++--------------------- 6 files changed, 53 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 5cb01f1..9ed4a8a 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,5 +1,6 @@ === 2.14 (unreleased) +* Possibility to pass custom formats * Fix of date-time regexp (time zone is required by RFC 3339) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index d464a02..2b4b7b2 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -144,6 +144,17 @@ def compile(definition, handlers={}, formats={}): You can pass mapping from URI to function that should be used to retrieve remote schemes used in your ``definition`` in parameter ``handlers``. + Also, you can pass mapping for custom formats. Key is the name of your + formatter and value can be regular expression which will be compiled or + callback returning `bool` (or you can raise your own exception). + + .. code-block:: python + + validate = fastjsonschema.compile(definition, formats={ + 'foo': r'foo|bar', + 'bar': lambda value: value in ('foo', 'bar'), + }) + Exception :any:`JsonSchemaDefinitionException` is raised when generating the code fails (bad definition). @@ -158,7 +169,7 @@ def compile(definition, handlers={}, formats={}): # pylint: disable=dangerous-default-value -def compile_to_code(definition, handlers={}): +def compile_to_code(definition, handlers={}, formats={}): """ Generates validation code for validating JSON schema passed in ``definition``. Example: @@ -181,7 +192,7 @@ def compile_to_code(definition, handlers={}): Exception :any:`JsonSchemaDefinitionException` is raised when generating the code fails (bad definition). """ - _, code_generator = _factory(definition, handlers) + _, code_generator = _factory(definition, handlers, formats) return ( 'VERSION = "' + VERSION + '"\n' + code_generator.global_state_code + '\n' + diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 9fc3aa1..ea8e876 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -34,7 +34,8 @@ class CodeGeneratorDraft04(CodeGenerator): } def __init__(self, definition, resolver=None, formats={}): - super().__init__(definition, resolver, formats) + super().__init__(definition, resolver) + self._custom_formats = formats self._json_keywords_to_function.update(( ('type', self.generate_type), ('enum', self.generate_enum), @@ -62,6 +63,12 @@ def __init__(self, definition, resolver=None, formats={}): ('dependencies', self.generate_dependencies), )) + @property + def global_state(self): + res = super().global_state + res['custom_formats'] = self._custom_formats + return res + def generate_type(self): """ Validation of type. Can be one type or list of types. @@ -239,24 +246,26 @@ def generate_format(self): """ with self.l('if isinstance({variable}, str):'): format_ = self._definition['format'] - if format_ in self.FORMAT_REGEXS: + # Checking custom formats - user is allowed to override default formats. + if format_ in self._custom_formats: + custom_format = self._custom_formats[format_] + if isinstance(custom_format, str): + self._generate_format(format_, format_ + '_re_pattern', custom_format) + else: + with self.l('if not custom_formats["{}"]({variable}):', format_): + self.l('raise JsonSchemaException("{name} must be {}")', format_) + elif format_ in self.FORMAT_REGEXS: format_regex = self.FORMAT_REGEXS[format_] self._generate_format(format_, format_ + '_re_pattern', format_regex) - # format regex is used only in meta schemas + # Format regex is used only in meta schemas. elif format_ == 'regex': with self.l('try:'): self.l('re.compile({variable})') with self.l('except Exception:'): self.l('raise JsonSchemaException("{name} must be a valid regex")') - - # format checking from format callable - if format_ in self._formats: - with self.l('try:'): - self.l('formats[{}]({variable})', repr(format_)) - with self.l('except Exception as e:'): - self.l('raise JsonSchemaException("{name} is not a valid {variable}") from e') else: - self.l('pass') + raise JsonSchemaDefinitionException('Undefined format %s'.format(format_)) + def _generate_format(self, format_name, regexp_name, regexp): if self._definition['format'] == format_name: diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index b4ec546..8d4c528 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -28,7 +28,7 @@ class CodeGenerator: INDENT = 4 # spaces - def __init__(self, definition, resolver=None, formats={}): + def __init__(self, definition, resolver=None): self._code = [] self._compile_regexps = {} @@ -49,14 +49,6 @@ def __init__(self, definition, resolver=None, formats={}): resolver = RefResolver.from_schema(definition) self._resolver = resolver - # Initialize formats values as callable - for key, value in formats.items(): - if isinstance(value, str): - formats[key] = lambda data: re.match(value, data) - elif isinstance(value, re.Pattern): - formats[key] = lambda data: value.match(data) - self._formats = formats - # add main function to `self._needed_validation_functions` self._needed_validation_functions[self._resolver.get_uri()] = self._resolver.get_scope_name() @@ -84,7 +76,6 @@ def global_state(self): REGEX_PATTERNS=self._compile_regexps, re=re, JsonSchemaException=JsonSchemaException, - formats=self._formats ) @property diff --git a/tests/conftest.py b/tests/conftest.py index 0fb695a..70d1596 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,7 +17,7 @@ def asserter(): def f(definition, value, expected, formats={}): # When test fails, it will show up code. - code_generator = CodeGeneratorDraft07(definition) + code_generator = CodeGeneratorDraft07(definition, formats=formats) print(code_generator.func_code) pprint(code_generator.global_state) diff --git a/tests/test_format.py b/tests/test_format.py index 58a5cd2..a41a5d5 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -39,28 +39,22 @@ def test_hostname(asserter, value, expected): asserter({'type': 'string', 'format': 'hostname'}, value, expected) -def __special_timestamp_format_checker(date_string: str) -> bool: - dt = datetime.datetime.fromisoformat(date_string).replace(tzinfo=datetime.timezone.utc) - dt_now = datetime.datetime.now(datetime.timezone.utc) - if dt > dt_now: - raise ValueError(f"{date_string} is in the future") - return True - - -pattern = "^(19|20)[0-9][0-9]-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01]) (0[0-9]|1[0-9]|2[0-3]):([0-5][0-9]):([" \ - "0-5][0-9])\\.[0-9]{6}$ " +exc = JsonSchemaException('data must be custom-format') +@pytest.mark.parametrize('value,expected,custom_format', [ + ('', exc, r'^[ab]$'), + ('', exc, lambda value: value in ('a', 'b')), + ('a', 'a', r'^[ab]$'), + ('a', 'a', lambda value: value in ('a', 'b')), + ('c', exc, r'^[ab]$'), + ('c', exc, lambda value: value in ('a', 'b')), +]) +def test_custom_format(asserter, value, expected, custom_format): + asserter({'format': 'custom-format'}, value, expected, formats={ + 'custom-format': custom_format, + }) -exc = JsonSchemaException('data is not a valid data') -@pytest.mark.parametrize('value, expected, formats', [ - ('', exc, {"special-timestamp": __special_timestamp_format_checker}), - ('bla', exc, {"special-timestamp": __special_timestamp_format_checker}), - ('2018-02-05T14:17:10.00', exc, {"special-timestamp": __special_timestamp_format_checker}), - ('2019-03-12 13:08:03.001000\n', exc, {"special-timestamp": __special_timestamp_format_checker}), - ('2999-03-12 13:08:03.001000', '2999-03-12 13:08:03.001000', exc, {"special-timestamp": __special_timestamp_format_checker}), - ('2019-03-12 13:08:03.001000', '2019-03-12 13:08:03.001000', {"special-timestamp": __special_timestamp_format_checker}), - ('2019-03-12 13:08:03.001000', '2019-03-12 13:08:03.001000', {"special-timestamp": pattern}), - ('2019-03-12 13:08:03.001000', '2019-03-12 13:08:03.001000', {"special-timestamp": re.compile(pattern)}), -]) -def test_special_datetime(asserter, value, expected, formats): - asserter({'type': 'string', 'format': 'special-timestamp'}, value, expected, formats=formats) +def test_custom_format_override(asserter): + asserter({'format': 'date-time'}, 'a', 'a', formats={ + 'date-time': r'^[ab]$', + }) From 4c8e5fc37f576fd66e78be6e34c83a3dc8ef0b99 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 25 Sep 2019 22:57:12 +0200 Subject: [PATCH 162/201] Raise JsonSchemaDefinitionException when definition of property is not valid --- CHANGELOG.txt | 1 + fastjsonschema/draft04.py | 2 ++ tests/test_format.py | 5 +---- tests/test_object.py | 13 +++++++++++-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 9ed4a8a..09415f5 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,7 @@ === 2.14 (unreleased) * Possibility to pass custom formats +* Raise JsonSchemaDefinitionException when definition of property is not valid * Fix of date-time regexp (time zone is required by RFC 3339) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index ea8e876..8faab61 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -446,6 +446,8 @@ def generate_properties(self): self.create_variable_keys() for key, prop_definition in self._definition['properties'].items(): key_name = re.sub(r'($[^a-zA-Z]|[^a-zA-Z0-9])', '', key) + if not isinstance(prop_definition, (dict, bool)): + raise JsonSchemaDefinitionException('{}[{}] must be object'.format(self._variable, key_name)) with self.l('if "{}" in {variable}_keys:', self.e(key)): self.l('{variable}_keys.remove("{}")', self.e(key)) self.l('{variable}__{0} = {variable}["{1}"]', key_name, self.e(key)) diff --git a/tests/test_format.py b/tests/test_format.py index a41a5d5..490d70b 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -1,11 +1,8 @@ -from builtins import ValueError - import datetime +import re import pytest -import re - from fastjsonschema import JsonSchemaException diff --git a/tests/test_object.py b/tests/test_object.py index 51d6c6a..d241f59 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -1,7 +1,7 @@ - import pytest -from fastjsonschema import JsonSchemaException +import fastjsonschema +from fastjsonschema import JsonSchemaDefinitionException, JsonSchemaException exc = JsonSchemaException('data must be object') @@ -73,6 +73,15 @@ def test_properties(asserter, value, expected): }, value, expected) +def test_invalid_properties(asserter): + with pytest.raises(JsonSchemaDefinitionException): + fastjsonschema.compile({ + 'properties': { + 'item': ['wrong'], + }, + }) + + @pytest.mark.parametrize('value, expected', [ ({}, {}), ({'a': 1}, {'a': 1}), From 3252d066a124bc18f214e7fb660634a5f58f20be Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 26 Sep 2019 17:58:51 +0200 Subject: [PATCH 163/201] Fix of uniqueItems when used with other than array type --- CHANGELOG.txt | 1 + fastjsonschema/draft04.py | 8 +++++--- tests/test_array.py | 9 +++++++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 09415f5..0abcf4b 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,5 +1,6 @@ === 2.14 (unreleased) +* Fix of uniqueItems when used with other than array type * Possibility to pass custom formats * Raise JsonSchemaDefinitionException when definition of property is not valid * Fix of date-time regexp (time zone is required by RFC 3339) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 8faab61..aa87008 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -337,9 +337,11 @@ def generate_unique_items(self): >>> timeit.timeit("np.unique(x).size == len(x)", "x=range(100)+range(100); import numpy as np", number=100000) 2.1439831256866455 """ - self.create_variable_with_length() - with self.l('if {variable}_len > len(set(str({variable}_x) for {variable}_x in {variable})):'): - self.l('raise JsonSchemaException("{name} must contain unique items")') + self.create_variable_is_list() + with self.l('if {variable}_is_list:'): + self.create_variable_with_length() + with self.l('if {variable}_len > len(set(str({variable}_x) for {variable}_x in {variable})):'): + self.l('raise JsonSchemaException("{name} must contain unique items")') def generate_items(self): """ diff --git a/tests/test_array.py b/tests/test_array.py index b4934ac..6c3444f 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -60,6 +60,15 @@ def test_unique_items(asserter, value, expected): }, value, expected) +def test_min_and_unique_items(asserter): + value = None + asserter({ + 'type': ['array', 'null'], + 'minItems': 1, + 'uniqueItems': True, + }, value, value) + + @pytest.mark.parametrize('value, expected', [ ([], []), ([1], [1]), From 49facc2e1710ffe37f0174a28771928b798c2280 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 26 Sep 2019 19:04:14 +0200 Subject: [PATCH 164/201] Optimization: do not do the same type checks, keep it in one block if possible --- CHANGELOG.txt | 1 + fastjsonschema/draft04.py | 6 +++--- fastjsonschema/generator.py | 2 ++ fastjsonschema/indent.py | 15 +++++++++++---- tests/benchmarks/test_benchmark.py | 3 +-- 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 0abcf4b..b4e5cc1 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,5 +1,6 @@ === 2.14 (unreleased) +* Optimization: do not do the same type checks, keep it in one block if possible * Fix of uniqueItems when used with other than array type * Possibility to pass custom formats * Raise JsonSchemaDefinitionException when definition of property is not valid diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index aa87008..b57364b 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -145,13 +145,13 @@ def generate_any_of(self): self.l('{variable}_any_of_count = 0') for definition_item in self._definition['anyOf']: # When we know it's passing (at least once), we do not need to do another expensive try-except. - with self.l('if not {variable}_any_of_count:'): + with self.l('if not {variable}_any_of_count:', optimize=False): with self.l('try:'): self.generate_func_code_block(definition_item, self._variable, self._variable_name, clear_variables=True) self.l('{variable}_any_of_count += 1') self.l('except JsonSchemaException: pass') - with self.l('if not {variable}_any_of_count:'): + with self.l('if not {variable}_any_of_count:', optimize=False): self.l('raise JsonSchemaException("{name} must be valid by one of anyOf definition")') def generate_one_of(self): @@ -173,7 +173,7 @@ def generate_one_of(self): self.l('{variable}_one_of_count = 0') for definition_item in self._definition['oneOf']: # When we know it's failing (one of means exactly once), we do not need to do another expensive try-except. - with self.l('if {variable}_one_of_count < 2:'): + with self.l('if {variable}_one_of_count < 2:', optimize=False): with self.l('try:'): self.generate_func_code_block(definition_item, self._variable, self._variable_name, clear_variables=True) self.l('{variable}_one_of_count += 1') diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 8d4c528..9d82065 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -34,6 +34,7 @@ def __init__(self, definition, resolver=None): self._variables = set() self._indent = 0 + self._indent_last_line = None self._variable = None self._variable_name = None self._root_definition = definition @@ -224,6 +225,7 @@ def l(self, line, *args, **kwds): line = line.format(*args, **context) line = line.replace('\n', '\\n').replace('\r', '\\r') self._code.append(spaces + line) + return line def e(self, string): """ diff --git a/fastjsonschema/indent.py b/fastjsonschema/indent.py index e27fced..83fccaa 100644 --- a/fastjsonschema/indent.py +++ b/fastjsonschema/indent.py @@ -3,18 +3,25 @@ def indent(func): Decorator for allowing to use method as normal method or with context manager for auto-indenting code blocks. """ - def wrapper(self, *args, **kwds): - func(self, *args, **kwds) - return Indent(self) + def wrapper(self, line, *args, optimize=True, **kwds): + last_line = self._indent_last_line + line = func(self, line, *args, **kwds) + # When two blocks have the same condition (such as value has to be dict), + # do the check only once and keep it under one block. + if optimize and last_line == line: + self._code.pop() + return Indent(self, line) return wrapper class Indent: - def __init__(self, instance): + def __init__(self, instance, line): self.instance = instance + self.line = line def __enter__(self): self.instance._indent += 1 def __exit__(self, type_, value, traceback): self.instance._indent -= 1 + self.instance._indent_last_line = self.line diff --git a/tests/benchmarks/test_benchmark.py b/tests/benchmarks/test_benchmark.py index 8da2c8a..31098b3 100644 --- a/tests/benchmarks/test_benchmark.py +++ b/tests/benchmarks/test_benchmark.py @@ -8,8 +8,7 @@ 'items': [ { 'type': 'number', - 'maximum': 10, - 'exclusiveMaximum': True, + 'exclusiveMaximum': 10, }, { 'type': 'string', From e4f1a5726cc9b8a3bc429c18819ebbc9278a4e45 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Sun, 6 Oct 2019 10:24:34 +0200 Subject: [PATCH 165/201] Indent fix --- fastjsonschema/indent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fastjsonschema/indent.py b/fastjsonschema/indent.py index 83fccaa..411c69f 100644 --- a/fastjsonschema/indent.py +++ b/fastjsonschema/indent.py @@ -10,6 +10,7 @@ def wrapper(self, line, *args, optimize=True, **kwds): # do the check only once and keep it under one block. if optimize and last_line == line: self._code.pop() + self._indent_last_line = line return Indent(self, line) return wrapper From d43ed34dae24e7195599dfb8e7d1b4be5d44598d Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Sun, 6 Oct 2019 10:42:43 +0200 Subject: [PATCH 166/201] Raise JSONSchemaDefinitionException when the property definition is not valid --- CHANGELOG.txt | 2 +- fastjsonschema/generator.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index b4e5cc1..19496bf 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,9 +1,9 @@ === 2.14 (unreleased) * Optimization: do not do the same type checks, keep it in one block if possible -* Fix of uniqueItems when used with other than array type * Possibility to pass custom formats * Raise JsonSchemaDefinitionException when definition of property is not valid +* Fix of uniqueItems when used with other than array type * Fix of date-time regexp (time zone is required by RFC 3339) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 9d82065..3c1e499 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -1,7 +1,7 @@ from collections import OrderedDict import re -from .exceptions import JsonSchemaException +from .exceptions import JsonSchemaException, JsonSchemaDefinitionException from .indent import indent from .ref_resolver import RefResolver @@ -156,6 +156,8 @@ def generate_func_code_block(self, definition, variable, variable_name, clear_va self._variables = backup_variables def _generate_func_code_block(self, definition): + if not isinstance(definition, dict): + raise JsonSchemaDefinitionException("definition must be an object") if '$ref' in definition: # needed because ref overrides any sibling keywords self.generate_ref() From 93194b890f81e140c099654cd65b4f9e82a23e6d Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Sun, 6 Oct 2019 19:59:05 +0200 Subject: [PATCH 167/201] Context in exceptions --- CHANGELOG.txt | 1 + fastjsonschema/__init__.py | 2 +- fastjsonschema/draft04.py | 62 ++++++++++++----------- fastjsonschema/draft06.py | 20 ++++---- fastjsonschema/draft07.py | 8 +-- fastjsonschema/exceptions.py | 24 +++++++-- fastjsonschema/generator.py | 6 +++ tests/conftest.py | 3 ++ tests/test_array.py | 23 ++++----- tests/test_boolean.py | 3 +- tests/test_common.py | 15 +++--- tests/test_default.py | 5 +- tests/test_exceptions.py | 17 +++++++ tests/test_format.py | 6 +-- tests/test_integration.py | 98 ++++++++++++++++++------------------ tests/test_null.py | 3 +- tests/test_number.py | 15 +++--- tests/test_object.py | 28 +++++------ tests/test_string.py | 11 ++-- 19 files changed, 195 insertions(+), 155 deletions(-) create mode 100644 tests/test_exceptions.py diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 19496bf..8777920 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,7 @@ === 2.14 (unreleased) * Optimization: do not do the same type checks, keep it in one block if possible +* More context in JsonSchemaException (value, variable_name, variable_path and definition) * Possibility to pass custom formats * Raise JsonSchemaDefinitionException when definition of property is not valid * Fix of uniqueItems when used with other than array type diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 2b4b7b2..08cc459 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -98,7 +98,7 @@ def validate(definition, data, handlers={}, formats={}): validate({'type': 'string'}, 'hello') # same as: compile({'type': 'string'})('hello') - Preffered is to use :any:`compile` function. + Preferred is to use :any:`compile` function. """ return compile(definition, handlers, formats)(data) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index b57364b..2885c9d 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -22,7 +22,7 @@ class CodeGeneratorDraft04(CodeGenerator): # pylint: disable=line-too-long # I was thinking about using ipaddress module instead of regexps for example, but it's big # difference in performance. With a module I got this difference: over 100 ms with a module - # vs. 9 ms with a regex! Other modules are also unefective or not available in standard + # vs. 9 ms with a regex! Other modules are also ineffective or not available in standard # library. Some regexps are not 100% precise but good enough, fast and without dependencies. FORMAT_REGEXS = { 'date-time': r'^\d{4}-[01]\d-[0-3]\d(t|T)[0-2]\d:[0-5]\d:[0-5]\d(?:\.\d+)?(?:[+-][0-2]\d:[0-5]\d|z|Z)\Z', @@ -89,7 +89,7 @@ def generate_type(self): extra = ' or isinstance({variable}, bool)'.format(variable=self._variable) with self.l('if not isinstance({variable}, ({})){}:', python_types, extra): - self.l('raise JsonSchemaException("{name} must be {}")', ' or '.join(types)) + self.exc('{name} must be {}', ' or '.join(types)) def generate_enum(self): """ @@ -105,7 +105,7 @@ def generate_enum(self): if not isinstance(enum, (list, tuple)): raise JsonSchemaDefinitionException('enum must be an array') with self.l('if {variable} not in {enum}:'): - self.l('raise JsonSchemaException("{name} must be one of {}")', self.e(enum)) + self.exc('{name} must be one of {}', self.e(enum)) def generate_all_of(self): """ @@ -152,7 +152,7 @@ def generate_any_of(self): self.l('except JsonSchemaException: pass') with self.l('if not {variable}_any_of_count:', optimize=False): - self.l('raise JsonSchemaException("{name} must be valid by one of anyOf definition")') + self.exc('{name} must be valid by one of anyOf definition') def generate_one_of(self): """ @@ -180,7 +180,7 @@ def generate_one_of(self): self.l('except JsonSchemaException: pass') with self.l('if {variable}_one_of_count != 1:'): - self.l('raise JsonSchemaException("{name} must be valid exactly by one of oneOf definition")') + self.exc('{name} must be valid exactly by one of oneOf definition') def generate_not(self): """ @@ -197,17 +197,18 @@ def generate_not(self): """ not_definition = self._definition['not'] if not_definition is True: - self.l('raise JsonSchemaException("{name} must not be there")') + self.exc('{name} must not be there') elif not_definition is False: return elif not not_definition: with self.l('if {}:', self._variable): - self.l('raise JsonSchemaException("{name} must not be valid by not definition")') + self.exc('{name} must not be valid by not definition') else: with self.l('try:'): self.generate_func_code_block(not_definition, self._variable, self._variable_name) self.l('except JsonSchemaException: pass') - self.l('else: raise JsonSchemaException("{name} must not be valid by not definition")') + with self.l('else:'): + self.exc('{name} must not be valid by not definition') def generate_min_length(self): with self.l('if isinstance({variable}, str):'): @@ -215,7 +216,7 @@ def generate_min_length(self): if not isinstance(self._definition['minLength'], int): raise JsonSchemaDefinitionException('minLength must be a number') with self.l('if {variable}_len < {minLength}:'): - self.l('raise JsonSchemaException("{name} must be longer than or equal to {minLength} characters")') + self.exc('{name} must be longer than or equal to {minLength} characters') def generate_max_length(self): with self.l('if isinstance({variable}, str):'): @@ -223,7 +224,7 @@ def generate_max_length(self): if not isinstance(self._definition['maxLength'], int): raise JsonSchemaDefinitionException('maxLength must be a number') with self.l('if {variable}_len > {maxLength}:'): - self.l('raise JsonSchemaException("{name} must be shorter than or equal to {maxLength} characters")') + self.exc('{name} must be shorter than or equal to {maxLength} characters') def generate_pattern(self): with self.l('if isinstance({variable}, str):'): @@ -232,7 +233,7 @@ def generate_pattern(self): end_of_string_fixed_pattern = DOLLAR_FINDER.sub(r'\\Z', pattern) self._compile_regexps[pattern] = re.compile(end_of_string_fixed_pattern) with self.l('if not REGEX_PATTERNS[{}].search({variable}):', repr(pattern)): - self.l('raise JsonSchemaException("{name} must match pattern {}")', safe_pattern) + self.exc('{name} must match pattern {}', safe_pattern) def generate_format(self): """ @@ -253,7 +254,7 @@ def generate_format(self): self._generate_format(format_, format_ + '_re_pattern', custom_format) else: with self.l('if not custom_formats["{}"]({variable}):', format_): - self.l('raise JsonSchemaException("{name} must be {}")', format_) + self.exc('{name} must be {}', format_) elif format_ in self.FORMAT_REGEXS: format_regex = self.FORMAT_REGEXS[format_] self._generate_format(format_, format_ + '_re_pattern', format_regex) @@ -262,7 +263,7 @@ def generate_format(self): with self.l('try:'): self.l('re.compile({variable})') with self.l('except Exception:'): - self.l('raise JsonSchemaException("{name} must be a valid regex")') + self.exc('{name} must be a valid regex') else: raise JsonSchemaDefinitionException('Undefined format %s'.format(format_)) @@ -272,7 +273,7 @@ def _generate_format(self, format_name, regexp_name, regexp): if not regexp_name in self._compile_regexps: self._compile_regexps[regexp_name] = re.compile(regexp) with self.l('if not REGEX_PATTERNS["{}"].match({variable}):', regexp_name): - self.l('raise JsonSchemaException("{name} must be {}")', format_name) + self.exc('{name} must be {}', format_name) def generate_minimum(self): with self.l('if isinstance({variable}, (int, float)):'): @@ -280,10 +281,10 @@ def generate_minimum(self): raise JsonSchemaDefinitionException('minimum must be a number') if self._definition.get('exclusiveMinimum', False): with self.l('if {variable} <= {minimum}:'): - self.l('raise JsonSchemaException("{name} must be bigger than {minimum}")') + self.exc('{name} must be bigger than {minimum}') else: with self.l('if {variable} < {minimum}:'): - self.l('raise JsonSchemaException("{name} must be bigger than or equal to {minimum}")') + self.exc('{name} must be bigger than or equal to {minimum}') def generate_maximum(self): with self.l('if isinstance({variable}, (int, float)):'): @@ -291,10 +292,10 @@ def generate_maximum(self): raise JsonSchemaDefinitionException('maximum must be a number') if self._definition.get('exclusiveMaximum', False): with self.l('if {variable} >= {maximum}:'): - self.l('raise JsonSchemaException("{name} must be smaller than {maximum}")') + self.exc('{name} must be smaller than {maximum}') else: with self.l('if {variable} > {maximum}:'): - self.l('raise JsonSchemaException("{name} must be smaller than or equal to {maximum}")') + self.exc('{name} must be smaller than or equal to {maximum}') def generate_multiple_of(self): with self.l('if isinstance({variable}, (int, float)):'): @@ -302,7 +303,7 @@ def generate_multiple_of(self): raise JsonSchemaDefinitionException('multipleOf must be a number') self.l('quotient = {variable} / {multipleOf}') with self.l('if int(quotient) != quotient:'): - self.l('raise JsonSchemaException("{name} must be multiple of {multipleOf}")') + self.exc('{name} must be multiple of {multipleOf}') def generate_min_items(self): self.create_variable_is_list() @@ -311,7 +312,7 @@ def generate_min_items(self): raise JsonSchemaDefinitionException('minItems must be a number') self.create_variable_with_length() with self.l('if {variable}_len < {minItems}:'): - self.l('raise JsonSchemaException("{name} must contain at least {minItems} items")') + self.exc('{name} must contain at least {minItems} items') def generate_max_items(self): self.create_variable_is_list() @@ -320,7 +321,7 @@ def generate_max_items(self): raise JsonSchemaDefinitionException('maxItems must be a number') self.create_variable_with_length() with self.l('if {variable}_len > {maxItems}:'): - self.l('raise JsonSchemaException("{name} must contain less than or equal to {maxItems} items")') + self.exc('{name} must contain less than or equal to {maxItems} items') def generate_unique_items(self): """ @@ -341,7 +342,7 @@ def generate_unique_items(self): with self.l('if {variable}_is_list:'): self.create_variable_with_length() with self.l('if {variable}_len > len(set(str({variable}_x) for {variable}_x in {variable})):'): - self.l('raise JsonSchemaException("{name} must contain unique items")') + self.exc('{name} must contain unique items') def generate_items(self): """ @@ -370,7 +371,7 @@ def generate_items(self): self.create_variable_with_length() if items_definition is False: with self.l('if {variable}:'): - self.l('raise JsonSchemaException("{name} must not be there")') + self.exc('{name} must not be there') elif isinstance(items_definition, list): for idx, item_definition in enumerate(items_definition): with self.l('if {variable}_len > {}:', idx): @@ -385,7 +386,8 @@ def generate_items(self): if 'additionalItems' in self._definition: if self._definition['additionalItems'] is False: - self.l('if {variable}_len > {}: raise JsonSchemaException("{name} must contain only specified items")', len(items_definition)) + with self.l('if {variable}_len > {}:', len(items_definition)): + self.exc('{name} must contain only specified items') else: with self.l('for {variable}_x, {variable}_item in enumerate({variable}[{0}:], {0}):', len(items_definition)): self.generate_func_code_block( @@ -409,7 +411,7 @@ def generate_min_properties(self): raise JsonSchemaDefinitionException('minProperties must be a number') self.create_variable_with_length() with self.l('if {variable}_len < {minProperties}:'): - self.l('raise JsonSchemaException("{name} must contain at least {minProperties} properties")') + self.exc('{name} must contain at least {minProperties} properties') def generate_max_properties(self): self.create_variable_is_dict() @@ -418,7 +420,7 @@ def generate_max_properties(self): raise JsonSchemaDefinitionException('maxProperties must be a number') self.create_variable_with_length() with self.l('if {variable}_len > {maxProperties}:'): - self.l('raise JsonSchemaException("{name} must contain less than or equal to {maxProperties} properties")') + self.exc('{name} must contain less than or equal to {maxProperties} properties') def generate_required(self): self.create_variable_is_dict() @@ -427,7 +429,7 @@ def generate_required(self): raise JsonSchemaDefinitionException('required must be an array') self.create_variable_with_length() with self.l('if not all(prop in {variable} for prop in {required}):'): - self.l('raise JsonSchemaException("{name} must contain {} properties")', self.e(self._definition['required'])) + self.exc('{name} must contain {} properties', self.e(self._definition['required'])) def generate_properties(self): """ @@ -525,7 +527,7 @@ def generate_additional_properties(self): ) else: with self.l('if {variable}_keys:'): - self.l('raise JsonSchemaException("{name} must contain only specified properties")') + self.exc('{name} must contain only specified properties') def generate_dependencies(self): """ @@ -553,10 +555,10 @@ def generate_dependencies(self): continue with self.l('if "{}" in {variable}_keys:', self.e(key)): if values is False: - self.l('raise JsonSchemaException("{} in {name} must not be there")', key) + self.exc('{} in {name} must not be there', key) elif isinstance(values, list): for value in values: with self.l('if "{}" not in {variable}_keys:', self.e(value)): - self.l('raise JsonSchemaException("{name} missing dependency {} for {}")', self.e(value), self.e(key)) + self.exc('{name} missing dependency {} for {}', self.e(value), self.e(key)) else: self.generate_func_code_block(values, self._variable, self._variable_name, clear_variables=True) diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py index e8521a6..abca80b 100644 --- a/fastjsonschema/draft06.py +++ b/fastjsonschema/draft06.py @@ -40,7 +40,7 @@ def generate_boolean_schema(self): True means everything is valid, False everything is invalid. """ if self._definition is False: - self.l('raise JsonSchemaException("{name} must not be there")') + self.exc('{name} must not be there') def generate_type(self): """ @@ -70,21 +70,21 @@ def generate_type(self): extra += ' or isinstance({variable}, bool)'.format(variable=self._variable) with self.l('if not isinstance({variable}, ({})){}:', python_types, extra): - self.l('raise JsonSchemaException("{name} must be {}")', ' or '.join(types)) + self.exc('{name} must be {}', ' or '.join(types)) def generate_exclusive_minimum(self): with self.l('if isinstance({variable}, (int, float)):'): if not isinstance(self._definition['exclusiveMinimum'], (int, float)): raise JsonSchemaDefinitionException('exclusiveMinimum must be an integer or a float') with self.l('if {variable} <= {exclusiveMinimum}:'): - self.l('raise JsonSchemaException("{name} must be bigger than {exclusiveMinimum}")') + self.exc('{name} must be bigger than {exclusiveMinimum}') def generate_exclusive_maximum(self): with self.l('if isinstance({variable}, (int, float)):'): if not isinstance(self._definition['exclusiveMaximum'], (int, float)): raise JsonSchemaDefinitionException('exclusiveMaximum must be an integer or a float') with self.l('if {variable} >= {exclusiveMaximum}:'): - self.l('raise JsonSchemaException("{name} must be smaller than {exclusiveMaximum}")') + self.exc('{name} must be smaller than {exclusiveMaximum}') def generate_property_names(self): """ @@ -106,7 +106,7 @@ def generate_property_names(self): elif property_names_definition is False: self.create_variable_keys() with self.l('if {variable}_keys:'): - self.l('raise JsonSchemaException("{name} must not be there")') + self.exc('{name} must not be there') else: self.create_variable_is_dict() with self.l('if {variable}_is_dict:'): @@ -124,7 +124,7 @@ def generate_property_names(self): with self.l('except JsonSchemaException:'): self.l('{variable}_property_names = False') with self.l('if not {variable}_property_names:'): - self.l('raise JsonSchemaException("{name} must be named by propertyName definition")') + self.exc('{name} must be named by propertyName definition') def generate_contains(self): """ @@ -145,10 +145,10 @@ def generate_contains(self): contains_definition = self._definition['contains'] if contains_definition is False: - self.l('raise JsonSchemaException("{name} is always invalid")') + self.exc('{name} is always invalid') elif contains_definition is True: with self.l('if not {variable}:'): - self.l('raise JsonSchemaException("{name} must not be empty")') + self.exc('{name} must not be empty') else: self.l('{variable}_contains = False') with self.l('for {variable}_key in {variable}:'): @@ -164,7 +164,7 @@ def generate_contains(self): self.l('except JsonSchemaException: pass') with self.l('if not {variable}_contains:'): - self.l('raise JsonSchemaException("{name} must contain one of contains definition")') + self.exc('{name} must contain one of contains definition') def generate_const(self): """ @@ -182,4 +182,4 @@ def generate_const(self): if isinstance(const, str): const = '"{}"'.format(const) with self.l('if {variable} != {}:', const): - self.l('raise JsonSchemaException("{name} must be same as const definition")') + self.exc('{name} must be same as const definition') diff --git a/fastjsonschema/draft07.py b/fastjsonschema/draft07.py index 3101328..8728a8c 100644 --- a/fastjsonschema/draft07.py +++ b/fastjsonschema/draft07.py @@ -88,9 +88,9 @@ def generate_content_encoding(self): self.l('import base64') self.l('{variable} = base64.b64decode({variable})') with self.l('except Exception:'): - self.l('raise JsonSchemaException("{name} must be encoded by base64")') + self.exc('{name} must be encoded by base64') with self.l('if {variable} == "":'): - self.l('raise JsonSchemaException("contentEncoding must be base64")') + self.exc('contentEncoding must be base64') def generate_content_media_type(self): """ @@ -107,10 +107,10 @@ def generate_content_media_type(self): with self.l('try:'): self.l('{variable} = {variable}.decode("utf-8")') with self.l('except Exception:'): - self.l('raise JsonSchemaException("{name} must encoded by utf8")') + self.exc('{name} must encoded by utf8') with self.l('if isinstance({variable}, str):'): with self.l('try:'): self.l('import json') self.l('{variable} = json.loads({variable})') with self.l('except Exception:'): - self.l('raise JsonSchemaException("{name} must be valid JSON")') + self.exc('{name} must be valid JSON') diff --git a/fastjsonschema/exceptions.py b/fastjsonschema/exceptions.py index 912bebe..0cb78e4 100644 --- a/fastjsonschema/exceptions.py +++ b/fastjsonschema/exceptions.py @@ -1,12 +1,30 @@ +import re + + +SPLIT_RE = re.compile(r'[\.\[\]]+') + + class JsonSchemaException(ValueError): """ - Exception raised by validation function. Contains ``message`` with - information what is wrong. + Exception raised by validation function. Available properties: + + * ``message`` with information what is wrong, + * ``value`` of invalid data, + * ``name`` as a string with a path in the input, + * ``path`` as an array with a path in the input, + * and ``definition`` which was broken. """ - def __init__(self, message): + def __init__(self, message, value=None, name=None, definition=None): super().__init__(message) self.message = message + self.value = value + self.name = name + self.definition = definition + + @property + def path(self): + return [item for item in SPLIT_RE.split(self.name) if item != ''] class JsonSchemaDefinitionException(JsonSchemaException): diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 3c1e499..23339fe 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -239,6 +239,12 @@ def e(self, string): """ return str(string).replace('"', '\\"') + def exc(self, msg, *args): + """ + """ + msg = 'raise JsonSchemaException("'+msg+'", value={variable}, name="{name}", definition={definition})' + self.l(msg, *args, definition=repr(self._definition)) + def create_variable_with_length(self): """ Append code for creating variable with length of that variable diff --git a/tests/conftest.py b/tests/conftest.py index 70d1596..ca5af75 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,6 +29,9 @@ def f(definition, value, expected, formats={}): with pytest.raises(JsonSchemaException) as exc: validator(value) assert exc.value.message == expected.message + assert exc.value.value == (value if expected.value == '{data}' else expected.value) + assert exc.value.name == expected.name + assert exc.value.definition == (definition if expected.definition == '{definition}' else expected.definition) else: assert validator(value) == expected return f diff --git a/tests/test_array.py b/tests/test_array.py index 6c3444f..4a1cc37 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -1,10 +1,9 @@ - import pytest from fastjsonschema import JsonSchemaException -exc = JsonSchemaException('data must be array') +exc = JsonSchemaException('data must be array', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ (0, exc), (None, exc), @@ -19,7 +18,7 @@ def test_array(asserter, value, expected): asserter({'type': 'array'}, value, expected) -exc = JsonSchemaException('data must contain less than or equal to 1 items') +exc = JsonSchemaException('data must contain less than or equal to 1 items', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ ([], []), ([1], [1]), @@ -33,7 +32,7 @@ def test_max_items(asserter, value, expected): }, value, expected) -exc = JsonSchemaException('data must contain at least 2 items') +exc = JsonSchemaException('data must contain at least 2 items', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ ([], exc), ([1], exc), @@ -50,7 +49,7 @@ def test_min_items(asserter, value, expected): @pytest.mark.parametrize('value, expected', [ ([], []), ([1], [1]), - ([1, 1], JsonSchemaException('data must contain unique items')), + ([1, 1], JsonSchemaException('data must contain unique items', value='{data}', name='data', definition='{definition}')), ([1, 2, 3], [1, 2, 3]), ]) def test_unique_items(asserter, value, expected): @@ -72,7 +71,7 @@ def test_min_and_unique_items(asserter): @pytest.mark.parametrize('value, expected', [ ([], []), ([1], [1]), - ([1, 'a'], JsonSchemaException('data[1] must be number')), + ([1, 'a'], JsonSchemaException('data[1] must be number', value='a', name='data[1]', definition={'type': 'number'})), ]) def test_items_all_same(asserter, value, expected): asserter({ @@ -85,7 +84,7 @@ def test_items_all_same(asserter, value, expected): ([], []), ([1], [1]), ([1, 'a'], [1, 'a']), - ([1, 2], JsonSchemaException('data[1] must be string')), + ([1, 2], JsonSchemaException('data[1] must be string', value=2, name='data[1]', definition={'type': 'string'})), ([1, 'a', 2], [1, 'a', 2]), ([1, 'a', 'b'], [1, 'a', 'b']), ]) @@ -103,8 +102,8 @@ def test_different_items(asserter, value, expected): ([], []), ([1], [1]), ([1, 'a'], [1, 'a']), - ([1, 2], JsonSchemaException('data[1] must be string')), - ([1, 'a', 2], JsonSchemaException('data[2] must be string')), + ([1, 2], JsonSchemaException('data[1] must be string', value=2, name='data[1]', definition={'type': 'string'})), + ([1, 'a', 2], JsonSchemaException('data[2] must be string', value=2, name='data[2]', definition={'type': 'string'})), ([1, 'a', 'b'], [1, 'a', 'b']), ]) def test_different_items_with_additional_items(asserter, value, expected): @@ -122,9 +121,9 @@ def test_different_items_with_additional_items(asserter, value, expected): ([], []), ([1], [1]), ([1, 'a'], [1, 'a']), - ([1, 2], JsonSchemaException('data[1] must be string')), - ([1, 'a', 2], JsonSchemaException('data must contain only specified items')), - ([1, 'a', 'b'], JsonSchemaException('data must contain only specified items')), + ([1, 2], JsonSchemaException('data[1] must be string', value=2, name='data[1]', definition={'type': 'string'})), + ([1, 'a', 2], JsonSchemaException('data must contain only specified items', value='{data}', name='data', definition='{definition}')), + ([1, 'a', 'b'], JsonSchemaException('data must contain only specified items', value='{data}', name='data', definition='{definition}')), ]) def test_different_items_without_additional_items(asserter, value, expected): asserter({ diff --git a/tests/test_boolean.py b/tests/test_boolean.py index 4a23018..6988c4f 100644 --- a/tests/test_boolean.py +++ b/tests/test_boolean.py @@ -1,10 +1,9 @@ - import pytest from fastjsonschema import JsonSchemaException -exc = JsonSchemaException('data must be boolean') +exc = JsonSchemaException('data must be boolean', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ (0, exc), (None, exc), diff --git a/tests/test_common.py b/tests/test_common.py index 3eeb747..ad2fbab 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -1,10 +1,9 @@ - import pytest from fastjsonschema import JsonSchemaException -exc = JsonSchemaException('data must be one of [1, 2, \'a\', "b\'c"]') +exc = JsonSchemaException('data must be one of [1, 2, \'a\', "b\'c"]', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ (1, 1), (2, 2), @@ -16,7 +15,7 @@ def test_enum(asserter, value, expected): asserter({'enum': [1, 2, 'a', "b'c"]}, value, expected) -exc = JsonSchemaException('data must be string or number') +exc = JsonSchemaException('data must be string or number', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ (0, 0), (None, exc), @@ -31,7 +30,7 @@ def test_types(asserter, value, expected): @pytest.mark.parametrize('value, expected', [ ('qwert', 'qwert'), - ('qwertz', JsonSchemaException('data must be shorter than or equal to 5 characters')), + ('qwertz', JsonSchemaException('data must be shorter than or equal to 5 characters', value='{data}', name='data', definition={'maxLength': 5})), ]) def test_all_of(asserter, value, expected): asserter({'allOf': [ @@ -40,7 +39,7 @@ def test_all_of(asserter, value, expected): ]}, value, expected) -exc = JsonSchemaException('data must be valid by one of anyOf definition') +exc = JsonSchemaException('data must be valid by one of anyOf definition', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ (0, 0), (None, exc), @@ -56,7 +55,7 @@ def test_any_of(asserter, value, expected): ]}, value, expected) -exc = JsonSchemaException('data must be valid exactly by one of oneOf definition') +exc = JsonSchemaException('data must be valid exactly by one of oneOf definition', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ (0, exc), (2, exc), @@ -71,7 +70,7 @@ def test_one_of(asserter, value, expected): ]}, value, expected) -exc = JsonSchemaException('data must be valid exactly by one of oneOf definition') +exc = JsonSchemaException('data must be valid exactly by one of oneOf definition', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ (0, exc), (2, exc), @@ -90,7 +89,7 @@ def test_one_of_factorized(asserter, value, expected): @pytest.mark.parametrize('value, expected', [ - (0, JsonSchemaException('data must not be valid by not definition')), + (0, JsonSchemaException('data must not be valid by not definition', value='{data}', name='data', definition='{definition}')), (True, True), ('abc', 'abc'), ([], []), diff --git a/tests/test_default.py b/tests/test_default.py index 54e40bb..616e4a7 100644 --- a/tests/test_default.py +++ b/tests/test_default.py @@ -1,11 +1,10 @@ - import pytest from fastjsonschema import JsonSchemaException @pytest.mark.parametrize('value, expected', [ - (None, JsonSchemaException('data must be object')), + (None, JsonSchemaException('data must be object', value='{data}', name='data', definition='{definition}')), ({}, {'a': '', 'b': 42, 'c': {}, 'd': []}), ({'a': 'abc'}, {'a': 'abc', 'b': 42, 'c': {}, 'd': []}), ({'b': 123}, {'a': '', 'b': 123, 'c': {}, 'd': []}), @@ -24,7 +23,7 @@ def test_default_in_object(asserter, value, expected): @pytest.mark.parametrize('value, expected', [ - (None, JsonSchemaException('data must be array')), + (None, JsonSchemaException('data must be array', value='{data}', name='data', definition='{definition}')), ([], ['', 42]), (['abc'], ['abc', 42]), (['abc', 123], ['abc', 123]), diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py new file mode 100644 index 0000000..e0482cd --- /dev/null +++ b/tests/test_exceptions.py @@ -0,0 +1,17 @@ +import pytest + +from fastjsonschema import JsonSchemaException + + +@pytest.mark.parametrize('value, expected', [ + ('data', ['data']), + ('data[0]', ['data', '0']), + ('data.foo', ['data', 'foo']), + ('data[1].bar', ['data', '1', 'bar']), + ('data.foo[2]', ['data', 'foo', '2']), + ('data.foo.bar[1][2]', ['data', 'foo', 'bar', '1', '2']), + ('data[1][2].foo.bar', ['data', '1', '2', 'foo', 'bar']), +]) +def test_exception_variable_path(value, expected): + exc = JsonSchemaException('msg', name=value) + assert exc.path == expected diff --git a/tests/test_format.py b/tests/test_format.py index 490d70b..45a58ec 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -6,7 +6,7 @@ from fastjsonschema import JsonSchemaException -exc = JsonSchemaException('data must be date-time') +exc = JsonSchemaException('data must be date-time', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ ('', exc), ('bla', exc), @@ -19,7 +19,7 @@ def test_datetime(asserter, value, expected): asserter({'type': 'string', 'format': 'date-time'}, value, expected) -exc = JsonSchemaException('data must be hostname') +exc = JsonSchemaException('data must be hostname', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ ('', exc), ('LDhsjf878&d', exc), @@ -36,7 +36,7 @@ def test_hostname(asserter, value, expected): asserter({'type': 'string', 'format': 'hostname'}, value, expected) -exc = JsonSchemaException('data must be custom-format') +exc = JsonSchemaException('data must be custom-format', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value,expected,custom_format', [ ('', exc, r'^[ab]$'), ('', exc, lambda value: value in ('a', 'b')), diff --git a/tests/test_integration.py b/tests/test_integration.py index f0ac8e9..ec9d80b 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,9 +1,48 @@ - import pytest from fastjsonschema import JsonSchemaException +definition = { + 'type': 'array', + 'items': [ + { + 'type': 'number', + 'maximum': 10, + 'exclusiveMaximum': True, + }, + { + 'type': 'string', + 'enum': ['hello', 'world'], + }, + { + 'type': 'array', + 'minItems': 1, + 'maxItems': 3, + 'items': [ + {'type': 'number'}, + {'type': 'string'}, + {'type': 'boolean'}, + ], + }, + { + 'type': 'object', + 'required': ['a', 'b'], + 'minProperties': 3, + 'properties': { + 'a': {'type': ['null', 'string']}, + 'b': {'type': ['null', 'string']}, + 'c': {'type': ['null', 'string'], 'default': 'abc'} + }, + 'additionalProperties': {'type': 'string'}, + }, + {'not': {'type': ['null']}}, + {'oneOf': [ + {'type': 'number', 'multipleOf': 3}, + {'type': 'number', 'multipleOf': 5}, + ]}, + ], +} @pytest.mark.parametrize('value, expected', [ ( [9, 'hello', [1, 'a', True], {'a': 'a', 'b': 'b', 'd': 'd'}, 42, 3], @@ -27,78 +66,39 @@ ), ( [10, 'world', [1], {'a': 'a', 'b': 'b', 'c': 'xy'}, 'str', 5], - JsonSchemaException('data[0] must be smaller than 10'), + JsonSchemaException('data[0] must be smaller than 10', value=10, name='data[0]', definition=definition['items'][0]), ), ( [9, 'xxx', [1], {'a': 'a', 'b': 'b', 'c': 'xy'}, 'str', 5], - JsonSchemaException('data[1] must be one of [\'hello\', \'world\']'), + JsonSchemaException('data[1] must be one of [\'hello\', \'world\']', value='xxx', name='data[1]', definition=definition['items'][1]), ), ( [9, 'hello', [], {'a': 'a', 'b': 'b', 'c': 'xy'}, 'str', 5], - JsonSchemaException('data[2] must contain at least 1 items'), + JsonSchemaException('data[2] must contain at least 1 items', value=[], name='data[2]', definition=definition['items'][2]), ), ( [9, 'hello', [1, 2, 3], {'a': 'a', 'b': 'b', 'c': 'xy'}, 'str', 5], - JsonSchemaException('data[2][1] must be string'), + JsonSchemaException('data[2][1] must be string', value=2, name='data[2][1]', definition={'type': 'string'}), ), ( [9, 'hello', [1], {'a': 'a', 'x': 'x', 'y': 'y'}, 'str', 5], - JsonSchemaException('data[3] must contain [\'a\', \'b\'] properties'), + JsonSchemaException('data[3] must contain [\'a\', \'b\'] properties', value={'a': 'a', 'x': 'x', 'y': 'y'}, name='data[3]', definition=definition['items'][3]), ), ( [9, 'hello', [1], {}, 'str', 5], - JsonSchemaException('data[3] must contain at least 3 properties'), + JsonSchemaException('data[3] must contain at least 3 properties', value={}, name='data[3]', definition=definition['items'][3]), ), ( [9, 'hello', [1], {'a': 'a', 'b': 'b', 'x': 'x'}, None, 5], - JsonSchemaException('data[4] must not be valid by not definition'), + JsonSchemaException('data[4] must not be valid by not definition', value=None, name='data[4]', definition=definition['items'][4]), ), ( [9, 'hello', [1], {'a': 'a', 'b': 'b', 'x': 'x'}, 42, 15], - JsonSchemaException('data[5] must be valid exactly by one of oneOf definition'), + JsonSchemaException('data[5] must be valid exactly by one of oneOf definition', value=15, name='data[5]', definition=definition['items'][5]), ), ]) def test_integration(asserter, value, expected): - asserter({ - 'type': 'array', - 'items': [ - { - 'type': 'number', - 'maximum': 10, - 'exclusiveMaximum': True, - }, - { - 'type': 'string', - 'enum': ['hello', 'world'], - }, - { - 'type': 'array', - 'minItems': 1, - 'maxItems': 3, - 'items': [ - {'type': 'number'}, - {'type': 'string'}, - {'type': 'boolean'}, - ], - }, - { - 'type': 'object', - 'required': ['a', 'b'], - 'minProperties': 3, - 'properties': { - 'a': {'type': ['null', 'string']}, - 'b': {'type': ['null', 'string']}, - 'c': {'type': ['null', 'string'], 'default': 'abc'} - }, - 'additionalProperties': {'type': 'string'}, - }, - {'not': {'type': ['null']}}, - {'oneOf': [ - {'type': 'number', 'multipleOf': 3}, - {'type': 'number', 'multipleOf': 5}, - ]}, - ], - }, value, expected) + asserter(definition, value, expected) def test_any_of_with_patterns(asserter): diff --git a/tests/test_null.py b/tests/test_null.py index 22db265..985a722 100644 --- a/tests/test_null.py +++ b/tests/test_null.py @@ -1,10 +1,9 @@ - import pytest from fastjsonschema import JsonSchemaException -exc = JsonSchemaException('data must be null') +exc = JsonSchemaException('data must be null', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ (0, exc), (None, None), diff --git a/tests/test_number.py b/tests/test_number.py index 72ee2c3..d77960e 100644 --- a/tests/test_number.py +++ b/tests/test_number.py @@ -1,4 +1,3 @@ - import pytest from fastjsonschema import JsonSchemaException @@ -22,11 +21,11 @@ def number_type(request): ]) def test_number(asserter, number_type, value, expected): if isinstance(expected, JsonSchemaException): - expected = JsonSchemaException(expected.message.format(number_type=number_type)) + expected = JsonSchemaException(expected.message.format(number_type=number_type), value='{data}', name='data', definition='{definition}') asserter({'type': number_type}, value, expected) -exc = JsonSchemaException('data must be smaller than or equal to 10') +exc = JsonSchemaException('data must be smaller than or equal to 10', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ (-5, -5), (5, 5), @@ -42,7 +41,7 @@ def test_maximum(asserter, number_type, value, expected): }, value, expected) -exc = JsonSchemaException('data must be smaller than 10') +exc = JsonSchemaException('data must be smaller than 10', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ (-5, -5), (5, 5), @@ -59,7 +58,7 @@ def test_exclusive_maximum(asserter, number_type, value, expected): }, value, expected) -exc = JsonSchemaException('data must be bigger than or equal to 10') +exc = JsonSchemaException('data must be bigger than or equal to 10', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ (-5, exc), (9, exc), @@ -74,7 +73,7 @@ def test_minimum(asserter, number_type, value, expected): }, value, expected) -exc = JsonSchemaException('data must be bigger than 10') +exc = JsonSchemaException('data must be bigger than 10', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ (-5, exc), (9, exc), @@ -90,7 +89,7 @@ def test_exclusive_minimum(asserter, number_type, value, expected): }, value, expected) -exc = JsonSchemaException('data must be multiple of 3') +exc = JsonSchemaException('data must be multiple of 3', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ (-4, exc), (-3, -3), @@ -121,7 +120,7 @@ def test_multiple_of(asserter, number_type, value, expected): def test_integer_is_not_number(asserter, value): asserter({ 'type': 'integer', - }, value, JsonSchemaException('data must be integer')) + }, value, JsonSchemaException('data must be integer', value='{data}', name='data', definition='{definition}')) @pytest.mark.parametrize('value', ( diff --git a/tests/test_object.py b/tests/test_object.py index d241f59..c6ac784 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -4,7 +4,7 @@ from fastjsonschema import JsonSchemaDefinitionException, JsonSchemaException -exc = JsonSchemaException('data must be object') +exc = JsonSchemaException('data must be object', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ (0, exc), (None, exc), @@ -22,7 +22,7 @@ def test_object(asserter, value, expected): @pytest.mark.parametrize('value, expected', [ ({}, {}), ({'a': 1}, {'a': 1}), - ({'a': 1, 'b': 2}, JsonSchemaException('data must contain less than or equal to 1 properties')), + ({'a': 1, 'b': 2}, JsonSchemaException('data must contain less than or equal to 1 properties', value='{data}', name='data', definition='{definition}')), ]) def test_max_properties(asserter, value, expected): asserter({ @@ -32,7 +32,7 @@ def test_max_properties(asserter, value, expected): @pytest.mark.parametrize('value, expected', [ - ({}, JsonSchemaException('data must contain at least 1 properties')), + ({}, JsonSchemaException('data must contain at least 1 properties', value='{data}', name='data', definition='{definition}')), ({'a': 1}, {'a': 1}), ({'a': 1, 'b': 2}, {'a': 1, 'b': 2}), ]) @@ -43,7 +43,7 @@ def test_min_properties(asserter, value, expected): }, value, expected) -exc = JsonSchemaException('data must contain [\'a\', \'b\'] properties') +exc = JsonSchemaException('data must contain [\'a\', \'b\'] properties', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ ({}, exc), ({'a': 1}, exc), @@ -60,7 +60,7 @@ def test_required(asserter, value, expected): ({}, {}), ({'a': 1}, {'a': 1}), ({'a': 1, 'b': ''}, {'a': 1, 'b': ''}), - ({'a': 1, 'b': 2}, JsonSchemaException('data.b must be string')), + ({'a': 1, 'b': 2}, JsonSchemaException('data.b must be string', value=2, name='data.b', definition={'type': 'string'})), ({'a': 1, 'b': '', 'any': True}, {'a': 1, 'b': '', 'any': True}), ]) def test_properties(asserter, value, expected): @@ -86,9 +86,9 @@ def test_invalid_properties(asserter): ({}, {}), ({'a': 1}, {'a': 1}), ({'a': 1, 'b': ''}, {'a': 1, 'b': ''}), - ({'a': 1, 'b': 2}, JsonSchemaException('data.b must be string')), + ({'a': 1, 'b': 2}, JsonSchemaException('data.b must be string', value=2, name='data.b', definition={'type': 'string'})), ({'a': 1, 'b': '', 'additional': ''}, {'a': 1, 'b': '', 'additional': ''}), - ({'a': 1, 'b': '', 'any': True}, JsonSchemaException('data.any must be string')), + ({'a': 1, 'b': '', 'any': True}, JsonSchemaException('data.any must be string', value=True, name='data.any', definition={'type': 'string'})), ]) def test_properties_with_additional_properties(asserter, value, expected): asserter({ @@ -105,9 +105,9 @@ def test_properties_with_additional_properties(asserter, value, expected): ({}, {}), ({'a': 1}, {'a': 1}), ({'a': 1, 'b': ''}, {'a': 1, 'b': ''}), - ({'a': 1, 'b': 2}, JsonSchemaException('data.b must be string')), - ({'a': 1, 'b': '', 'any': True}, JsonSchemaException('data must contain only specified properties')), - ({'cd': True}, JsonSchemaException('data must contain only specified properties')), + ({'a': 1, 'b': 2}, JsonSchemaException('data.b must be string', value=2, name='data.b', definition={'type': 'string'})), + ({'a': 1, 'b': '', 'any': True}, JsonSchemaException('data must contain only specified properties', value='{data}', name='data', definition='{definition}')), + ({'cd': True}, JsonSchemaException('data must contain only specified properties', value='{data}', name='data', definition='{definition}')), ({'c_d': True}, {'c_d': True}), ]) def test_properties_without_additional_properties(asserter, value, expected): @@ -126,7 +126,7 @@ def test_properties_without_additional_properties(asserter, value, expected): ({}, {}), ({'a': 1}, {'a': 1}), ({'xa': 1}, {'xa': 1}), - ({'xa': ''}, JsonSchemaException('data.xa must be number')), + ({'xa': ''}, JsonSchemaException('data.xa must be number', value='', name='data.xa', definition={'type': 'number'})), ({'xbx': ''}, {'xbx': ''}), ]) def test_pattern_properties(asserter, value, expected): @@ -145,7 +145,7 @@ def test_pattern_properties(asserter, value, expected): ({'a': 1}, {'a': 1}), ({'b': True}, {'b': True}), ({'c': ''}, {'c': ''}), - ({'d': 1}, JsonSchemaException('data.d must be string')), + ({'d': 1}, JsonSchemaException('data.d must be string', value=1, name='data.d', definition={'type': 'string'})), ]) def test_additional_properties(asserter, value, expected): asserter({ @@ -160,7 +160,7 @@ def test_additional_properties(asserter, value, expected): @pytest.mark.parametrize('value, expected', [ ({'id': 1}, {'id': 1}), - ({'id': 'a'}, JsonSchemaException('data.id must be integer')), + ({'id': 'a'}, JsonSchemaException('data.id must be integer', value='a', name='data.id', definition={'type': 'integer'})), ]) def test_object_with_id_property(asserter, value, expected): asserter({ @@ -173,7 +173,7 @@ def test_object_with_id_property(asserter, value, expected): @pytest.mark.parametrize('value, expected', [ ({'$ref': 'ref://to.somewhere'}, {'$ref': 'ref://to.somewhere'}), - ({'$ref': 1}, JsonSchemaException('data.$ref must be string')), + ({'$ref': 1}, JsonSchemaException('data.$ref must be string', value=1, name='data.$ref', definition={'type': 'string'})), ]) def test_object_with_ref_property(asserter, value, expected): asserter({ diff --git a/tests/test_string.py b/tests/test_string.py index 7f2e5cd..dd87ba5 100644 --- a/tests/test_string.py +++ b/tests/test_string.py @@ -1,10 +1,9 @@ - import pytest from fastjsonschema import JsonSchemaException -exc = JsonSchemaException('data must be string') +exc = JsonSchemaException('data must be string', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ (0, exc), (None, exc), @@ -18,7 +17,7 @@ def test_string(asserter, value, expected): asserter({'type': 'string'}, value, expected) -exc = JsonSchemaException('data must be shorter than or equal to 5 characters') +exc = JsonSchemaException('data must be shorter than or equal to 5 characters', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ ('', ''), ('qwer', 'qwer'), @@ -33,7 +32,7 @@ def test_max_length(asserter, value, expected): }, value, expected) -exc = JsonSchemaException('data must be longer than or equal to 5 characters') +exc = JsonSchemaException('data must be longer than or equal to 5 characters', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ ('', exc), ('qwer', exc), @@ -48,7 +47,7 @@ def test_min_length(asserter, value, expected): }, value, expected) -exc = JsonSchemaException('data must match pattern ^[ab]*[^ab]+(c{2}|d)$') +exc = JsonSchemaException('data must match pattern ^[ab]*[^ab]+(c{2}|d)$', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ ('', exc), ('aacc', exc), @@ -84,7 +83,7 @@ def test_pattern_with_escape_no_warnings(asserter): assert len(record) == 0 -exc = JsonSchemaException('data must be a valid regex') +exc = JsonSchemaException('data must be a valid regex', value='{data}', name='data', definition='{definition}') @pytest.mark.parametrize('value, expected', [ ('[a-z]', '[a-z]'), ('[a-z', exc), From a0d31894b4d0810299654560fa6505397f8b4adc Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 8 Oct 2019 19:35:58 +0200 Subject: [PATCH 168/201] rule and rule_definition properties on Exception --- CHANGELOG.txt | 2 +- fastjsonschema/draft04.py | 58 ++++++++++++++++++------------------ fastjsonschema/draft06.py | 18 +++++------ fastjsonschema/exceptions.py | 21 +++++++++---- fastjsonschema/generator.py | 6 ++-- tests/conftest.py | 1 + tests/test_array.py | 22 +++++++------- tests/test_boolean.py | 2 +- tests/test_common.py | 14 ++++----- tests/test_default.py | 4 +-- tests/test_exceptions.py | 13 ++++++++ tests/test_format.py | 6 ++-- tests/test_integration.py | 16 +++++----- tests/test_null.py | 2 +- tests/test_number.py | 14 ++++----- tests/test_object.py | 28 ++++++++--------- tests/test_string.py | 10 +++---- 17 files changed, 130 insertions(+), 107 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 8777920..b8f2ca1 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,7 +1,7 @@ === 2.14 (unreleased) * Optimization: do not do the same type checks, keep it in one block if possible -* More context in JsonSchemaException (value, variable_name, variable_path and definition) +* More context in JsonSchemaException (value, variable_name, variable_path, definition, rule and rule_definition) * Possibility to pass custom formats * Raise JsonSchemaDefinitionException when definition of property is not valid * Fix of uniqueItems when used with other than array type diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 2885c9d..054b55e 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -89,7 +89,7 @@ def generate_type(self): extra = ' or isinstance({variable}, bool)'.format(variable=self._variable) with self.l('if not isinstance({variable}, ({})){}:', python_types, extra): - self.exc('{name} must be {}', ' or '.join(types)) + self.exc('{name} must be {}', ' or '.join(types), rule='type') def generate_enum(self): """ @@ -105,7 +105,7 @@ def generate_enum(self): if not isinstance(enum, (list, tuple)): raise JsonSchemaDefinitionException('enum must be an array') with self.l('if {variable} not in {enum}:'): - self.exc('{name} must be one of {}', self.e(enum)) + self.exc('{name} must be one of {}', self.e(enum), rule='enum') def generate_all_of(self): """ @@ -152,7 +152,7 @@ def generate_any_of(self): self.l('except JsonSchemaException: pass') with self.l('if not {variable}_any_of_count:', optimize=False): - self.exc('{name} must be valid by one of anyOf definition') + self.exc('{name} must be valid by one of anyOf definition', rule='anyOf') def generate_one_of(self): """ @@ -180,7 +180,7 @@ def generate_one_of(self): self.l('except JsonSchemaException: pass') with self.l('if {variable}_one_of_count != 1:'): - self.exc('{name} must be valid exactly by one of oneOf definition') + self.exc('{name} must be valid exactly by one of oneOf definition', rule='oneOf') def generate_not(self): """ @@ -197,18 +197,18 @@ def generate_not(self): """ not_definition = self._definition['not'] if not_definition is True: - self.exc('{name} must not be there') + self.exc('{name} must not be there', rule='not') elif not_definition is False: return elif not not_definition: with self.l('if {}:', self._variable): - self.exc('{name} must not be valid by not definition') + self.exc('{name} must not be valid by not definition', rule='not') else: with self.l('try:'): self.generate_func_code_block(not_definition, self._variable, self._variable_name) self.l('except JsonSchemaException: pass') with self.l('else:'): - self.exc('{name} must not be valid by not definition') + self.exc('{name} must not be valid by not definition', rule='not') def generate_min_length(self): with self.l('if isinstance({variable}, str):'): @@ -216,7 +216,7 @@ def generate_min_length(self): if not isinstance(self._definition['minLength'], int): raise JsonSchemaDefinitionException('minLength must be a number') with self.l('if {variable}_len < {minLength}:'): - self.exc('{name} must be longer than or equal to {minLength} characters') + self.exc('{name} must be longer than or equal to {minLength} characters', rule='minLength') def generate_max_length(self): with self.l('if isinstance({variable}, str):'): @@ -224,7 +224,7 @@ def generate_max_length(self): if not isinstance(self._definition['maxLength'], int): raise JsonSchemaDefinitionException('maxLength must be a number') with self.l('if {variable}_len > {maxLength}:'): - self.exc('{name} must be shorter than or equal to {maxLength} characters') + self.exc('{name} must be shorter than or equal to {maxLength} characters', rule='maxLength') def generate_pattern(self): with self.l('if isinstance({variable}, str):'): @@ -233,7 +233,7 @@ def generate_pattern(self): end_of_string_fixed_pattern = DOLLAR_FINDER.sub(r'\\Z', pattern) self._compile_regexps[pattern] = re.compile(end_of_string_fixed_pattern) with self.l('if not REGEX_PATTERNS[{}].search({variable}):', repr(pattern)): - self.exc('{name} must match pattern {}', safe_pattern) + self.exc('{name} must match pattern {}', safe_pattern, rule='pattern') def generate_format(self): """ @@ -254,7 +254,7 @@ def generate_format(self): self._generate_format(format_, format_ + '_re_pattern', custom_format) else: with self.l('if not custom_formats["{}"]({variable}):', format_): - self.exc('{name} must be {}', format_) + self.exc('{name} must be {}', format_, rule='format') elif format_ in self.FORMAT_REGEXS: format_regex = self.FORMAT_REGEXS[format_] self._generate_format(format_, format_ + '_re_pattern', format_regex) @@ -263,7 +263,7 @@ def generate_format(self): with self.l('try:'): self.l('re.compile({variable})') with self.l('except Exception:'): - self.exc('{name} must be a valid regex') + self.exc('{name} must be a valid regex', rule='format') else: raise JsonSchemaDefinitionException('Undefined format %s'.format(format_)) @@ -273,7 +273,7 @@ def _generate_format(self, format_name, regexp_name, regexp): if not regexp_name in self._compile_regexps: self._compile_regexps[regexp_name] = re.compile(regexp) with self.l('if not REGEX_PATTERNS["{}"].match({variable}):', regexp_name): - self.exc('{name} must be {}', format_name) + self.exc('{name} must be {}', format_name, rule='format') def generate_minimum(self): with self.l('if isinstance({variable}, (int, float)):'): @@ -281,10 +281,10 @@ def generate_minimum(self): raise JsonSchemaDefinitionException('minimum must be a number') if self._definition.get('exclusiveMinimum', False): with self.l('if {variable} <= {minimum}:'): - self.exc('{name} must be bigger than {minimum}') + self.exc('{name} must be bigger than {minimum}', rule='minimum') else: with self.l('if {variable} < {minimum}:'): - self.exc('{name} must be bigger than or equal to {minimum}') + self.exc('{name} must be bigger than or equal to {minimum}', rule='minimum') def generate_maximum(self): with self.l('if isinstance({variable}, (int, float)):'): @@ -292,10 +292,10 @@ def generate_maximum(self): raise JsonSchemaDefinitionException('maximum must be a number') if self._definition.get('exclusiveMaximum', False): with self.l('if {variable} >= {maximum}:'): - self.exc('{name} must be smaller than {maximum}') + self.exc('{name} must be smaller than {maximum}', rule='maximum') else: with self.l('if {variable} > {maximum}:'): - self.exc('{name} must be smaller than or equal to {maximum}') + self.exc('{name} must be smaller than or equal to {maximum}', rule='maximum') def generate_multiple_of(self): with self.l('if isinstance({variable}, (int, float)):'): @@ -303,7 +303,7 @@ def generate_multiple_of(self): raise JsonSchemaDefinitionException('multipleOf must be a number') self.l('quotient = {variable} / {multipleOf}') with self.l('if int(quotient) != quotient:'): - self.exc('{name} must be multiple of {multipleOf}') + self.exc('{name} must be multiple of {multipleOf}', rule='multipleOf') def generate_min_items(self): self.create_variable_is_list() @@ -312,7 +312,7 @@ def generate_min_items(self): raise JsonSchemaDefinitionException('minItems must be a number') self.create_variable_with_length() with self.l('if {variable}_len < {minItems}:'): - self.exc('{name} must contain at least {minItems} items') + self.exc('{name} must contain at least {minItems} items', rule='minItems') def generate_max_items(self): self.create_variable_is_list() @@ -321,7 +321,7 @@ def generate_max_items(self): raise JsonSchemaDefinitionException('maxItems must be a number') self.create_variable_with_length() with self.l('if {variable}_len > {maxItems}:'): - self.exc('{name} must contain less than or equal to {maxItems} items') + self.exc('{name} must contain less than or equal to {maxItems} items', rule='maxItems') def generate_unique_items(self): """ @@ -342,7 +342,7 @@ def generate_unique_items(self): with self.l('if {variable}_is_list:'): self.create_variable_with_length() with self.l('if {variable}_len > len(set(str({variable}_x) for {variable}_x in {variable})):'): - self.exc('{name} must contain unique items') + self.exc('{name} must contain unique items', rule='uniqueItems') def generate_items(self): """ @@ -371,7 +371,7 @@ def generate_items(self): self.create_variable_with_length() if items_definition is False: with self.l('if {variable}:'): - self.exc('{name} must not be there') + self.exc('{name} must not be there', rule='items') elif isinstance(items_definition, list): for idx, item_definition in enumerate(items_definition): with self.l('if {variable}_len > {}:', idx): @@ -387,7 +387,7 @@ def generate_items(self): if 'additionalItems' in self._definition: if self._definition['additionalItems'] is False: with self.l('if {variable}_len > {}:', len(items_definition)): - self.exc('{name} must contain only specified items') + self.exc('{name} must contain only specified items', rule='items') else: with self.l('for {variable}_x, {variable}_item in enumerate({variable}[{0}:], {0}):', len(items_definition)): self.generate_func_code_block( @@ -411,7 +411,7 @@ def generate_min_properties(self): raise JsonSchemaDefinitionException('minProperties must be a number') self.create_variable_with_length() with self.l('if {variable}_len < {minProperties}:'): - self.exc('{name} must contain at least {minProperties} properties') + self.exc('{name} must contain at least {minProperties} properties', rule='minProperties') def generate_max_properties(self): self.create_variable_is_dict() @@ -420,7 +420,7 @@ def generate_max_properties(self): raise JsonSchemaDefinitionException('maxProperties must be a number') self.create_variable_with_length() with self.l('if {variable}_len > {maxProperties}:'): - self.exc('{name} must contain less than or equal to {maxProperties} properties') + self.exc('{name} must contain less than or equal to {maxProperties} properties', rule='maxProperties') def generate_required(self): self.create_variable_is_dict() @@ -429,7 +429,7 @@ def generate_required(self): raise JsonSchemaDefinitionException('required must be an array') self.create_variable_with_length() with self.l('if not all(prop in {variable} for prop in {required}):'): - self.exc('{name} must contain {} properties', self.e(self._definition['required'])) + self.exc('{name} must contain {} properties', self.e(self._definition['required']), rule='required') def generate_properties(self): """ @@ -527,7 +527,7 @@ def generate_additional_properties(self): ) else: with self.l('if {variable}_keys:'): - self.exc('{name} must contain only specified properties') + self.exc('{name} must contain only specified properties', rule='additionalProperties') def generate_dependencies(self): """ @@ -555,10 +555,10 @@ def generate_dependencies(self): continue with self.l('if "{}" in {variable}_keys:', self.e(key)): if values is False: - self.exc('{} in {name} must not be there', key) + self.exc('{} in {name} must not be there', key, rule='dependencies') elif isinstance(values, list): for value in values: with self.l('if "{}" not in {variable}_keys:', self.e(value)): - self.exc('{name} missing dependency {} for {}', self.e(value), self.e(key)) + self.exc('{name} missing dependency {} for {}', self.e(value), self.e(key), rule='dependencies') else: self.generate_func_code_block(values, self._variable, self._variable_name, clear_variables=True) diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py index abca80b..adbf655 100644 --- a/fastjsonschema/draft06.py +++ b/fastjsonschema/draft06.py @@ -70,21 +70,21 @@ def generate_type(self): extra += ' or isinstance({variable}, bool)'.format(variable=self._variable) with self.l('if not isinstance({variable}, ({})){}:', python_types, extra): - self.exc('{name} must be {}', ' or '.join(types)) + self.exc('{name} must be {}', ' or '.join(types), rule='type') def generate_exclusive_minimum(self): with self.l('if isinstance({variable}, (int, float)):'): if not isinstance(self._definition['exclusiveMinimum'], (int, float)): raise JsonSchemaDefinitionException('exclusiveMinimum must be an integer or a float') with self.l('if {variable} <= {exclusiveMinimum}:'): - self.exc('{name} must be bigger than {exclusiveMinimum}') + self.exc('{name} must be bigger than {exclusiveMinimum}', rule='exclusiveMinimum') def generate_exclusive_maximum(self): with self.l('if isinstance({variable}, (int, float)):'): if not isinstance(self._definition['exclusiveMaximum'], (int, float)): raise JsonSchemaDefinitionException('exclusiveMaximum must be an integer or a float') with self.l('if {variable} >= {exclusiveMaximum}:'): - self.exc('{name} must be smaller than {exclusiveMaximum}') + self.exc('{name} must be smaller than {exclusiveMaximum}', rule='exclusiveMaximum') def generate_property_names(self): """ @@ -106,7 +106,7 @@ def generate_property_names(self): elif property_names_definition is False: self.create_variable_keys() with self.l('if {variable}_keys:'): - self.exc('{name} must not be there') + self.exc('{name} must not be there', rule='propertyNames') else: self.create_variable_is_dict() with self.l('if {variable}_is_dict:'): @@ -124,7 +124,7 @@ def generate_property_names(self): with self.l('except JsonSchemaException:'): self.l('{variable}_property_names = False') with self.l('if not {variable}_property_names:'): - self.exc('{name} must be named by propertyName definition') + self.exc('{name} must be named by propertyName definition', rule='propertyNames') def generate_contains(self): """ @@ -145,10 +145,10 @@ def generate_contains(self): contains_definition = self._definition['contains'] if contains_definition is False: - self.exc('{name} is always invalid') + self.exc('{name} is always invalid', rule='contains') elif contains_definition is True: with self.l('if not {variable}:'): - self.exc('{name} must not be empty') + self.exc('{name} must not be empty', rule='contains') else: self.l('{variable}_contains = False') with self.l('for {variable}_key in {variable}:'): @@ -164,7 +164,7 @@ def generate_contains(self): self.l('except JsonSchemaException: pass') with self.l('if not {variable}_contains:'): - self.exc('{name} must contain one of contains definition') + self.exc('{name} must contain one of contains definition', rule='contains') def generate_const(self): """ @@ -182,4 +182,4 @@ def generate_const(self): if isinstance(const, str): const = '"{}"'.format(const) with self.l('if {variable} != {}:', const): - self.exc('{name} must be same as const definition') + self.exc('{name} must be same as const definition', rule='const') diff --git a/fastjsonschema/exceptions.py b/fastjsonschema/exceptions.py index 0cb78e4..d33fb8f 100644 --- a/fastjsonschema/exceptions.py +++ b/fastjsonschema/exceptions.py @@ -8,24 +8,33 @@ class JsonSchemaException(ValueError): """ Exception raised by validation function. Available properties: - * ``message`` with information what is wrong, - * ``value`` of invalid data, - * ``name`` as a string with a path in the input, - * ``path`` as an array with a path in the input, - * and ``definition`` which was broken. + * ``message`` containing human-readable information what is wrong (e.g. ``data.property[index] must be smaller than or equal to 42``), + * invalid ``value`` (e.g. ``60``), + * ``name`` of a path in the data structure (e.g. ``data.propery[index]``), + * ``path`` as an array in the data structure (e.g. ``['data', 'propery', 'index']``), + * the whole ``definition`` which the ``value`` has to fulfil (e.g. ``{'type': 'number', 'maximum': 42}``), + * ``rule`` which the ``value`` is breaking (e.g. ``maximum``) + * and ``rule_definition`` (e.g. ``42``). """ - def __init__(self, message, value=None, name=None, definition=None): + def __init__(self, message, value=None, name=None, definition=None, rule=None): super().__init__(message) self.message = message self.value = value self.name = name self.definition = definition + self.rule = rule @property def path(self): return [item for item in SPLIT_RE.split(self.name) if item != ''] + @property + def rule_definition(self): + if not self.rule or not self.definition: + return None + return self.definition.get(self.rule) + class JsonSchemaDefinitionException(JsonSchemaException): """ diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index 23339fe..c5916c5 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -239,11 +239,11 @@ def e(self, string): """ return str(string).replace('"', '\\"') - def exc(self, msg, *args): + def exc(self, msg, *args, rule=None): """ """ - msg = 'raise JsonSchemaException("'+msg+'", value={variable}, name="{name}", definition={definition})' - self.l(msg, *args, definition=repr(self._definition)) + msg = 'raise JsonSchemaException("'+msg+'", value={variable}, name="{name}", definition={definition}, rule={rule})' + self.l(msg, *args, definition=repr(self._definition), rule=repr(rule)) def create_variable_with_length(self): """ diff --git a/tests/conftest.py b/tests/conftest.py index ca5af75..756429e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,6 +32,7 @@ def f(definition, value, expected, formats={}): assert exc.value.value == (value if expected.value == '{data}' else expected.value) assert exc.value.name == expected.name assert exc.value.definition == (definition if expected.definition == '{definition}' else expected.definition) + assert exc.value.rule == expected.rule else: assert validator(value) == expected return f diff --git a/tests/test_array.py b/tests/test_array.py index 4a1cc37..33f6a06 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -3,7 +3,7 @@ from fastjsonschema import JsonSchemaException -exc = JsonSchemaException('data must be array', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be array', value='{data}', name='data', definition='{definition}', rule='type') @pytest.mark.parametrize('value, expected', [ (0, exc), (None, exc), @@ -18,7 +18,7 @@ def test_array(asserter, value, expected): asserter({'type': 'array'}, value, expected) -exc = JsonSchemaException('data must contain less than or equal to 1 items', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must contain less than or equal to 1 items', value='{data}', name='data', definition='{definition}', rule='maxItems') @pytest.mark.parametrize('value, expected', [ ([], []), ([1], [1]), @@ -32,7 +32,7 @@ def test_max_items(asserter, value, expected): }, value, expected) -exc = JsonSchemaException('data must contain at least 2 items', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must contain at least 2 items', value='{data}', name='data', definition='{definition}', rule='minItems') @pytest.mark.parametrize('value, expected', [ ([], exc), ([1], exc), @@ -49,7 +49,7 @@ def test_min_items(asserter, value, expected): @pytest.mark.parametrize('value, expected', [ ([], []), ([1], [1]), - ([1, 1], JsonSchemaException('data must contain unique items', value='{data}', name='data', definition='{definition}')), + ([1, 1], JsonSchemaException('data must contain unique items', value='{data}', name='data', definition='{definition}', rule='uniqueItems')), ([1, 2, 3], [1, 2, 3]), ]) def test_unique_items(asserter, value, expected): @@ -71,7 +71,7 @@ def test_min_and_unique_items(asserter): @pytest.mark.parametrize('value, expected', [ ([], []), ([1], [1]), - ([1, 'a'], JsonSchemaException('data[1] must be number', value='a', name='data[1]', definition={'type': 'number'})), + ([1, 'a'], JsonSchemaException('data[1] must be number', value='a', name='data[1]', definition={'type': 'number'}, rule='type')), ]) def test_items_all_same(asserter, value, expected): asserter({ @@ -84,7 +84,7 @@ def test_items_all_same(asserter, value, expected): ([], []), ([1], [1]), ([1, 'a'], [1, 'a']), - ([1, 2], JsonSchemaException('data[1] must be string', value=2, name='data[1]', definition={'type': 'string'})), + ([1, 2], JsonSchemaException('data[1] must be string', value=2, name='data[1]', definition={'type': 'string'}, rule='type')), ([1, 'a', 2], [1, 'a', 2]), ([1, 'a', 'b'], [1, 'a', 'b']), ]) @@ -102,8 +102,8 @@ def test_different_items(asserter, value, expected): ([], []), ([1], [1]), ([1, 'a'], [1, 'a']), - ([1, 2], JsonSchemaException('data[1] must be string', value=2, name='data[1]', definition={'type': 'string'})), - ([1, 'a', 2], JsonSchemaException('data[2] must be string', value=2, name='data[2]', definition={'type': 'string'})), + ([1, 2], JsonSchemaException('data[1] must be string', value=2, name='data[1]', definition={'type': 'string'}, rule='type')), + ([1, 'a', 2], JsonSchemaException('data[2] must be string', value=2, name='data[2]', definition={'type': 'string'}, rule='type')), ([1, 'a', 'b'], [1, 'a', 'b']), ]) def test_different_items_with_additional_items(asserter, value, expected): @@ -121,9 +121,9 @@ def test_different_items_with_additional_items(asserter, value, expected): ([], []), ([1], [1]), ([1, 'a'], [1, 'a']), - ([1, 2], JsonSchemaException('data[1] must be string', value=2, name='data[1]', definition={'type': 'string'})), - ([1, 'a', 2], JsonSchemaException('data must contain only specified items', value='{data}', name='data', definition='{definition}')), - ([1, 'a', 'b'], JsonSchemaException('data must contain only specified items', value='{data}', name='data', definition='{definition}')), + ([1, 2], JsonSchemaException('data[1] must be string', value=2, name='data[1]', definition={'type': 'string'}, rule='type')), + ([1, 'a', 2], JsonSchemaException('data must contain only specified items', value='{data}', name='data', definition='{definition}', rule='items')), + ([1, 'a', 'b'], JsonSchemaException('data must contain only specified items', value='{data}', name='data', definition='{definition}', rule='items')), ]) def test_different_items_without_additional_items(asserter, value, expected): asserter({ diff --git a/tests/test_boolean.py b/tests/test_boolean.py index 6988c4f..c3c1765 100644 --- a/tests/test_boolean.py +++ b/tests/test_boolean.py @@ -3,7 +3,7 @@ from fastjsonschema import JsonSchemaException -exc = JsonSchemaException('data must be boolean', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be boolean', value='{data}', name='data', definition='{definition}', rule='type') @pytest.mark.parametrize('value, expected', [ (0, exc), (None, exc), diff --git a/tests/test_common.py b/tests/test_common.py index ad2fbab..bb0133b 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -3,7 +3,7 @@ from fastjsonschema import JsonSchemaException -exc = JsonSchemaException('data must be one of [1, 2, \'a\', "b\'c"]', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be one of [1, 2, \'a\', "b\'c"]', value='{data}', name='data', definition='{definition}', rule='enum') @pytest.mark.parametrize('value, expected', [ (1, 1), (2, 2), @@ -15,7 +15,7 @@ def test_enum(asserter, value, expected): asserter({'enum': [1, 2, 'a', "b'c"]}, value, expected) -exc = JsonSchemaException('data must be string or number', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be string or number', value='{data}', name='data', definition='{definition}', rule='type') @pytest.mark.parametrize('value, expected', [ (0, 0), (None, exc), @@ -30,7 +30,7 @@ def test_types(asserter, value, expected): @pytest.mark.parametrize('value, expected', [ ('qwert', 'qwert'), - ('qwertz', JsonSchemaException('data must be shorter than or equal to 5 characters', value='{data}', name='data', definition={'maxLength': 5})), + ('qwertz', JsonSchemaException('data must be shorter than or equal to 5 characters', value='{data}', name='data', definition={'maxLength': 5}, rule='maxLength')), ]) def test_all_of(asserter, value, expected): asserter({'allOf': [ @@ -39,7 +39,7 @@ def test_all_of(asserter, value, expected): ]}, value, expected) -exc = JsonSchemaException('data must be valid by one of anyOf definition', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be valid by one of anyOf definition', value='{data}', name='data', definition='{definition}', rule='anyOf') @pytest.mark.parametrize('value, expected', [ (0, 0), (None, exc), @@ -55,7 +55,7 @@ def test_any_of(asserter, value, expected): ]}, value, expected) -exc = JsonSchemaException('data must be valid exactly by one of oneOf definition', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be valid exactly by one of oneOf definition', value='{data}', name='data', definition='{definition}', rule='oneOf') @pytest.mark.parametrize('value, expected', [ (0, exc), (2, exc), @@ -70,7 +70,7 @@ def test_one_of(asserter, value, expected): ]}, value, expected) -exc = JsonSchemaException('data must be valid exactly by one of oneOf definition', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be valid exactly by one of oneOf definition', value='{data}', name='data', definition='{definition}', rule='oneOf') @pytest.mark.parametrize('value, expected', [ (0, exc), (2, exc), @@ -89,7 +89,7 @@ def test_one_of_factorized(asserter, value, expected): @pytest.mark.parametrize('value, expected', [ - (0, JsonSchemaException('data must not be valid by not definition', value='{data}', name='data', definition='{definition}')), + (0, JsonSchemaException('data must not be valid by not definition', value='{data}', name='data', definition='{definition}', rule='not')), (True, True), ('abc', 'abc'), ([], []), diff --git a/tests/test_default.py b/tests/test_default.py index 616e4a7..28449f1 100644 --- a/tests/test_default.py +++ b/tests/test_default.py @@ -4,7 +4,7 @@ @pytest.mark.parametrize('value, expected', [ - (None, JsonSchemaException('data must be object', value='{data}', name='data', definition='{definition}')), + (None, JsonSchemaException('data must be object', value='{data}', name='data', definition='{definition}', rule='type')), ({}, {'a': '', 'b': 42, 'c': {}, 'd': []}), ({'a': 'abc'}, {'a': 'abc', 'b': 42, 'c': {}, 'd': []}), ({'b': 123}, {'a': '', 'b': 123, 'c': {}, 'd': []}), @@ -23,7 +23,7 @@ def test_default_in_object(asserter, value, expected): @pytest.mark.parametrize('value, expected', [ - (None, JsonSchemaException('data must be array', value='{data}', name='data', definition='{definition}')), + (None, JsonSchemaException('data must be array', value='{data}', name='data', definition='{definition}', rule='type')), ([], ['', 42]), (['abc'], ['abc', 42]), (['abc', 123], ['abc', 123]), diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index e0482cd..8e8d0a6 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -15,3 +15,16 @@ def test_exception_variable_path(value, expected): exc = JsonSchemaException('msg', name=value) assert exc.path == expected + + +@pytest.mark.parametrize('definition, rule, expected_rule_definition', [ + (None, None, None), + ({}, None, None), + ({'type': 'string'}, None, None), + ({'type': 'string'}, 'unique', None), + ({'type': 'string'}, 'type', 'string'), + (None, 'type', None), +]) +def test_exception_rule_definition(definition, rule, expected_rule_definition): + exc = JsonSchemaException('msg', definition=definition, rule=rule) + assert exc.rule_definition == expected_rule_definition diff --git a/tests/test_format.py b/tests/test_format.py index 45a58ec..c309b57 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -6,7 +6,7 @@ from fastjsonschema import JsonSchemaException -exc = JsonSchemaException('data must be date-time', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be date-time', value='{data}', name='data', definition='{definition}', rule='format') @pytest.mark.parametrize('value, expected', [ ('', exc), ('bla', exc), @@ -19,7 +19,7 @@ def test_datetime(asserter, value, expected): asserter({'type': 'string', 'format': 'date-time'}, value, expected) -exc = JsonSchemaException('data must be hostname', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be hostname', value='{data}', name='data', definition='{definition}', rule='format') @pytest.mark.parametrize('value, expected', [ ('', exc), ('LDhsjf878&d', exc), @@ -36,7 +36,7 @@ def test_hostname(asserter, value, expected): asserter({'type': 'string', 'format': 'hostname'}, value, expected) -exc = JsonSchemaException('data must be custom-format', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be custom-format', value='{data}', name='data', definition='{definition}', rule='format') @pytest.mark.parametrize('value,expected,custom_format', [ ('', exc, r'^[ab]$'), ('', exc, lambda value: value in ('a', 'b')), diff --git a/tests/test_integration.py b/tests/test_integration.py index ec9d80b..5ea2c08 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -66,35 +66,35 @@ ), ( [10, 'world', [1], {'a': 'a', 'b': 'b', 'c': 'xy'}, 'str', 5], - JsonSchemaException('data[0] must be smaller than 10', value=10, name='data[0]', definition=definition['items'][0]), + JsonSchemaException('data[0] must be smaller than 10', value=10, name='data[0]', definition=definition['items'][0], rule='maximum'), ), ( [9, 'xxx', [1], {'a': 'a', 'b': 'b', 'c': 'xy'}, 'str', 5], - JsonSchemaException('data[1] must be one of [\'hello\', \'world\']', value='xxx', name='data[1]', definition=definition['items'][1]), + JsonSchemaException('data[1] must be one of [\'hello\', \'world\']', value='xxx', name='data[1]', definition=definition['items'][1], rule='enum'), ), ( [9, 'hello', [], {'a': 'a', 'b': 'b', 'c': 'xy'}, 'str', 5], - JsonSchemaException('data[2] must contain at least 1 items', value=[], name='data[2]', definition=definition['items'][2]), + JsonSchemaException('data[2] must contain at least 1 items', value=[], name='data[2]', definition=definition['items'][2], rule='minItems'), ), ( [9, 'hello', [1, 2, 3], {'a': 'a', 'b': 'b', 'c': 'xy'}, 'str', 5], - JsonSchemaException('data[2][1] must be string', value=2, name='data[2][1]', definition={'type': 'string'}), + JsonSchemaException('data[2][1] must be string', value=2, name='data[2][1]', definition={'type': 'string'}, rule='type'), ), ( [9, 'hello', [1], {'a': 'a', 'x': 'x', 'y': 'y'}, 'str', 5], - JsonSchemaException('data[3] must contain [\'a\', \'b\'] properties', value={'a': 'a', 'x': 'x', 'y': 'y'}, name='data[3]', definition=definition['items'][3]), + JsonSchemaException('data[3] must contain [\'a\', \'b\'] properties', value={'a': 'a', 'x': 'x', 'y': 'y'}, name='data[3]', definition=definition['items'][3], rule='required'), ), ( [9, 'hello', [1], {}, 'str', 5], - JsonSchemaException('data[3] must contain at least 3 properties', value={}, name='data[3]', definition=definition['items'][3]), + JsonSchemaException('data[3] must contain at least 3 properties', value={}, name='data[3]', definition=definition['items'][3], rule='minProperties'), ), ( [9, 'hello', [1], {'a': 'a', 'b': 'b', 'x': 'x'}, None, 5], - JsonSchemaException('data[4] must not be valid by not definition', value=None, name='data[4]', definition=definition['items'][4]), + JsonSchemaException('data[4] must not be valid by not definition', value=None, name='data[4]', definition=definition['items'][4], rule='not'), ), ( [9, 'hello', [1], {'a': 'a', 'b': 'b', 'x': 'x'}, 42, 15], - JsonSchemaException('data[5] must be valid exactly by one of oneOf definition', value=15, name='data[5]', definition=definition['items'][5]), + JsonSchemaException('data[5] must be valid exactly by one of oneOf definition', value=15, name='data[5]', definition=definition['items'][5], rule='oneOf'), ), ]) def test_integration(asserter, value, expected): diff --git a/tests/test_null.py b/tests/test_null.py index 985a722..969d12e 100644 --- a/tests/test_null.py +++ b/tests/test_null.py @@ -3,7 +3,7 @@ from fastjsonschema import JsonSchemaException -exc = JsonSchemaException('data must be null', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be null', value='{data}', name='data', definition='{definition}', rule='type') @pytest.mark.parametrize('value, expected', [ (0, exc), (None, None), diff --git a/tests/test_number.py b/tests/test_number.py index d77960e..88bad13 100644 --- a/tests/test_number.py +++ b/tests/test_number.py @@ -21,11 +21,11 @@ def number_type(request): ]) def test_number(asserter, number_type, value, expected): if isinstance(expected, JsonSchemaException): - expected = JsonSchemaException(expected.message.format(number_type=number_type), value='{data}', name='data', definition='{definition}') + expected = JsonSchemaException(expected.message.format(number_type=number_type), value='{data}', name='data', definition='{definition}', rule='type') asserter({'type': number_type}, value, expected) -exc = JsonSchemaException('data must be smaller than or equal to 10', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be smaller than or equal to 10', value='{data}', name='data', definition='{definition}', rule='maximum') @pytest.mark.parametrize('value, expected', [ (-5, -5), (5, 5), @@ -41,7 +41,7 @@ def test_maximum(asserter, number_type, value, expected): }, value, expected) -exc = JsonSchemaException('data must be smaller than 10', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be smaller than 10', value='{data}', name='data', definition='{definition}', rule='maximum') @pytest.mark.parametrize('value, expected', [ (-5, -5), (5, 5), @@ -58,7 +58,7 @@ def test_exclusive_maximum(asserter, number_type, value, expected): }, value, expected) -exc = JsonSchemaException('data must be bigger than or equal to 10', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be bigger than or equal to 10', value='{data}', name='data', definition='{definition}', rule='minimum') @pytest.mark.parametrize('value, expected', [ (-5, exc), (9, exc), @@ -73,7 +73,7 @@ def test_minimum(asserter, number_type, value, expected): }, value, expected) -exc = JsonSchemaException('data must be bigger than 10', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be bigger than 10', value='{data}', name='data', definition='{definition}', rule='minimum') @pytest.mark.parametrize('value, expected', [ (-5, exc), (9, exc), @@ -89,7 +89,7 @@ def test_exclusive_minimum(asserter, number_type, value, expected): }, value, expected) -exc = JsonSchemaException('data must be multiple of 3', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be multiple of 3', value='{data}', name='data', definition='{definition}', rule='multipleOf') @pytest.mark.parametrize('value, expected', [ (-4, exc), (-3, -3), @@ -120,7 +120,7 @@ def test_multiple_of(asserter, number_type, value, expected): def test_integer_is_not_number(asserter, value): asserter({ 'type': 'integer', - }, value, JsonSchemaException('data must be integer', value='{data}', name='data', definition='{definition}')) + }, value, JsonSchemaException('data must be integer', value='{data}', name='data', definition='{definition}', rule='type')) @pytest.mark.parametrize('value', ( diff --git a/tests/test_object.py b/tests/test_object.py index c6ac784..26ea4e2 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -4,7 +4,7 @@ from fastjsonschema import JsonSchemaDefinitionException, JsonSchemaException -exc = JsonSchemaException('data must be object', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be object', value='{data}', name='data', definition='{definition}', rule='type') @pytest.mark.parametrize('value, expected', [ (0, exc), (None, exc), @@ -22,7 +22,7 @@ def test_object(asserter, value, expected): @pytest.mark.parametrize('value, expected', [ ({}, {}), ({'a': 1}, {'a': 1}), - ({'a': 1, 'b': 2}, JsonSchemaException('data must contain less than or equal to 1 properties', value='{data}', name='data', definition='{definition}')), + ({'a': 1, 'b': 2}, JsonSchemaException('data must contain less than or equal to 1 properties', value='{data}', name='data', definition='{definition}', rule='maxProperties')), ]) def test_max_properties(asserter, value, expected): asserter({ @@ -32,7 +32,7 @@ def test_max_properties(asserter, value, expected): @pytest.mark.parametrize('value, expected', [ - ({}, JsonSchemaException('data must contain at least 1 properties', value='{data}', name='data', definition='{definition}')), + ({}, JsonSchemaException('data must contain at least 1 properties', value='{data}', name='data', definition='{definition}', rule='minProperties')), ({'a': 1}, {'a': 1}), ({'a': 1, 'b': 2}, {'a': 1, 'b': 2}), ]) @@ -43,7 +43,7 @@ def test_min_properties(asserter, value, expected): }, value, expected) -exc = JsonSchemaException('data must contain [\'a\', \'b\'] properties', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must contain [\'a\', \'b\'] properties', value='{data}', name='data', definition='{definition}', rule='required') @pytest.mark.parametrize('value, expected', [ ({}, exc), ({'a': 1}, exc), @@ -60,7 +60,7 @@ def test_required(asserter, value, expected): ({}, {}), ({'a': 1}, {'a': 1}), ({'a': 1, 'b': ''}, {'a': 1, 'b': ''}), - ({'a': 1, 'b': 2}, JsonSchemaException('data.b must be string', value=2, name='data.b', definition={'type': 'string'})), + ({'a': 1, 'b': 2}, JsonSchemaException('data.b must be string', value=2, name='data.b', definition={'type': 'string'}, rule='type')), ({'a': 1, 'b': '', 'any': True}, {'a': 1, 'b': '', 'any': True}), ]) def test_properties(asserter, value, expected): @@ -86,9 +86,9 @@ def test_invalid_properties(asserter): ({}, {}), ({'a': 1}, {'a': 1}), ({'a': 1, 'b': ''}, {'a': 1, 'b': ''}), - ({'a': 1, 'b': 2}, JsonSchemaException('data.b must be string', value=2, name='data.b', definition={'type': 'string'})), + ({'a': 1, 'b': 2}, JsonSchemaException('data.b must be string', value=2, name='data.b', definition={'type': 'string'}, rule='type')), ({'a': 1, 'b': '', 'additional': ''}, {'a': 1, 'b': '', 'additional': ''}), - ({'a': 1, 'b': '', 'any': True}, JsonSchemaException('data.any must be string', value=True, name='data.any', definition={'type': 'string'})), + ({'a': 1, 'b': '', 'any': True}, JsonSchemaException('data.any must be string', value=True, name='data.any', definition={'type': 'string'}, rule='type')), ]) def test_properties_with_additional_properties(asserter, value, expected): asserter({ @@ -105,9 +105,9 @@ def test_properties_with_additional_properties(asserter, value, expected): ({}, {}), ({'a': 1}, {'a': 1}), ({'a': 1, 'b': ''}, {'a': 1, 'b': ''}), - ({'a': 1, 'b': 2}, JsonSchemaException('data.b must be string', value=2, name='data.b', definition={'type': 'string'})), - ({'a': 1, 'b': '', 'any': True}, JsonSchemaException('data must contain only specified properties', value='{data}', name='data', definition='{definition}')), - ({'cd': True}, JsonSchemaException('data must contain only specified properties', value='{data}', name='data', definition='{definition}')), + ({'a': 1, 'b': 2}, JsonSchemaException('data.b must be string', value=2, name='data.b', definition={'type': 'string'}, rule='type')), + ({'a': 1, 'b': '', 'any': True}, JsonSchemaException('data must contain only specified properties', value='{data}', name='data', definition='{definition}', rule='additionalProperties')), + ({'cd': True}, JsonSchemaException('data must contain only specified properties', value='{data}', name='data', definition='{definition}', rule='additionalProperties')), ({'c_d': True}, {'c_d': True}), ]) def test_properties_without_additional_properties(asserter, value, expected): @@ -126,7 +126,7 @@ def test_properties_without_additional_properties(asserter, value, expected): ({}, {}), ({'a': 1}, {'a': 1}), ({'xa': 1}, {'xa': 1}), - ({'xa': ''}, JsonSchemaException('data.xa must be number', value='', name='data.xa', definition={'type': 'number'})), + ({'xa': ''}, JsonSchemaException('data.xa must be number', value='', name='data.xa', definition={'type': 'number'}, rule='type')), ({'xbx': ''}, {'xbx': ''}), ]) def test_pattern_properties(asserter, value, expected): @@ -145,7 +145,7 @@ def test_pattern_properties(asserter, value, expected): ({'a': 1}, {'a': 1}), ({'b': True}, {'b': True}), ({'c': ''}, {'c': ''}), - ({'d': 1}, JsonSchemaException('data.d must be string', value=1, name='data.d', definition={'type': 'string'})), + ({'d': 1}, JsonSchemaException('data.d must be string', value=1, name='data.d', definition={'type': 'string'}, rule='type')), ]) def test_additional_properties(asserter, value, expected): asserter({ @@ -160,7 +160,7 @@ def test_additional_properties(asserter, value, expected): @pytest.mark.parametrize('value, expected', [ ({'id': 1}, {'id': 1}), - ({'id': 'a'}, JsonSchemaException('data.id must be integer', value='a', name='data.id', definition={'type': 'integer'})), + ({'id': 'a'}, JsonSchemaException('data.id must be integer', value='a', name='data.id', definition={'type': 'integer'}, rule='type')), ]) def test_object_with_id_property(asserter, value, expected): asserter({ @@ -173,7 +173,7 @@ def test_object_with_id_property(asserter, value, expected): @pytest.mark.parametrize('value, expected', [ ({'$ref': 'ref://to.somewhere'}, {'$ref': 'ref://to.somewhere'}), - ({'$ref': 1}, JsonSchemaException('data.$ref must be string', value=1, name='data.$ref', definition={'type': 'string'})), + ({'$ref': 1}, JsonSchemaException('data.$ref must be string', value=1, name='data.$ref', definition={'type': 'string'}, rule='type')), ]) def test_object_with_ref_property(asserter, value, expected): asserter({ diff --git a/tests/test_string.py b/tests/test_string.py index dd87ba5..139ec6c 100644 --- a/tests/test_string.py +++ b/tests/test_string.py @@ -3,7 +3,7 @@ from fastjsonschema import JsonSchemaException -exc = JsonSchemaException('data must be string', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be string', value='{data}', name='data', definition='{definition}', rule='type') @pytest.mark.parametrize('value, expected', [ (0, exc), (None, exc), @@ -17,7 +17,7 @@ def test_string(asserter, value, expected): asserter({'type': 'string'}, value, expected) -exc = JsonSchemaException('data must be shorter than or equal to 5 characters', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be shorter than or equal to 5 characters', value='{data}', name='data', definition='{definition}', rule='maxLength') @pytest.mark.parametrize('value, expected', [ ('', ''), ('qwer', 'qwer'), @@ -32,7 +32,7 @@ def test_max_length(asserter, value, expected): }, value, expected) -exc = JsonSchemaException('data must be longer than or equal to 5 characters', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be longer than or equal to 5 characters', value='{data}', name='data', definition='{definition}', rule='minLength') @pytest.mark.parametrize('value, expected', [ ('', exc), ('qwer', exc), @@ -47,7 +47,7 @@ def test_min_length(asserter, value, expected): }, value, expected) -exc = JsonSchemaException('data must match pattern ^[ab]*[^ab]+(c{2}|d)$', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must match pattern ^[ab]*[^ab]+(c{2}|d)$', value='{data}', name='data', definition='{definition}', rule='pattern') @pytest.mark.parametrize('value, expected', [ ('', exc), ('aacc', exc), @@ -83,7 +83,7 @@ def test_pattern_with_escape_no_warnings(asserter): assert len(record) == 0 -exc = JsonSchemaException('data must be a valid regex', value='{data}', name='data', definition='{definition}') +exc = JsonSchemaException('data must be a valid regex', value='{data}', name='data', definition='{definition}', rule='format') @pytest.mark.parametrize('value, expected', [ ('[a-z]', '[a-z]'), ('[a-z', exc), From 7def41ce986ecff821d5add1733907094da42434 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 8 Oct 2019 19:39:03 +0200 Subject: [PATCH 169/201] Version 2.14.0 --- CHANGELOG.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index b8f2ca1..7dcaaf6 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,4 +1,4 @@ -=== 2.14 (unreleased) +=== 2.14.0 (2019-10-08) * Optimization: do not do the same type checks, keep it in one block if possible * More context in JsonSchemaException (value, variable_name, variable_path, definition, rule and rule_definition) From 83494f39bb54f5b4819a81dd3d348c6608ee6672 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Tue, 8 Oct 2019 19:41:27 +0200 Subject: [PATCH 170/201] Version 2.14.0 --- fastjsonschema/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index dbf4c44..38dd49a 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.13' +VERSION = '2.14.0' From c2bfae3ab0730d4abe76e3115e4cae4410a7c0a1 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 9 Oct 2019 08:52:01 +0200 Subject: [PATCH 171/201] Fix of undefined format exception message --- CHANGELOG.txt | 5 +++++ fastjsonschema/draft04.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 7dcaaf6..701d5bf 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +=== unreleased + +* Fix of undefined format exception message + + === 2.14.0 (2019-10-08) * Optimization: do not do the same type checks, keep it in one block if possible diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 054b55e..1ea2ed0 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -265,7 +265,7 @@ def generate_format(self): with self.l('except Exception:'): self.exc('{name} must be a valid regex', rule='format') else: - raise JsonSchemaDefinitionException('Undefined format %s'.format(format_)) + raise JsonSchemaDefinitionException('Unknown format: {}'.format(format_)) def _generate_format(self, format_name, regexp_name, regexp): From c4d648ff847df5f20009b997dc896f4882a63a3c Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 9 Oct 2019 08:59:06 +0200 Subject: [PATCH 172/201] Version 2.14.1 --- CHANGELOG.txt | 2 +- fastjsonschema/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 701d5bf..405800d 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,4 +1,4 @@ -=== unreleased +=== 2.14.1 (2019-10-09) * Fix of undefined format exception message diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index 38dd49a..93e67f9 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.14.0' +VERSION = '2.14.1' From 82793ed8525fc1527dfe916b4f3927b8e5595f21 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 9 Oct 2019 09:10:51 +0200 Subject: [PATCH 173/201] Better docs --- fastjsonschema/exceptions.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fastjsonschema/exceptions.py b/fastjsonschema/exceptions.py index d33fb8f..b555356 100644 --- a/fastjsonschema/exceptions.py +++ b/fastjsonschema/exceptions.py @@ -15,6 +15,9 @@ class JsonSchemaException(ValueError): * the whole ``definition`` which the ``value`` has to fulfil (e.g. ``{'type': 'number', 'maximum': 42}``), * ``rule`` which the ``value`` is breaking (e.g. ``maximum``) * and ``rule_definition`` (e.g. ``42``). + + .. versionchanged:: 2.14.0 + Added all extra properties. """ def __init__(self, message, value=None, name=None, definition=None, rule=None): From 52071a81b2913088d8f8420c6523e2b2231418aa Mon Sep 17 00:00:00 2001 From: Greg Kuhlmann Date: Fri, 11 Oct 2019 22:57:59 -0500 Subject: [PATCH 174/201] Minor correction to documentation --- fastjsonschema/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 08cc459..0795a0a 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -95,7 +95,7 @@ def validate(definition, data, handlers={}, formats={}): import fastjsonschema - validate({'type': 'string'}, 'hello') + fastjsonschema.validate({'type': 'string'}, 'hello') # same as: compile({'type': 'string'})('hello') Preferred is to use :any:`compile` function. From a0754ebc3676993e6fdb3f7e68f96ca22ce7aec4 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Sun, 27 Oct 2019 11:25:51 +0100 Subject: [PATCH 175/201] Fiix of additionalProperties set to true in JSON schema 4 --- CHANGELOG.txt | 5 +++++ fastjsonschema/draft04.py | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 405800d..82f789b 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +=== unreleased + +* Fix of `additionalProperties=true` for JSON schema 4 + + === 2.14.1 (2019-10-09) * Fix of undefined format exception message diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 1ea2ed0..824a870 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -515,7 +515,9 @@ def generate_additional_properties(self): with self.l('if {variable}_is_dict:'): self.create_variable_keys() add_prop_definition = self._definition["additionalProperties"] - if add_prop_definition: + if add_prop_definition == True: + return + elif add_prop_definition: properties_keys = list(self._definition.get("properties", {}).keys()) with self.l('for {variable}_key in {variable}_keys:'): with self.l('if {variable}_key not in {}:', properties_keys): From 5104e25bae58eb290ad0d0044034aef03f9b5955 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 8 Nov 2019 21:26:06 -0500 Subject: [PATCH 176/201] include LICENSE in distributed tarball --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..1aba38f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include LICENSE From fd1706ffc343effa7eb8aeee28e117522351fed3 Mon Sep 17 00:00:00 2001 From: Konstantin Valetov Date: Fri, 29 Nov 2019 16:26:12 +0300 Subject: [PATCH 177/201] try: block should not be optimized --- fastjsonschema/draft04.py | 8 ++++---- tests/test_compile_to_code.py | 37 +++++++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 824a870..f97a93b 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -146,7 +146,7 @@ def generate_any_of(self): for definition_item in self._definition['anyOf']: # When we know it's passing (at least once), we do not need to do another expensive try-except. with self.l('if not {variable}_any_of_count:', optimize=False): - with self.l('try:'): + with self.l('try:', optimize=False): self.generate_func_code_block(definition_item, self._variable, self._variable_name, clear_variables=True) self.l('{variable}_any_of_count += 1') self.l('except JsonSchemaException: pass') @@ -174,7 +174,7 @@ def generate_one_of(self): for definition_item in self._definition['oneOf']: # When we know it's failing (one of means exactly once), we do not need to do another expensive try-except. with self.l('if {variable}_one_of_count < 2:', optimize=False): - with self.l('try:'): + with self.l('try:', optimize=False): self.generate_func_code_block(definition_item, self._variable, self._variable_name, clear_variables=True) self.l('{variable}_one_of_count += 1') self.l('except JsonSchemaException: pass') @@ -204,7 +204,7 @@ def generate_not(self): with self.l('if {}:', self._variable): self.exc('{name} must not be valid by not definition', rule='not') else: - with self.l('try:'): + with self.l('try:', optimize=False): self.generate_func_code_block(not_definition, self._variable, self._variable_name) self.l('except JsonSchemaException: pass') with self.l('else:'): @@ -260,7 +260,7 @@ def generate_format(self): self._generate_format(format_, format_ + '_re_pattern', format_regex) # Format regex is used only in meta schemas. elif format_ == 'regex': - with self.l('try:'): + with self.l('try:', optimize=False): self.l('re.compile({variable})') with self.l('except Exception:'): self.exc('{name} must be a valid regex', rule='format') diff --git a/tests/test_compile_to_code.py b/tests/test_compile_to_code.py index 079f358..509e044 100644 --- a/tests/test_compile_to_code.py +++ b/tests/test_compile_to_code.py @@ -2,7 +2,7 @@ import pytest import shutil -from fastjsonschema import JsonSchemaException, compile_to_code +from fastjsonschema import compile_to_code, compile as compile_spec @pytest.yield_fixture(autouse=True) def run_around_tests(): @@ -50,4 +50,37 @@ def test_compile_to_code_ipv6_regex(): 'ip': '2001:0db8:85a3:0000:0000:8a2e:0370:7334' }) == { 'ip': '2001:0db8:85a3:0000:0000:8a2e:0370:7334' - } \ No newline at end of file + } + +# https://github.com/horejsek/python-fastjsonschema/issues/74 +def test_compile_complex_one_of_all_of(): + compile_spec({ + "oneOf": [ + { + "required": [ + "schema" + ] + }, + { + "required": [ + "content" + ], + "allOf": [ + { + "not": { + "required": [ + "style" + ] + } + }, + { + "not": { + "required": [ + "explode" + ] + } + } + ] + } + ] + }) From 2fc222c2a24108d53b5729f4c69a302276e2d48f Mon Sep 17 00:00:00 2001 From: leiserfg Date: Mon, 9 Dec 2019 22:25:01 +0100 Subject: [PATCH 178/201] Use decimal for multipleOf implementation and add respective tests --- fastjsonschema/draft04.py | 3 ++- tests/test_number.py | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index f97a93b..f972174 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -301,7 +301,8 @@ def generate_multiple_of(self): with self.l('if isinstance({variable}, (int, float)):'): if not isinstance(self._definition['multipleOf'], (int, float)): raise JsonSchemaDefinitionException('multipleOf must be a number') - self.l('quotient = {variable} / {multipleOf}') + self.l('from decimal import Decimal') + self.l('quotient = Decimal(repr({variable})) / Decimal(repr({multipleOf}))') with self.l('if int(quotient) != quotient:'): self.exc('{name} must be multiple of {multipleOf}', rule='multipleOf') diff --git a/tests/test_number.py b/tests/test_number.py index 88bad13..dd2f6b0 100644 --- a/tests/test_number.py +++ b/tests/test_number.py @@ -111,6 +111,30 @@ def test_multiple_of(asserter, number_type, value, expected): }, value, expected) + +exc = JsonSchemaException('data must be multiple of 0.0001', value='{data}', name='data', definition='{definition}', rule='multipleOf') +@pytest.mark.parametrize('value, expected', [ + (0.00751, exc), + (0.0075, 0.0075), +]) +def test_multiple_of_float(asserter, value, expected): + asserter({ + 'type': 'number', + 'multipleOf': 0.0001, + }, value, expected) + +exc = JsonSchemaException('data must be multiple of 1.5', value='{data}', name='data', definition='{definition}', rule='multipleOf') +@pytest.mark.parametrize('value, expected', [ + (0, 0), + (4.5, 4.5), + (35, exc), +]) +def test_multiple_of_float_1_5(asserter, value, expected): + asserter({ + 'type': 'number', + 'multipleOf': 1.5, + }, value, expected) + @pytest.mark.parametrize('value', ( 1.0, 0.1, From 2aee6eb286428cd97a456425e6aadff607f67d8e Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 11 Dec 2019 16:40:26 +0100 Subject: [PATCH 179/201] Use Decimal for multipleOf when the value is float --- CHANGELOG.txt | 1 + fastjsonschema/draft04.py | 11 +++++++++-- fastjsonschema/generator.py | 37 ++++++++++++++++++++----------------- tests/json_schema/utils.py | 5 ++++- tests/test_number.py | 14 +++++++++----- 5 files changed, 43 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 82f789b..b956629 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,7 @@ === unreleased * Fix of `additionalProperties=true` for JSON schema 4 +* Use decimal for multipleOf implementation and add respective tests === 2.14.1 (2019-10-09) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index f972174..01cc07e 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -1,3 +1,4 @@ +import decimal import re from .exceptions import JsonSchemaDefinitionException @@ -301,8 +302,14 @@ def generate_multiple_of(self): with self.l('if isinstance({variable}, (int, float)):'): if not isinstance(self._definition['multipleOf'], (int, float)): raise JsonSchemaDefinitionException('multipleOf must be a number') - self.l('from decimal import Decimal') - self.l('quotient = Decimal(repr({variable})) / Decimal(repr({multipleOf}))') + # For proper multiplication check of floats we need to use decimals, + # because for example 19.01 / 0.01 = 1901.0000000000002. + if isinstance(self._definition['multipleOf'], float): + self._extra_imports_lines.append('from decimal import Decimal') + self._extra_imports_objects['Decimal'] = decimal.Decimal + self.l('quotient = Decimal(repr({variable})) / Decimal(repr({multipleOf}))') + else: + self.l('quotient = {variable} / {multipleOf}') with self.l('if int(quotient) != quotient:'): self.exc('{name} must be multiple of {multipleOf}', rule='multipleOf') diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index c5916c5..fc5b4b1 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -32,6 +32,12 @@ def __init__(self, definition, resolver=None): self._code = [] self._compile_regexps = {} + # Any extra library should be here to be imported only once. + # Lines are imports to be printed in the file and objects + # key-value pair to pass to compile function directly. + self._extra_imports_lines = [] + self._extra_imports_objects = {} + self._variables = set() self._indent = 0 self._indent_last_line = None @@ -74,6 +80,7 @@ def global_state(self): self._generate_func_code() return dict( + **self._extra_imports_objects, REGEX_PATTERNS=self._compile_regexps, re=re, JsonSchemaException=JsonSchemaException, @@ -88,26 +95,22 @@ def global_state_code(self): self._generate_func_code() if not self._compile_regexps: - return '\n'.join( - [ - 'from fastjsonschema import JsonSchemaException', - '', - '', - ] - ) - regexs = ['"{}": re.compile(r"{}")'.format(key, value.pattern) for key, value in self._compile_regexps.items()] - return '\n'.join( - [ - 'import re', + return '\n'.join(self._extra_imports_lines + [ 'from fastjsonschema import JsonSchemaException', '', '', - 'REGEX_PATTERNS = {', - ' ' + ',\n '.join(regexs), - '}', - '', - ] - ) + ]) + regexs = ['"{}": re.compile(r"{}")'.format(key, value.pattern) for key, value in self._compile_regexps.items()] + return '\n'.join(self._extra_imports_lines + [ + 'import re', + 'from fastjsonschema import JsonSchemaException', + '', + '', + 'REGEX_PATTERNS = {', + ' ' + ',\n '.join(regexs), + '}', + '', + ]) def _generate_func_code(self): diff --git a/tests/json_schema/utils.py b/tests/json_schema/utils.py index d6a91a5..ef6d516 100644 --- a/tests/json_schema/utils.py +++ b/tests/json_schema/utils.py @@ -67,7 +67,10 @@ def template_test(schema_version, schema, data, is_valid): """ # For debug purposes. When test fails, it will print stdout. resolver = RefResolver.from_schema(schema, handlers={'http': remotes_handler}) - print(_get_code_generator_class(schema_version)(schema, resolver=resolver).func_code) + + debug_generator = _get_code_generator_class(schema_version)(schema, resolver=resolver) + print(debug_generator.global_state_code) + print(debug_generator.func_code) # JSON schema test suits do not contain schema version. # Our library needs to know that or it would use always the latest implementation. diff --git a/tests/test_number.py b/tests/test_number.py index dd2f6b0..12f5861 100644 --- a/tests/test_number.py +++ b/tests/test_number.py @@ -111,7 +111,6 @@ def test_multiple_of(asserter, number_type, value, expected): }, value, expected) - exc = JsonSchemaException('data must be multiple of 0.0001', value='{data}', name='data', definition='{definition}', rule='multipleOf') @pytest.mark.parametrize('value, expected', [ (0.00751, exc), @@ -123,18 +122,23 @@ def test_multiple_of_float(asserter, value, expected): 'multipleOf': 0.0001, }, value, expected) -exc = JsonSchemaException('data must be multiple of 1.5', value='{data}', name='data', definition='{definition}', rule='multipleOf') + +exc = JsonSchemaException('data must be multiple of 0.01', value='{data}', name='data', definition='{definition}', rule='multipleOf') @pytest.mark.parametrize('value, expected', [ (0, 0), - (4.5, 4.5), - (35, exc), + (0.01, 0.01), + (0.1, 0.1), + (19.01, 19.01), + (0.001, exc), + (19.001, exc), ]) def test_multiple_of_float_1_5(asserter, value, expected): asserter({ 'type': 'number', - 'multipleOf': 1.5, + 'multipleOf': 0.01, }, value, expected) + @pytest.mark.parametrize('value', ( 1.0, 0.1, From 7e949967063d29636187b8ccd846f102f46eb957 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 11 Dec 2019 16:59:46 +0100 Subject: [PATCH 180/201] Better escaping of definition names --- CHANGELOG.txt | 3 +- fastjsonschema/ref_resolver.py | 320 ++++++++++++++++----------------- 2 files changed, 162 insertions(+), 161 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index b956629..4dfc74b 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,7 +1,8 @@ -=== unreleased +=== 2.14.2 (2019-12-11) * Fix of `additionalProperties=true` for JSON schema 4 * Use decimal for multipleOf implementation and add respective tests +* Better escaping of definition names === 2.14.1 (2019-10-09) diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index d82973a..fe5979f 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -1,160 +1,160 @@ -""" -JSON Schema URI resolution scopes and dereferencing - -https://tools.ietf.org/id/draft-zyp-json-schema-04.html#rfc.section.7 - -Code adapted from https://github.com/Julian/jsonschema -""" - -import contextlib -import json -import re -from urllib import parse as urlparse -from urllib.parse import unquote -from urllib.request import urlopen - - -from .exceptions import JsonSchemaException - - -def resolve_path(schema, fragment): - """ - Return definition from path. - - Path is unescaped according https://tools.ietf.org/html/rfc6901 - """ - fragment = fragment.lstrip('/') - parts = unquote(fragment).split('/') if fragment else [] - for part in parts: - part = part.replace('~1', '/').replace('~0', '~') - if isinstance(schema, list): - schema = schema[int(part)] - elif part in schema: - schema = schema[part] - else: - raise JsonSchemaException('Unresolvable ref: {}'.format(part)) - return schema - - -def normalize(uri): - return urlparse.urlsplit(uri).geturl() - - -def resolve_remote(uri, handlers): - """ - Resolve a remote ``uri``. - - .. note:: - - urllib library is used to fetch requests from the remote ``uri`` - if handlers does notdefine otherwise. - """ - scheme = urlparse.urlsplit(uri).scheme - if scheme in handlers: - result = handlers[scheme](uri) - else: - req = urlopen(uri) - encoding = req.info().get_content_charset() or 'utf-8' - result = json.loads(req.read().decode(encoding),) - return result - - -class RefResolver: - """ - Resolve JSON References. - """ - - # pylint: disable=dangerous-default-value,too-many-arguments - def __init__(self, base_uri, schema, store={}, cache=True, handlers={}): - """ - `base_uri` is URI of the referring document from the `schema`. - """ - self.base_uri = base_uri - self.resolution_scope = base_uri - self.schema = schema - self.store = store - self.cache = cache - self.handlers = handlers - self.walk(schema) - - @classmethod - def from_schema(cls, schema, handlers={}, **kwargs): - """ - Construct a resolver from a JSON schema object. - """ - return cls( - schema.get('$id', schema.get('id', '')) if isinstance(schema, dict) else '', - schema, - handlers=handlers, - **kwargs - ) - - @contextlib.contextmanager - def in_scope(self, scope: str): - """ - Context manager to handle current scope. - """ - old_scope = self.resolution_scope - self.resolution_scope = urlparse.urljoin(old_scope, scope) - try: - yield - finally: - self.resolution_scope = old_scope - - @contextlib.contextmanager - def resolving(self, ref: str): - """ - Context manager which resolves a JSON ``ref`` and enters the - resolution scope of this ref. - """ - new_uri = urlparse.urljoin(self.resolution_scope, ref) - uri, fragment = urlparse.urldefrag(new_uri) - - if normalize(uri) in self.store: - schema = self.store[normalize(uri)] - elif not uri or uri == self.base_uri: - schema = self.schema - else: - schema = resolve_remote(uri, self.handlers) - if self.cache: - self.store[normalize(uri)] = schema - - old_base_uri, old_schema = self.base_uri, self.schema - self.base_uri, self.schema = uri, schema - try: - with self.in_scope(uri): - yield resolve_path(schema, fragment) - finally: - self.base_uri, self.schema = old_base_uri, old_schema - - def get_uri(self): - return normalize(self.resolution_scope) - - def get_scope_name(self): - """ - Get current scope and return it as a valid function name. - """ - name = 'validate_' + unquote(self.resolution_scope).replace('~1', '_').replace('~0', '_').replace('"', '') - name = re.sub(r'[:/#\.\-\%]', '_', name) - name = name.lower().rstrip('_') - return name - - def walk(self, node: dict): - """ - Walk thru schema and dereferencing ``id`` and ``$ref`` instances - """ - if isinstance(node, bool): - pass - elif '$ref' in node and isinstance(node['$ref'], str): - ref = node['$ref'] - node['$ref'] = urlparse.urljoin(self.resolution_scope, ref) - elif 'id' in node and isinstance(node['id'], str): - with self.in_scope(node['id']): - self.store[normalize(self.resolution_scope)] = node - for _, item in node.items(): - if isinstance(item, dict): - self.walk(item) - else: - for _, item in node.items(): - if isinstance(item, dict): - self.walk(item) +""" +JSON Schema URI resolution scopes and dereferencing + +https://tools.ietf.org/id/draft-zyp-json-schema-04.html#rfc.section.7 + +Code adapted from https://github.com/Julian/jsonschema +""" + +import contextlib +import json +import re +from urllib import parse as urlparse +from urllib.parse import unquote +from urllib.request import urlopen + + +from .exceptions import JsonSchemaException + + +def resolve_path(schema, fragment): + """ + Return definition from path. + + Path is unescaped according https://tools.ietf.org/html/rfc6901 + """ + fragment = fragment.lstrip('/') + parts = unquote(fragment).split('/') if fragment else [] + for part in parts: + part = part.replace('~1', '/').replace('~0', '~') + if isinstance(schema, list): + schema = schema[int(part)] + elif part in schema: + schema = schema[part] + else: + raise JsonSchemaException('Unresolvable ref: {}'.format(part)) + return schema + + +def normalize(uri): + return urlparse.urlsplit(uri).geturl() + + +def resolve_remote(uri, handlers): + """ + Resolve a remote ``uri``. + + .. note:: + + urllib library is used to fetch requests from the remote ``uri`` + if handlers does notdefine otherwise. + """ + scheme = urlparse.urlsplit(uri).scheme + if scheme in handlers: + result = handlers[scheme](uri) + else: + req = urlopen(uri) + encoding = req.info().get_content_charset() or 'utf-8' + result = json.loads(req.read().decode(encoding),) + return result + + +class RefResolver: + """ + Resolve JSON References. + """ + + # pylint: disable=dangerous-default-value,too-many-arguments + def __init__(self, base_uri, schema, store={}, cache=True, handlers={}): + """ + `base_uri` is URI of the referring document from the `schema`. + """ + self.base_uri = base_uri + self.resolution_scope = base_uri + self.schema = schema + self.store = store + self.cache = cache + self.handlers = handlers + self.walk(schema) + + @classmethod + def from_schema(cls, schema, handlers={}, **kwargs): + """ + Construct a resolver from a JSON schema object. + """ + return cls( + schema.get('$id', schema.get('id', '')) if isinstance(schema, dict) else '', + schema, + handlers=handlers, + **kwargs + ) + + @contextlib.contextmanager + def in_scope(self, scope: str): + """ + Context manager to handle current scope. + """ + old_scope = self.resolution_scope + self.resolution_scope = urlparse.urljoin(old_scope, scope) + try: + yield + finally: + self.resolution_scope = old_scope + + @contextlib.contextmanager + def resolving(self, ref: str): + """ + Context manager which resolves a JSON ``ref`` and enters the + resolution scope of this ref. + """ + new_uri = urlparse.urljoin(self.resolution_scope, ref) + uri, fragment = urlparse.urldefrag(new_uri) + + if normalize(uri) in self.store: + schema = self.store[normalize(uri)] + elif not uri or uri == self.base_uri: + schema = self.schema + else: + schema = resolve_remote(uri, self.handlers) + if self.cache: + self.store[normalize(uri)] = schema + + old_base_uri, old_schema = self.base_uri, self.schema + self.base_uri, self.schema = uri, schema + try: + with self.in_scope(uri): + yield resolve_path(schema, fragment) + finally: + self.base_uri, self.schema = old_base_uri, old_schema + + def get_uri(self): + return normalize(self.resolution_scope) + + def get_scope_name(self): + """ + Get current scope and return it as a valid function name. + """ + name = 'validate_' + unquote(self.resolution_scope).replace('~1', '_').replace('~0', '_').replace('"', '') + name = re.sub(r'($[^a-zA-Z]|[^a-zA-Z0-9])', '_', name) + name = name.lower().rstrip('_') + return name + + def walk(self, node: dict): + """ + Walk thru schema and dereferencing ``id`` and ``$ref`` instances + """ + if isinstance(node, bool): + pass + elif '$ref' in node and isinstance(node['$ref'], str): + ref = node['$ref'] + node['$ref'] = urlparse.urljoin(self.resolution_scope, ref) + elif 'id' in node and isinstance(node['id'], str): + with self.in_scope(node['id']): + self.store[normalize(self.resolution_scope)] = node + for _, item in node.items(): + if isinstance(item, dict): + self.walk(item) + else: + for _, item in node.items(): + if isinstance(item, dict): + self.walk(item) From ba998abe175463db7482c5b89c9f032043ef67f2 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 11 Dec 2019 17:00:24 +0100 Subject: [PATCH 181/201] Version 2.14.2 --- fastjsonschema/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index 93e67f9..094e9da 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.14.1' +VERSION = '2.14.2' From 9b59b9568f28e67f56621837c918a04e1f25310e Mon Sep 17 00:00:00 2001 From: Konstantin Valetov Date: Wed, 18 Dec 2019 12:25:45 +0300 Subject: [PATCH 182/201] fair performance test + robust script --- performance.py | 43 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/performance.py b/performance.py index 8669acd..31824a3 100644 --- a/performance.py +++ b/performance.py @@ -1,5 +1,7 @@ -from textwrap import dedent +import importlib.util import timeit +import tempfile +from textwrap import dedent # apt-get install jsonschema json-spec validictory import fastjsonschema @@ -72,14 +74,35 @@ fastjsonschema_validate = fastjsonschema.compile(JSON_SCHEMA) -fast_compiled = lambda value, _: fastjsonschema_validate(value) -fast_not_compiled = lambda value, json_schema: fastjsonschema.compile(json_schema)(value) -with open('temp/performance.py', 'w') as f: - f.write(fastjsonschema.compile_to_code(JSON_SCHEMA)) -from temp.performance import validate -fast_file = lambda value, _: validate(value) +def fast_compiled(value, _): + fastjsonschema_validate(value) + + +def fast_not_compiled(value, json_schema): + fastjsonschema.compile(json_schema)(value) + + +validator_class = jsonschema.validators.validator_for(JSON_SCHEMA) +validator = validator_class(JSON_SCHEMA) + + +def jsonschema_compiled(value, _): + validator.validate(value) + + +with tempfile.NamedTemporaryFile('w', suffix='.py') as tmp_file: + tmp_file.write(fastjsonschema.compile_to_code(JSON_SCHEMA)) + tmp_file.flush() + spec = importlib.util.spec_from_file_location("temp.performance", tmp_file.name) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + +def fast_file(value, _): + module.validate(value) + jsonspec = load(JSON_SCHEMA) @@ -97,6 +120,7 @@ def t(func, valid_values=True): fast_compiled, fast_file, fast_not_compiled, + jsonschema_compiled, ) """ @@ -115,7 +139,7 @@ def t(func, valid_values=True): """.format(func)) res = timeit.timeit(code, setup, number=NUMBER) - print('{:<20} {:<10} ==> {}'.format(module, 'valid' if valid_values else 'invalid', res)) + print('{:<20} {:<10} ==> {:10.7f}'.format(module, 'valid' if valid_values else 'invalid', res)) print('Number: {}'.format(NUMBER)) @@ -132,6 +156,9 @@ def t(func, valid_values=True): t('jsonschema.validate') t('jsonschema.validate', valid_values=False) +t('jsonschema_compiled') +t('jsonschema_compiled', valid_values=False) + t('jsonspec.validate') t('jsonspec.validate', valid_values=False) From 346ac60b67bd8803bdc66f492e441d0d4e299b60 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Wed, 26 Feb 2020 17:51:06 +0000 Subject: [PATCH 183/201] Initial base test --- tests/test_array.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_array.py b/tests/test_array.py index 33f6a06..b0626b4 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -134,3 +134,19 @@ def test_different_items_without_additional_items(asserter, value, expected): ], 'additionalItems': False, }, value, expected) + + +@pytest.mark.parametrize('value, expected', [ + ((), ()), + (('a',), ('a',)), + (('a', 'b'), ('a', 'b')) + +]) +def test_tuples_as_arrays(asserter, value, expected): + asserter({ + 'type': 'array', + 'items': [ + {'type': 'string'}, + ], + 'additionalItems': False, + }, value, expected) From be185f2281fb55e9920c0de9566e447a4365a27d Mon Sep 17 00:00:00 2001 From: epiphyte Date: Wed, 26 Feb 2020 19:28:29 +0000 Subject: [PATCH 184/201] WIP - ugly implementation --- fastjsonschema/draft04.py | 25 +++++++++++++++++-------- fastjsonschema/draft06.py | 11 ++++++++++- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 01cc07e..5906a12 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -6,13 +6,13 @@ JSON_TYPE_TO_PYTHON_TYPE = { - 'null': 'NoneType', - 'boolean': 'bool', - 'number': 'int, float', - 'integer': 'int', - 'string': 'str', - 'array': 'list', - 'object': 'dict', + 'null': ('NoneType',), + 'boolean': ('bool',), + 'number': ('int', 'float',), + 'integer': ('int',), + 'string': ('str',), + 'array': ('list', 'tuple'), + 'object': ('dict',), } DOLLAR_FINDER = re.compile(r"(? Date: Wed, 26 Feb 2020 22:37:59 +0000 Subject: [PATCH 185/201] Another potential spot.. --- fastjsonschema/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastjsonschema/generator.py b/fastjsonschema/generator.py index fc5b4b1..17cebd6 100644 --- a/fastjsonschema/generator.py +++ b/fastjsonschema/generator.py @@ -281,7 +281,7 @@ def create_variable_is_list(self): if variable_name in self._variables: return self._variables.add(variable_name) - self.l('{variable}_is_list = isinstance({variable}, list)') + self.l('{variable}_is_list = isinstance({variable}, (list, tuple))') def create_variable_is_dict(self): """ From dce8d7501ed1de1043851a702fa47ba89d1bc5d3 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Thu, 27 Feb 2020 04:16:32 +0000 Subject: [PATCH 186/201] Add mixed arrays test, update integration test --- tests/test_array.py | 15 +++++++++++++++ tests/test_integration.py | 4 ++++ 2 files changed, 19 insertions(+) diff --git a/tests/test_array.py b/tests/test_array.py index b0626b4..feb8508 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -150,3 +150,18 @@ def test_tuples_as_arrays(asserter, value, expected): ], 'additionalItems': False, }, value, expected) + + +@pytest.mark.parametrize('value, expected', [ + ({'a': [], 'b': ()}, {'a': [], 'b': ()}), + ({'a': (1, 2), 'b': (3, 4)}, {'a': (1, 2), 'b': (3, 4)}), +]) +def test_mixed_arrays(asserter, value, expected): + asserter({ + 'type': 'object', + 'properties': { + 'a': {'type': 'array'}, + 'b': {'type': 'array'}, + }, + }, value, expected) + diff --git a/tests/test_integration.py b/tests/test_integration.py index 5ea2c08..27135f9 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -52,6 +52,10 @@ [9, 'world', [1], {'a': 'a', 'b': 'b', 'd': 'd'}, 42, 3], [9, 'world', [1], {'a': 'a', 'b': 'b', 'c': 'abc', 'd': 'd'}, 42, 3], ), + ( + (9, 'world', (1,), {'a': 'a', 'b': 'b', 'd': 'd'}, 42, 3), + (9, 'world', (1,), {'a': 'a', 'b': 'b', 'c': 'abc', 'd': 'd'}, 42, 3), + ), ( [9, 'world', [1], {'a': 'a', 'b': 'b', 'c': 'xy'}, 42, 3], [9, 'world', [1], {'a': 'a', 'b': 'b', 'c': 'xy'}, 42, 3], From 5d02834f38d9323c99da8ba2f31db0c3f43f1bb2 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Thu, 27 Feb 2020 04:49:55 +0000 Subject: [PATCH 187/201] test cleanup --- tests/test_array.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_array.py b/tests/test_array.py index feb8508..7205b04 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -139,16 +139,17 @@ def test_different_items_without_additional_items(asserter, value, expected): @pytest.mark.parametrize('value, expected', [ ((), ()), (('a',), ('a',)), - (('a', 'b'), ('a', 'b')) - + (('a', 'b'), ('a', 'b')), + (('a', 'b', 3), JsonSchemaException('data[2] must be string', value=3, name='data[2]', + definition={'type': 'string'}, rule='type')) ]) def test_tuples_as_arrays(asserter, value, expected): asserter({ + '$schema': 'http://json-schema.org/draft-06/schema', 'type': 'array', - 'items': [ + 'items': {'type': 'string'}, - ], - 'additionalItems': False, + }, value, expected) From 262fb45d66cea4111c3c0ae2c6c21278a78bd6b3 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Thu, 27 Feb 2020 04:50:31 +0000 Subject: [PATCH 188/201] Cleanup tests --- tests/test_array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_array.py b/tests/test_array.py index 7205b04..3d77435 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -141,7 +141,7 @@ def test_different_items_without_additional_items(asserter, value, expected): (('a',), ('a',)), (('a', 'b'), ('a', 'b')), (('a', 'b', 3), JsonSchemaException('data[2] must be string', value=3, name='data[2]', - definition={'type': 'string'}, rule='type')) + definition={'type': 'string'}, rule='type')), ]) def test_tuples_as_arrays(asserter, value, expected): asserter({ From 40a9c0d006e1112c690ad787b5ab24aee76a7621 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Thu, 27 Feb 2020 04:53:00 +0000 Subject: [PATCH 189/201] incremental cleanup --- fastjsonschema/draft04.py | 5 +---- fastjsonschema/draft06.py | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 5906a12..036f5ee 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -86,10 +86,7 @@ def generate_type(self): for t in types: # print(t) vals = JSON_TYPE_TO_PYTHON_TYPE[t] - # print(vals) - for v in vals: - # print(v) - _ptypes.append(v) + _ptypes.extend(vals) python_types = ', '.join(_ptypes) except KeyError as exc: raise JsonSchemaDefinitionException('Unknown type: {}'.format(exc)) diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py index 5ff4f0a..8e81535 100644 --- a/fastjsonschema/draft06.py +++ b/fastjsonschema/draft06.py @@ -61,9 +61,7 @@ def generate_type(self): # print(t) vals = JSON_TYPE_TO_PYTHON_TYPE[t] # print(vals) - for v in vals: - # print(v) - _ptypes.append(v) + _ptypes.extend(vals) python_types = ', '.join(_ptypes) except KeyError as exc: raise JsonSchemaDefinitionException('Unknown type: {}'.format(exc)) From 05d108ea2c76231a8ca38d694e7780d07ecd30b2 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Thu, 27 Feb 2020 04:56:03 +0000 Subject: [PATCH 190/201] cleanup --- fastjsonschema/draft04.py | 6 +----- fastjsonschema/draft06.py | 7 +------ 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 036f5ee..b38fbc6 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -82,11 +82,7 @@ def generate_type(self): types = enforce_list(self._definition['type']) try: _ptypes = [] - # [_ptypes.extend(JSON_TYPE_TO_PYTHON_TYPE[t]) for t in types] - for t in types: - # print(t) - vals = JSON_TYPE_TO_PYTHON_TYPE[t] - _ptypes.extend(vals) + [_ptypes.extend(JSON_TYPE_TO_PYTHON_TYPE[t]) for t in types] python_types = ', '.join(_ptypes) except KeyError as exc: raise JsonSchemaDefinitionException('Unknown type: {}'.format(exc)) diff --git a/fastjsonschema/draft06.py b/fastjsonschema/draft06.py index 8e81535..f959d11 100644 --- a/fastjsonschema/draft06.py +++ b/fastjsonschema/draft06.py @@ -56,12 +56,7 @@ def generate_type(self): types = enforce_list(self._definition['type']) try: _ptypes = [] - # [_ptypes.extend(JSON_TYPE_TO_PYTHON_TYPE[t]) for t in types] - for t in types: - # print(t) - vals = JSON_TYPE_TO_PYTHON_TYPE[t] - # print(vals) - _ptypes.extend(vals) + [_ptypes.extend(JSON_TYPE_TO_PYTHON_TYPE[t]) for t in types] python_types = ', '.join(_ptypes) except KeyError as exc: raise JsonSchemaDefinitionException('Unknown type: {}'.format(exc)) From f9eeb60ff4ab081c41b491fd5be3d5b32e93f1a7 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Thu, 27 Feb 2020 05:01:18 +0000 Subject: [PATCH 191/201] Revert extra change --- fastjsonschema/draft04.py | 18 ++++++++---------- fastjsonschema/draft06.py | 4 +--- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index b38fbc6..6c50a21 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -6,13 +6,13 @@ JSON_TYPE_TO_PYTHON_TYPE = { - 'null': ('NoneType',), - 'boolean': ('bool',), - 'number': ('int', 'float',), - 'integer': ('int',), - 'string': ('str',), - 'array': ('list', 'tuple'), - 'object': ('dict',), + 'null': 'NoneType', + 'boolean': 'bool', + 'number': 'int, float', + 'integer': 'int', + 'string': 'str', + 'array': 'list, tuple', + 'object': 'dict', } DOLLAR_FINDER = re.compile(r"(? Date: Thu, 27 Feb 2020 09:11:03 +0100 Subject: [PATCH 192/201] Version 2.14.3 --- CHANGELOG.txt | 5 +++++ fastjsonschema/__init__.py | 26 ++++++++++++++------------ fastjsonschema/version.py | 2 +- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 4dfc74b..743d0fc 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +=== 2.14.3 (2020-02-27) + +* Tuple is also valid array + + === 2.14.2 (2019-12-11) * Fix of `additionalProperties=true` for JSON schema 4 diff --git a/fastjsonschema/__init__.py b/fastjsonschema/__init__.py index 0795a0a..f7715b8 100644 --- a/fastjsonschema/__init__.py +++ b/fastjsonschema/__init__.py @@ -41,18 +41,20 @@ .. code-block:: bash $ make performance - fast_compiled valid ==> 0.030474655970465392 - fast_compiled invalid ==> 0.0017561429995112121 - fast_file valid ==> 0.028758891974575818 - fast_file invalid ==> 0.0017655809642747045 - fast_not_compiled valid ==> 4.597834145999514 - fast_not_compiled invalid ==> 1.139162228035275 - jsonschema valid ==> 5.014410221017897 - jsonschema invalid ==> 1.1362981660058722 - jsonspec valid ==> 8.1144932230236 - jsonspec invalid ==> 2.0143173419637606 - validictory valid ==> 0.4084212710149586 - validictory invalid ==> 0.026061681972350925 + fast_compiled valid ==> 0.0464646 + fast_compiled invalid ==> 0.0030227 + fast_file valid ==> 0.0461219 + fast_file invalid ==> 0.0030608 + fast_not_compiled valid ==> 11.4627202 + fast_not_compiled invalid ==> 2.5726230 + jsonschema valid ==> 7.5844927 + jsonschema invalid ==> 1.9204665 + jsonschema_compiled valid ==> 0.6938364 + jsonschema_compiled invalid ==> 0.0359244 + jsonspec valid ==> 9.0715843 + jsonspec invalid ==> 2.1650488 + validictory valid ==> 0.4874793 + validictory invalid ==> 0.0232244 This library follows and implements `JSON schema draft-04, draft-06, and draft-07 `_. Sometimes it's not perfectly clear so I recommend also diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index 094e9da..95dd5a8 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.14.2' +VERSION = '2.14.3' From b93e0a3189235e6d005c5be5af265b38dfbd4861 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 19 Mar 2020 08:38:25 +0100 Subject: [PATCH 193/201] Fix property --- CHANGELOG.txt | 5 +++++ fastjsonschema/ref_resolver.py | 14 ++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 743d0fc..0340fce 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +=== 2.14.4 (unreleased) + +* Fix $id property + + === 2.14.3 (2020-02-27) * Tuple is also valid array diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index fe5979f..19b3a0f 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -13,10 +13,16 @@ from urllib.parse import unquote from urllib.request import urlopen - from .exceptions import JsonSchemaException +def get_id(schema): + """ + Originally ID was `id` and since v7 it's `$id`. + """ + return schema.get('$id', schema.get('id', '')) + + def resolve_path(schema, fragment): """ Return definition from path. @@ -83,7 +89,7 @@ def from_schema(cls, schema, handlers={}, **kwargs): Construct a resolver from a JSON schema object. """ return cls( - schema.get('$id', schema.get('id', '')) if isinstance(schema, dict) else '', + get_id(schema) if isinstance(schema, dict) else '', schema, handlers=handlers, **kwargs @@ -148,8 +154,8 @@ def walk(self, node: dict): elif '$ref' in node and isinstance(node['$ref'], str): ref = node['$ref'] node['$ref'] = urlparse.urljoin(self.resolution_scope, ref) - elif 'id' in node and isinstance(node['id'], str): - with self.in_scope(node['id']): + elif ('$id' in node or 'id' in node) and isinstance(get_id(node), str): + with self.in_scope(get_id(node)): self.store[normalize(self.resolution_scope)] = node for _, item in node.items(): if isinstance(item, dict): From 61c6997a8348b8df9b22e029ca2ba35ef441fbb8 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 19 Mar 2020 08:58:51 +0100 Subject: [PATCH 194/201] Add extra properties to error message when additionalProperties are set to False --- CHANGELOG.txt | 1 + fastjsonschema/draft04.py | 2 +- tests/test_object.py | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 0340fce..9b8c599 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,7 @@ === 2.14.4 (unreleased) * Fix $id property +* Add extra properties to error message when additionalProperties are set to False === 2.14.3 (2020-02-27) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 6c50a21..f81f8d2 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -537,7 +537,7 @@ def generate_additional_properties(self): ) else: with self.l('if {variable}_keys:'): - self.exc('{name} must contain only specified properties', rule='additionalProperties') + self.exc('{name} must not contain "+str({variable}_keys)+" properties', rule='additionalProperties') def generate_dependencies(self): """ diff --git a/tests/test_object.py b/tests/test_object.py index 26ea4e2..83fe0e1 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -106,8 +106,8 @@ def test_properties_with_additional_properties(asserter, value, expected): ({'a': 1}, {'a': 1}), ({'a': 1, 'b': ''}, {'a': 1, 'b': ''}), ({'a': 1, 'b': 2}, JsonSchemaException('data.b must be string', value=2, name='data.b', definition={'type': 'string'}, rule='type')), - ({'a': 1, 'b': '', 'any': True}, JsonSchemaException('data must contain only specified properties', value='{data}', name='data', definition='{definition}', rule='additionalProperties')), - ({'cd': True}, JsonSchemaException('data must contain only specified properties', value='{data}', name='data', definition='{definition}', rule='additionalProperties')), + ({'a': 1, 'b': '', 'any': True}, JsonSchemaException('data must not contain {\'any\'} properties', value='{data}', name='data', definition='{definition}', rule='additionalProperties')), + ({'cd': True}, JsonSchemaException('data must not contain {\'cd\'} properties', value='{data}', name='data', definition='{definition}', rule='additionalProperties')), ({'c_d': True}, {'c_d': True}), ]) def test_properties_without_additional_properties(asserter, value, expected): From 2c88f44c9a05cc86ce315c67d64419a20eaeb639 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 19 Mar 2020 09:05:59 +0100 Subject: [PATCH 195/201] Better exception message when referencing schema is not valid JSON --- CHANGELOG.txt | 1 + fastjsonschema/ref_resolver.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 9b8c599..2754308 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,6 +2,7 @@ * Fix $id property * Add extra properties to error message when additionalProperties are set to False +* Better exception message when referencing schema is not valid JSON === 2.14.3 (2020-02-27) diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index 19b3a0f..bb45597 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -13,7 +13,7 @@ from urllib.parse import unquote from urllib.request import urlopen -from .exceptions import JsonSchemaException +from .exceptions import JsonSchemaDefinitionException def get_id(schema): @@ -38,7 +38,7 @@ def resolve_path(schema, fragment): elif part in schema: schema = schema[part] else: - raise JsonSchemaException('Unresolvable ref: {}'.format(part)) + raise JsonSchemaDefinitionException('Unresolvable ref: {}'.format(part)) return schema @@ -61,7 +61,10 @@ def resolve_remote(uri, handlers): else: req = urlopen(uri) encoding = req.info().get_content_charset() or 'utf-8' - result = json.loads(req.read().decode(encoding),) + try: + result = json.loads(req.read().decode(encoding),) + except ValueError as exc: + raise JsonSchemaDefinitionException('{} failed to decode: {}'.format(uri, exc)) return result From 8c8a05c76bd6a8a346dcd9185d95baab110a04c5 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 19 Mar 2020 09:09:24 +0100 Subject: [PATCH 196/201] Version 2.14.4 --- CHANGELOG.txt | 2 +- Makefile | 2 +- fastjsonschema/version.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 2754308..e95a4a8 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,4 +1,4 @@ -=== 2.14.4 (unreleased) +=== 2.14.4 (2020-03-19) * Fix $id property * Add extra properties to error message when additionalProperties are set to False diff --git a/Makefile b/Makefile index e04874f..26b6d1f 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ doc: cd docs; make upload: venv - ${PYTHON} setup.py register sdist upload + ${PYTHON} setup.py register sdist bdist_wheel upload deb: venv ${PYTHON} setup.py --command-packages=stdeb.command bdist_deb diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index 95dd5a8..b3091ab 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.14.3' +VERSION = '2.14.4' From 400b11f8e9d41708a366c8af98c889cdc1392794 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 17 Aug 2020 18:55:25 +0200 Subject: [PATCH 197/201] Fix missing dependenices --- fastjsonschema/draft04.py | 12 ++++++++---- tests/test_object.py | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index f81f8d2..5c40de7 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -58,10 +58,11 @@ def __init__(self, definition, resolver=None, formats={}): ('minProperties', self.generate_min_properties), ('maxProperties', self.generate_max_properties), ('required', self.generate_required), + # Check dependencies before properties generates default values. + ('dependencies', self.generate_dependencies), ('properties', self.generate_properties), ('patternProperties', self.generate_pattern_properties), ('additionalProperties', self.generate_additional_properties), - ('dependencies', self.generate_dependencies), )) @property @@ -559,16 +560,19 @@ def generate_dependencies(self): """ self.create_variable_is_dict() with self.l('if {variable}_is_dict:'): - self.create_variable_keys() + isEmpty = True for key, values in self._definition["dependencies"].items(): if values == [] or values is True: continue - with self.l('if "{}" in {variable}_keys:', self.e(key)): + isEmpty = False + with self.l('if "{}" in {variable}:', self.e(key)): if values is False: self.exc('{} in {name} must not be there', key, rule='dependencies') elif isinstance(values, list): for value in values: - with self.l('if "{}" not in {variable}_keys:', self.e(value)): + with self.l('if "{}" not in {variable}:', self.e(value)): self.exc('{name} missing dependency {} for {}', self.e(value), self.e(key), rule='dependencies') else: self.generate_func_code_block(values, self._variable, self._variable_name, clear_variables=True) + if isEmpty: + self.l('pass') diff --git a/tests/test_object.py b/tests/test_object.py index 83fe0e1..181b453 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -182,3 +182,25 @@ def test_object_with_ref_property(asserter, value, expected): "$ref": {"type": "string"} } }, value, expected) + + +@pytest.mark.parametrize('value, expected', [ + ({}, {}), + ({'foo': 'foo'}, JsonSchemaException('data missing dependency bar for foo', value={'foo': 'foo'}, name='data', definition='{definition}', rule='dependencies')), + ({'foo': 'foo', 'bar': 'bar'}, {'foo': 'foo', 'bar': 'bar'}), +]) +def test_dependencies(asserter, value, expected): + asserter({ + 'type': 'object', + "properties": { + "foo": { + "type": "string" + }, + "bar": { + "type": "string" + } + }, + "dependencies": { + "foo": ["bar"], + }, + }, value, expected) \ No newline at end of file From f5b3f4da97bd74dee26bec584ade16f73ed6c64a Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 17 Aug 2020 19:19:07 +0200 Subject: [PATCH 198/201] Fix schema cache --- fastjsonschema/ref_resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastjsonschema/ref_resolver.py b/fastjsonschema/ref_resolver.py index bb45597..38cf7e0 100644 --- a/fastjsonschema/ref_resolver.py +++ b/fastjsonschema/ref_resolver.py @@ -119,7 +119,7 @@ def resolving(self, ref: str): new_uri = urlparse.urljoin(self.resolution_scope, ref) uri, fragment = urlparse.urldefrag(new_uri) - if normalize(uri) in self.store: + if uri and normalize(uri) in self.store: schema = self.store[normalize(uri)] elif not uri or uri == self.base_uri: schema = self.schema From 78ae1ae6b93bd1647bbed4e8fd8fb8cdd4252a01 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Mon, 17 Aug 2020 19:20:44 +0200 Subject: [PATCH 199/201] Version 2.14.5 --- CHANGELOG.txt | 6 ++++++ fastjsonschema/version.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index e95a4a8..b8e3038 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,9 @@ +=== 2.14.5 (2020-08-17) + +* Fix missing dependencies +* Fix schema cache + + === 2.14.4 (2020-03-19) * Fix $id property diff --git a/fastjsonschema/version.py b/fastjsonschema/version.py index b3091ab..32f7293 100644 --- a/fastjsonschema/version.py +++ b/fastjsonschema/version.py @@ -1 +1 @@ -VERSION = '2.14.4' +VERSION = '2.14.5' From 4b0d76cb07d78f047813c6bcda01879ee0a0216c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ho=C5=99ej=C5=A1ek?= Date: Thu, 1 Oct 2020 09:43:18 +0200 Subject: [PATCH 200/201] Create codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 71 +++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..2fe42c9 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,71 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +name: "CodeQL" + +on: + push: + branches: [master] + pull_request: + # The branches below must be a subset of the branches above + branches: [master] + schedule: + - cron: '0 1 * * 4' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] + language: ['python'] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From a0e1a5a6fcaf80cf1994d19838c54dcc5d17d8ee Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 6 Oct 2020 13:40:59 -0400 Subject: [PATCH 201/201] Add support to validate Decimals. --- fastjsonschema/draft04.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fastjsonschema/draft04.py b/fastjsonschema/draft04.py index 5c40de7..bfa8049 100644 --- a/fastjsonschema/draft04.py +++ b/fastjsonschema/draft04.py @@ -8,7 +8,7 @@ JSON_TYPE_TO_PYTHON_TYPE = { 'null': 'NoneType', 'boolean': 'bool', - 'number': 'int, float', + 'number': 'int, float, Decimal', 'integer': 'int', 'string': 'str', 'array': 'list, tuple', @@ -69,6 +69,7 @@ def __init__(self, definition, resolver=None, formats={}): def global_state(self): res = super().global_state res['custom_formats'] = self._custom_formats + res['Decimal'] = decimal.Decimal return res def generate_type(self):