From a7a4723e1173b0c1b6cbec846399040e6f5d9388 Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Fri, 5 Jun 2020 07:41:01 +0200 Subject: [PATCH] Changes for type annotations --- .travis.yml | 7 ---- .../travis/{run_pylint.sh => run_checks.sh} | 0 config/travis/runtests.sh | 4 +- ...{jobs_pylint3 => jobs_lint_and_type_check} | 2 +- data/templates/.travis.yml/jobs_linux | 7 ---- data/templates/setup.py/import_module | 4 +- data/templates/{ => travis}/install.sh | 8 ++-- .../run_checks-pylint.sh} | 15 +++++-- .../travis/run_checks-pylint_and_mypy.sh | 40 +++++++++++++++++++ data/templates/{ => travis}/run_coverage.sh | 0 data/templates/{ => travis}/run_python3.sh | 0 .../{ => travis}/run_with_timeout.sh | 0 data/templates/{ => travis}/runtests.sh | 8 ++-- l2tdevtools/dependency_writers/dpkg.py | 4 +- l2tdevtools/dependency_writers/travis.py | 31 +++++++++----- l2tdevtools/dependency_writers/travis_yml.py | 2 +- l2tdevtools/dpkg_files.py | 6 +-- setup.py | 4 +- tools/update-dependencies.py | 10 ++++- 19 files changed, 101 insertions(+), 51 deletions(-) rename config/travis/{run_pylint.sh => run_checks.sh} (100%) rename data/templates/.travis.yml/{jobs_pylint3 => jobs_lint_and_type_check} (69%) rename data/templates/{ => travis}/install.sh (93%) rename data/templates/{run_pylint.sh => travis/run_checks-pylint.sh} (71%) create mode 100755 data/templates/travis/run_checks-pylint_and_mypy.sh rename data/templates/{ => travis}/run_coverage.sh (100%) rename data/templates/{ => travis}/run_python3.sh (100%) rename data/templates/{ => travis}/run_with_timeout.sh (100%) rename data/templates/{ => travis}/runtests.sh (91%) diff --git a/.travis.yml b/.travis.yml index c6fd335a..b93bf538 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,13 +39,6 @@ jobs: python: 3.8 services: - docker - - name: "Ubuntu Focal (20.04) (Docker) with Python 3.5 (tox)" - env: [TOXENV="py35", UBUNTU_VERSION="20.04"] - group: edge - language: python - python: 3.5 - services: - - docker - name: "Ubuntu Focal (20.04) (Docker) with Python 3.6 (tox)" env: [TOXENV="py36", UBUNTU_VERSION="20.04"] group: edge diff --git a/config/travis/run_pylint.sh b/config/travis/run_checks.sh similarity index 100% rename from config/travis/run_pylint.sh rename to config/travis/run_checks.sh diff --git a/config/travis/runtests.sh b/config/travis/runtests.sh index 64363ec0..4b01ded1 100755 --- a/config/travis/runtests.sh +++ b/config/travis/runtests.sh @@ -19,7 +19,7 @@ then elif test "${TARGET}" = "pylint"; then - TEST_COMMAND="./config/travis/run_pylint.sh"; + TEST_COMMAND="./config/travis/run_checks.sh"; else TEST_COMMAND="./config/travis/run_python3.sh"; fi @@ -53,7 +53,7 @@ then elif test "${TARGET}" = "pylint"; then - TEST_COMMAND="./config/travis/run_pylint.sh"; + TEST_COMMAND="./config/travis/run_checks.sh"; else TEST_COMMAND="./config/travis/run_python3.sh"; fi diff --git a/data/templates/.travis.yml/jobs_pylint3 b/data/templates/.travis.yml/jobs_lint_and_type_check similarity index 69% rename from data/templates/.travis.yml/jobs_pylint3 rename to data/templates/.travis.yml/jobs_lint_and_type_check index e4f2d344..4decbf00 100644 --- a/data/templates/.travis.yml/jobs_pylint3 +++ b/data/templates/.travis.yml/jobs_lint_and_type_check @@ -1,5 +1,5 @@ - name: "Pylint on Ubuntu Bionic (20.04) (Docker) with Python 3.8" - env: [TARGET="pylint", UBUNTU_VERSION="20.04"] + env: [TARGET="lint_and_type_check", UBUNTU_VERSION="20.04"] group: edge language: python python: 3.8 diff --git a/data/templates/.travis.yml/jobs_linux b/data/templates/.travis.yml/jobs_linux index 02852863..b504bfe2 100644 --- a/data/templates/.travis.yml/jobs_linux +++ b/data/templates/.travis.yml/jobs_linux @@ -26,13 +26,6 @@ python: 3.8 services: - docker - - name: "Ubuntu Focal (20.04) (Docker) with Python 3.5 (tox)" - env: [TOXENV="py35", UBUNTU_VERSION="20.04"] - group: edge - language: python - python: 3.5 - services: - - docker - name: "Ubuntu Focal (20.04) (Docker) with Python 3.6 (tox)" env: [TOXENV="py36", UBUNTU_VERSION="20.04"] group: edge diff --git a/data/templates/setup.py/import_module b/data/templates/setup.py/import_module index c653fc90..862f9446 100644 --- a/data/templates/setup.py/import_module +++ b/data/templates/setup.py/import_module @@ -1,7 +1,7 @@ version_tuple = (sys.version_info[0], sys.version_info[1]) -if version_tuple < (3, 5): +if version_tuple < (3, 6): print(( - 'Unsupported Python version: {0:s}, version 3.5 or higher ' + 'Unsupported Python version: {0:s}, version 3.6 or higher ' 'required.').format(sys.version)) sys.exit(1) diff --git a/data/templates/install.sh b/data/templates/travis/install.sh similarity index 93% rename from data/templates/install.sh rename to data/templates/travis/install.sh index 9a596057..54ae765e 100755 --- a/data/templates/install.sh +++ b/data/templates/travis/install.sh @@ -37,9 +37,9 @@ then else RPM_PACKAGES=""; - if test $${TARGET} = "pylint"; + if test $${TARGET} = "lint_and_type_check"; then - RPM_PACKAGES="$${RPM_PACKAGES} findutils pylint"; + RPM_PACKAGES="$${RPM_PACKAGES} findutils pylint python3-mypy"; fi RPM_PACKAGES="$${RPM_PACKAGES} python3 $${RPM_PYTHON3_DEPENDENCIES} $${RPM_PYTHON3_TEST_DEPENDENCIES}"; fi @@ -87,9 +87,9 @@ then then DPKG_PACKAGES="$${DPKG_PACKAGES} sudo"; - elif test $${TARGET} = "pylint"; + elif test $${TARGET} = "lint_and_type_check"; then - DPKG_PACKAGES="$${DPKG_PACKAGES} python3-distutils pylint"; + DPKG_PACKAGES="$${DPKG_PACKAGES} mypy pylint python3-distutils"; fi if test "$${TARGET}" != "jenkins3"; then diff --git a/data/templates/run_pylint.sh b/data/templates/travis/run_checks-pylint.sh similarity index 71% rename from data/templates/run_pylint.sh rename to data/templates/travis/run_checks-pylint.sh index a4a59f45..99ed9cca 100755 --- a/data/templates/run_pylint.sh +++ b/data/templates/travis/run_checks-pylint.sh @@ -1,12 +1,14 @@ #!/bin/bash # -# Script to run pylint on Travis-CI. +# Script to run lint checks on Travis-CI. # # This file is generated by l2tdevtools update-dependencies.py, any dependency # related changes should be made in dependencies.ini. -# Exit on error. -set -e; +EXIT_SUCCESS=0; +EXIT_FAILURE=1; + +RESULT=$${EXIT_SUCCESS}; pylint --version @@ -19,4 +21,11 @@ do echo "Checking: $${FILE}"; pylint --rcfile=.pylintrc $${FILE}; + + if test $$? -ne $${EXIT_SUCCESS}; + then + RESULT=$${EXIT_FAILURE}; + fi done + +exit $${RESULT}; diff --git a/data/templates/travis/run_checks-pylint_and_mypy.sh b/data/templates/travis/run_checks-pylint_and_mypy.sh new file mode 100755 index 00000000..e46943cd --- /dev/null +++ b/data/templates/travis/run_checks-pylint_and_mypy.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# +# Script to run lint and type checks on Travis-CI. +# +# This file is generated by l2tdevtools update-dependencies.py, any dependency +# related changes should be made in dependencies.ini. + +EXIT_SUCCESS=0; +EXIT_FAILURE=1; + +RESULT=$${EXIT_SUCCESS}; + +pylint --version + +# Ignore setup.py for now due to: +# setup.py:15:0: E0001: Cannot import 'distutils.command.bdist_msi' due to +# syntax error 'expected an indented block (, line 347)' (syntax-error) + +for FILE in $$(find ${paths_to_lint} -name \*.py); +do + echo "Checking: $${FILE}"; + + pylint --rcfile=.pylintrc $${FILE}; + + if test $$? -ne $${EXIT_SUCCESS}; + then + RESULT=$${EXIT_FAILURE}; + fi +done + +mypy --version + +mypy --strict ${project_name}; + +if test $$? -ne $${EXIT_SUCCESS}; +then + RESULT=$${EXIT_FAILURE}; +fi + +exit $${RESULT}; diff --git a/data/templates/run_coverage.sh b/data/templates/travis/run_coverage.sh similarity index 100% rename from data/templates/run_coverage.sh rename to data/templates/travis/run_coverage.sh diff --git a/data/templates/run_python3.sh b/data/templates/travis/run_python3.sh similarity index 100% rename from data/templates/run_python3.sh rename to data/templates/travis/run_python3.sh diff --git a/data/templates/run_with_timeout.sh b/data/templates/travis/run_with_timeout.sh similarity index 100% rename from data/templates/run_with_timeout.sh rename to data/templates/travis/run_with_timeout.sh diff --git a/data/templates/runtests.sh b/data/templates/travis/runtests.sh similarity index 91% rename from data/templates/runtests.sh rename to data/templates/travis/runtests.sh index bbcb9778..ad427894 100755 --- a/data/templates/runtests.sh +++ b/data/templates/travis/runtests.sh @@ -17,9 +17,9 @@ then then TEST_COMMAND="tox -e $${TOXENV}"; - elif test "$${TARGET}" = "pylint"; + elif test "$${TARGET}" = "lint_and_type_check"; then - TEST_COMMAND="./config/travis/run_pylint.sh"; + TEST_COMMAND="./config/travis/run_checks.sh"; else TEST_COMMAND="./config/travis/run_python3.sh"; fi @@ -51,9 +51,9 @@ then then TEST_COMMAND="./config/jenkins/linux/run_end_to_end_tests_py3.sh travis"; - elif test "$${TARGET}" = "pylint"; + elif test "$${TARGET}" = "lint_and_type_check"; then - TEST_COMMAND="./config/travis/run_pylint.sh"; + TEST_COMMAND="./config/travis/run_checks.sh"; else TEST_COMMAND="./config/travis/run_python3.sh"; fi diff --git a/l2tdevtools/dependency_writers/dpkg.py b/l2tdevtools/dependency_writers/dpkg.py index 0f6628d1..9b2fc775 100644 --- a/l2tdevtools/dependency_writers/dpkg.py +++ b/l2tdevtools/dependency_writers/dpkg.py @@ -34,7 +34,7 @@ class DPKGControlWriter(interface.DependencyFileWriter): 'Maintainer: {maintainer:s}', 'Build-Depends: debhelper (>= 9), dh-python, {build_dependencies:s}', 'Standards-Version: 4.1.4', - 'X-Python3-Version: >= 3.5', + 'X-Python3-Version: >= 3.6', 'Homepage: {homepage_url:s}', ''] # yapf: disable @@ -97,7 +97,7 @@ def Write(self): if python3_dependencies: python3_dependencies = '{0:s}, '.format(python3_dependencies) - build_dependencies = ['python3-all (>= 3.5~)', 'python3-setuptools'] + build_dependencies = ['python3-all (>= 3.6~)', 'python3-setuptools'] if self._project_definition.name == 'timesketch': build_dependencies.insert(0, 'dh-systemd (>= 1.5)') build_dependencies.append('python3-pip') diff --git a/l2tdevtools/dependency_writers/travis.py b/l2tdevtools/dependency_writers/travis.py index d6616c2d..de8d2d0c 100644 --- a/l2tdevtools/dependency_writers/travis.py +++ b/l2tdevtools/dependency_writers/travis.py @@ -12,7 +12,7 @@ class TravisInstallScriptWriter(interface.DependencyFileWriter): """Travis-CI install.sh file writer.""" - _TEMPLATE_FILE = os.path.join('data', 'templates', 'install.sh') + _TEMPLATE_FILE = os.path.join('data', 'templates', 'travis', 'install.sh') PATH = os.path.join('config', 'travis', 'install.sh') @@ -53,7 +53,8 @@ def Write(self): class TravisRunCoverageScriptWriter(interface.DependencyFileWriter): """Travis-CI run_coverage.sh file writer.""" - _TEMPLATE_FILE = os.path.join('data', 'templates', 'run_coverage.sh') + _TEMPLATE_FILE = os.path.join( + 'data', 'templates', 'travis', 'run_coverage.sh') PATH = os.path.join('config', 'travis', 'run_coverage.sh') @@ -70,12 +71,12 @@ def Write(self): file_object.write(file_content) -class TravisRunPylintScriptWriter(interface.DependencyFileWriter): - """Travis-CI run_pylint.sh file writer.""" +class TravisRunChecksScriptWriter(interface.DependencyFileWriter): + """Travis-CI run_checks.sh file writer.""" - _TEMPLATE_FILE = os.path.join('data', 'templates', 'run_pylint.sh') + PATH = os.path.join('config', 'travis', 'run_checks.sh') - PATH = os.path.join('config', 'travis', 'run_pylint.sh') + _PROJECTS_WITH_TYPING = frozenset(['dfdatetime', 'dfwinreg']) def Write(self): """Writes a runtests.sh file.""" @@ -91,10 +92,17 @@ def Write(self): # paths_to_lint.insert(0, 'setup.py') template_mappings = { - 'paths_to_lint': ' '.join(paths_to_lint) + 'paths_to_lint': ' '.join(paths_to_lint), + 'project_name': self._project_definition.name, } - template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) + if self._project_definition.name in self._PROJECTS_WITH_TYPING: + template_file = 'run_checks-pylint_and_mypy.sh' + else: + template_file = 'run_checks-pylint.sh' + + template_file = os.path.join( + self._l2tdevtools_path, 'data', 'templates', 'travis', template_file) file_content = self._GenerateFromTemplate(template_file, template_mappings) with io.open(self.PATH, 'w', encoding='utf-8') as file_object: @@ -104,7 +112,7 @@ def Write(self): class TravisRunPython3ScriptWriter(interface.DependencyFileWriter): """Travis-CI run_python3.sh file writer.""" - _TEMPLATE_FILE = os.path.join('data', 'templates', 'run_python3.sh') + _TEMPLATE_FILE = os.path.join('data', 'templates', 'travis', 'run_python3.sh') PATH = os.path.join('config', 'travis', 'run_python3.sh') @@ -122,7 +130,7 @@ def Write(self): class TravisRunTestsScriptWriter(interface.DependencyFileWriter): """Travis-CI runtests.sh file writer.""" - _TEMPLATE_FILE = os.path.join('data', 'templates', 'runtests.sh') + _TEMPLATE_FILE = os.path.join('data', 'templates', 'travis', 'runtests.sh') PATH = os.path.join('config', 'travis', 'runtests.sh') @@ -142,7 +150,8 @@ def Write(self): class TravisRunWithTimeoutScriptWriter(interface.DependencyFileWriter): """Travis-CI run_with_timeout.sh file writer.""" - _TEMPLATE_FILE = os.path.join('data', 'templates', 'run_with_timeout.sh') + _TEMPLATE_FILE = os.path.join( + 'data', 'templates', 'travis', 'run_with_timeout.sh') PATH = os.path.join('config', 'travis', 'run_with_timeout.sh') diff --git a/l2tdevtools/dependency_writers/travis_yml.py b/l2tdevtools/dependency_writers/travis_yml.py index 975bf4e6..cdb36407 100644 --- a/l2tdevtools/dependency_writers/travis_yml.py +++ b/l2tdevtools/dependency_writers/travis_yml.py @@ -51,7 +51,7 @@ def Write(self): file_content.append(template_data) template_data = self._GenerateFromTemplate( - 'jobs_pylint3', template_mappings) + 'jobs_lint_and_type_check', template_mappings) file_content.append(template_data) template_data = self._GenerateFromTemplate( diff --git a/l2tdevtools/dpkg_files.py b/l2tdevtools/dpkg_files.py index d51d265f..e99707fd 100644 --- a/l2tdevtools/dpkg_files.py +++ b/l2tdevtools/dpkg_files.py @@ -85,7 +85,7 @@ class DPKGBuildFilesGenerator(object): 'Maintainer: {upstream_maintainer:s}', 'Build-Depends: debhelper (>= 9){build_depends:s}', 'Standards-Version: 4.1.4', - 'X-Python3-Version: >= 3.5', + 'X-Python3-Version: >= 3.6', 'Homepage: {upstream_homepage:s}', '', 'Package: {python3_package_name:s}', @@ -99,7 +99,7 @@ class DPKGBuildFilesGenerator(object): 'Package: {source_package_name:s}-tools', 'Architecture: all', ('Depends: {python3_package_name:s} (>= ${{binary:Version}}), ' - 'python3 (>= 3.5~), ${{python3:Depends}}, ${{misc:Depends}}'), + 'python3 (>= 3.6~), ${{python3:Depends}}, ${{misc:Depends}}'), 'Description: Tools of {description_name:s}', ' {description_long:s}', ''] @@ -311,7 +311,7 @@ def _GenerateControlFile(self, dpkg_path): elif self._project_definition.build_system == 'setup_py': build_depends.append('dh-python') - python3_build_depends.append('python3-all (>= 3.5~)') + python3_build_depends.append('python3-all (>= 3.6~)') python3_build_depends.append('python3-setuptools') if self._project_definition.architecture_dependent: diff --git a/setup.py b/setup.py index fbe34f53..bf876e1e 100755 --- a/setup.py +++ b/setup.py @@ -24,9 +24,9 @@ bdist_rpm = None version_tuple = (sys.version_info[0], sys.version_info[1]) -if version_tuple < (3, 5): +if version_tuple < (3, 6): print(( - 'Unsupported Python version: {0:s}, version 3.5 or higher ' + 'Unsupported Python version: {0:s}, version 3.6 or higher ' 'required.').format(sys.version)) sys.exit(1) diff --git a/tools/update-dependencies.py b/tools/update-dependencies.py index d29267a3..cd235434 100755 --- a/tools/update-dependencies.py +++ b/tools/update-dependencies.py @@ -54,8 +54,8 @@ def Main(): pylint_rc.PylintRcWriter, travis.TravisRunWithTimeoutScriptWriter, requirements.RequirementsWriter, requirements.TestRequirementsWriter, setup.SetupCfgWriter, setup.SetupPyWriter, - travis.TravisInstallScriptWriter, travis.TravisRunCoverageScriptWriter, - travis.TravisRunPylintScriptWriter, travis.TravisRunPython3ScriptWriter, + travis.TravisInstallScriptWriter, travis.TravisRunChecksScriptWriter, + travis.TravisRunCoverageScriptWriter, travis.TravisRunPython3ScriptWriter, travis.TravisRunTestsScriptWriter, travis.TravisRunWithTimeoutScriptWriter, travis_yml.TravisYMLWriter): writer = writer_class( @@ -106,6 +106,12 @@ def Main(): if os.path.isfile(script_path): os.remove(script_path) + script_path = os.path.join('config', 'travis', 'run_checks.sh') + if os.path.isfile(script_path): + script_path = os.path.join('config', 'travis', 'run_pylint.sh') + if os.path.isfile(script_path): + os.remove(script_path) + return True