diff --git a/docs/source/conf.py b/docs/source/conf.py index b204203..4002d4f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -39,6 +39,8 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', ] diff --git a/ets/__init__.py b/ets/__init__.py new file mode 100644 index 0000000..4c04bd0 --- /dev/null +++ b/ets/__init__.py @@ -0,0 +1,29 @@ +# (C) Copyright 2005-2020 Enthought, Inc., Austin, TX +# All rights reserved. +# +# This software is provided without warranty under the terms of the BSD +# license included in LICENSE.txt and may be redistributed only under +# the conditions described in the aforementioned license. The license +# is also available online at http://www.enthought.com/licenses/BSD.txt +# +# Thanks for using Enthought open source! + +""" Toolkit for environment management and continuous integration. + +The :mod:`ets` package provides utilities for environment management +and building continuous integration infrastructure that is used by the +various Enthought Tool Suite projects. + +It is designed to be installed in a "bootstrap" environment, such as a +developers' everyday Python environment. Once installed it is designed +provide a toolkit for writing continuous integration and deployment +scripts, and provides a toolkit for doing common operations, such as +creating venv or EDM environments, installing modules, running tests, +linting, and generating documentation. + +It could potentially also be used to generate demo environments and run +demo scripts to highlight the capabilities of various libraries. + +For an example of usage, see the ``etstool.py`` script in the main ets +directory. +""" diff --git a/ets/click_helpers.py b/ets/click_helpers.py new file mode 100644 index 0000000..49d5987 --- /dev/null +++ b/ets/click_helpers.py @@ -0,0 +1,89 @@ +# (C) Copyright 2005-2020 Enthought, Inc., Austin, TX +# All rights reserved. +# +# This software is provided without warranty under the terms of the BSD +# license included in LICENSE.txt and may be redistributed only under +# the conditions described in the aforementioned license. The license +# is also available online at http://www.enthought.com/licenses/BSD.txt +# +# Thanks for using Enthought open source! + +""" +Helpers for creating click-based scripts. +""" + +import sys + +import click + +from .edm import EDM +from .venv import Venv + + +#: The runtimes that we generally support. Modify this in-place to +#: change what might be supported for a particular script. +supported_runtimes = ["3.5", "3.6", "3.7", "3.8", "3.9"] + +#: The default runtime version is the current Python version. +default_runtime = '{}.{}'.format(*sys.version_info[:2]) + + +environment_manager = { + 'venv': Venv, + 'edm': EDM, +} + + +def create_environment_manager(ctx, param, value): + """ Create an environment object. """ + environment = ctx.params.get('environment') + runtime = ctx.params.get('runtime') + if environment is None: + environment = "ets-test-{runtime}".format(runtime=runtime) + ctx.params['environment'] = environment + + cls = environment_manager[value] + return cls(environment, runtime) + + +runtime_option = click.option( + "--runtime", + default=default_runtime, + type=click.Choice(supported_runtimes), + show_default=True, + help="Python runtime version for the development environment", +) +environment_option = click.option( + "--environment", + default=None, + help="Name of the environment to install", +) +environment_manager_option = click.option( + "--environment-manager", + default='venv', + help="Environment management tool to use", + callback=create_environment_manager +) + +editable_option = click.option( + "--editable/--not-editable", + default=False, + help="Install main package in 'editable' mode? [default: --not-editable]", +) +verbose_option = click.option( + "--verbose/--quiet", + default=True, + help="Run tests in verbose mode? [default: --verbose]", +) + +docs_option = click.option( + "--docs/--no-docs", + default=True, + help="Install documentation dependencies.", +) + +tests_option = click.option( + "--tests/--no-tests", + default=True, + help="Install test dependencies.", +) diff --git a/ets/copyright_header.py b/ets/copyright_header.py new file mode 100644 index 0000000..25d6331 --- /dev/null +++ b/ets/copyright_header.py @@ -0,0 +1,224 @@ +# (C) Copyright 2005-2020 Enthought, Inc., Austin, TX +# All rights reserved. +# +# This software is provided without warranty under the terms of the BSD +# license included in LICENSE.txt and may be redistributed only under +# the conditions described in the aforementioned license. The license +# is also available online at http://www.enthought.com/licenses/BSD.txt +# +# Thanks for using Enthought open source! + +""" Flake8 plugin to check copyright headers. """ + +import re + +# Minimum end year for the copyright statement. +MINIMUM_END_YEAR = 2020 + +# Regular expression to match things of the form "2019" or of the form +# "2010-2019". +YEAR_RANGE = r"(?P\d{4})(?:\-(?P\d{4}))?" + +# Generic copyright, used for searching for multiple copyright headers. +GENERIC_COPYRIGHT = re.compile("# .*Copyright .*Enthought", re.IGNORECASE) + +# Template for a regular expression for the copyright header. +PRODUCT_CODE_HEADER_TEMPLATE = r""" +# \(C\) Copyright {year_range} {company_name} +# All rights reserved\. +# +# This software is provided without warranty under the terms of the BSD +# license included in LICENSE\.txt and may be redistributed only under +# the conditions described in the aforementioned license\. The license +# is also available online at http://www\.enthought\.com/licenses/BSD\.txt +# +# Thanks for using Enthought open source! +""".lstrip() + +ENTHOUGHT_PRODUCT_CODE_HEADER = re.compile( + PRODUCT_CODE_HEADER_TEMPLATE.format( + company_name=r"Enthought, Inc\., Austin, TX", year_range=YEAR_RANGE, + ) +) + + +def parse_years(header_text): + """ + Parse a copyright year range from a header string. + + Looks for a year range of the form "2019" or "2010-2019", and + returns the start and end year. + + If there are multiple year ranges, parses only the first. + + Parameters + ---------- + header_text : str + The text to be parsed. Could be the entire copyright header, + or a single line from the copyright header. + + Returns + ------- + start_year, end_year : int + Start year and end year described by the range. + match_pos : int + Position within the string at which the match occurred. + + Raises + ------ + ValueError + If no year range is recognised from the given string. + """ + years_match = re.search(YEAR_RANGE, header_text) + if not years_match: + raise ValueError("No year range found in the given string.") + + start_year = int(years_match.group("start_year")) + end_year_str = years_match.group("end_year") + end_year = int(end_year_str) if end_year_str is not None else start_year + return start_year, end_year, years_match.start() + + +class HeaderError: + """ + Base class for the copyright header errors. + """ + + def __init__(self, lineno, col_offset): + self.lineno = lineno + self.col_offset = col_offset + + @property + def full_message(self): + """ + Full message in the form expected by flake8 (including the error code). + """ + return "{} {}".format(self.code, self.message) + + +class MissingCopyrightHeaderError(HeaderError): + """ + Error reported when no copyright header can be identified. + """ + + code = "H101" + message = "Missing copyright header" + + +class DuplicateCopyrightHeaderError(HeaderError): + """ + Error reported if multiple copyright headers found. + """ + + code = "H102" + message = "Multiple copyright headers found" + + +class IncorrectCopyrightHeaderError(HeaderError): + """ Error reported for incorrect copyright header. + + Ths error is reported if a copyright header is found, but its wording + doesn't match the officially approved wording. + """ + + code = "H103" + message = "Wrong copyright header found" + + +class OutdatedCopyrightYearError(HeaderError): + """ Error reported for incorrect copyright year information. + + This error is reported if the copyright header doesn't have the correct + year information in it. + """ + + code = "H104" + + def __init__(self, lineno, col_offset, end_year): + super().__init__(lineno=lineno, col_offset=col_offset) + self.end_year = end_year + + @property + def message(self): + """ Property returning the error message. """ + return ( + "Copyright end year ({}) out of date. The year should be at " + "least {}.".format(self.end_year, MINIMUM_END_YEAR) + ) + + +def copyright_header(lines): + """ + Check copyright header presence and accuracy in a Python file. + """ + file_contents = "".join(lines) + + # Empty files don't need a copyright header. + if not file_contents: + return + + # Not an empty file. See if we have a copyright header at all. + copyrights_found = [] + for lineno, line in enumerate(lines, start=1): + if re.match(GENERIC_COPYRIGHT, line): + copyrights_found.append(lineno) + + if not copyrights_found: + yield MissingCopyrightHeaderError( + lineno=1, col_offset=0, + ) + return + + if len(copyrights_found) > 1: + # Multiple possible copyright statements; report each one + # beyond the first, but we still check the first for + # correctness below. + for lineno in copyrights_found[1:]: + yield DuplicateCopyrightHeaderError( + lineno=lineno, col_offset=0, + ) + + # Check that the first copyright statement is the right one. + remaining_file_content = "".join(lines[copyrights_found[0]-1:]) + header_match = ENTHOUGHT_PRODUCT_CODE_HEADER.match(remaining_file_content) + if header_match is None: + yield IncorrectCopyrightHeaderError( + lineno=1, col_offset=0, + ) + return + + # Check the year range in the header. + for lineno, line in enumerate(lines, start=1): + try: + start_year, end_year, match_pos = parse_years(line) + except ValueError: + pass + else: + break + + if end_year < MINIMUM_END_YEAR: + yield OutdatedCopyrightYearError( + lineno=lineno, col_offset=match_pos, end_year=end_year, + ) + + +class CopyrightHeaderExtension(object): + """ + Flake8 extension for checking ETS copyright headers. + """ + + name = "headers" + version = "1.1.0" + + def __init__(self, tree, lines): + self.lines = lines + + def run(self): + """ Run the copyright header plugin. """ + for error in copyright_header(self.lines): + yield ( + error.lineno, + error.col_offset, + error.full_message, + type(self), + ) diff --git a/ets/edm.py b/ets/edm.py new file mode 100644 index 0000000..e7f2611 --- /dev/null +++ b/ets/edm.py @@ -0,0 +1,195 @@ +# (C) Copyright 2005-2020 Enthought, Inc., Austin, TX +# All rights reserved. +# +# This software is provided without warranty under the terms of the BSD +# license included in LICENSE.txt and may be redistributed only under +# the conditions described in the aforementioned license. The license +# is also available online at http://www.enthought.com/licenses/BSD.txt +# +# Thanks for using Enthought open source! + +""" Code for working with EDM-based Python environments. """ + +import os +import shutil +import sys + +import click + +from .environment import Environment + +#: Command templates for operations. +CREATE_COMMAND = "{edm} environments create {environment} --version={runtime}" +CLEAN_COMMAND = "{edm} environments remove {environment} --purge -y" +INSTALL_COMMAND = "{edm} install -e {environment} -y" +REMOVE_COMMAND = "{edm} plumbing remove-package --environment {environment} --force" # noqa: E501 +INVOKE_MODULE = "{edm} run -- python -m" +INVOKE_SCRIPT = "{edm} run --" + + +class EDM(Environment): + """ An EDM-based Python environment. + + Attributes + ---------- + manager : str + A human-readable name of the environment manager. + environment : str + The name of the environment. + runtime : str + The Python runtime version for the environment. + edm : str + The location of the EDM executable. + """ + + #: The user-facing name of the environment manager. + manager = "edm" + + def __init__(self, environment, runtime): + super().__init__(environment, runtime) + self.edm = locate_edm() + + def create(self, force=True): + """ Create the environment. + + Parameters + ---------- + force : bool + If force is True, any existing environment with the same name + will be removed. + """ + create_command = CREATE_COMMAND.split() + if force: + create_command.append("--force") + self.execute([create_command]) + + def clean(self): + """ Clean-up the environment. + + This attempts to destroy the environment, if possible. + """ + remove_command = CLEAN_COMMAND.split() + self.execute([remove_command]) + + def install(self, packages): + """ Install packages. + + This uses edm where the package is available for edm, otherwise + trying pip. + + Parameters + ---------- + packages : list of package dicts + List of package specifications to install. A package + specification is a dictionary with items of the form:: + + manager: specification + + where manager is the name of the manager and specification is + a string which specifies a package in a way that the manager + understands. + """ + edm_packages = [ + package['edm'] for package in packages + if 'edm' in package + ] + if edm_packages: + install_command = INSTALL_COMMAND.split() + edm_packages + self.execute([install_command]) + + other_packages = [ + package['pip'] for package in packages + if 'edm' not in package + ] + if other_packages: + self.pip_install(other_packages) + + def uninstall(self, packages): + """ Uninstall packages. + + If the package is known to EDM, this attempts to uninstall it, + otherwise it tries using pip. + + Parameters + ---------- + packages : list of package dicts + List of package specifications to uninstall. A package + specification is a dictionary with items of the form:: + + manager: specification + + where manager is the name of the manager and specification is + a string which specifies a package in a way that the manager + understands. + """ + edm_packages = [ + package['edm'] for package in packages + if 'edm' in package + ] + if edm_packages: + remove_command = REMOVE_COMMAND + edm_packages + self.execute([remove_command]) + + other_packages = [ + package['pip'] for package in packages + if 'edm' not in package + ] + if other_packages: + self.pip_uninstall(other_packages) + + def invoke_module(self, module, *args): + """ Run a module using ``python -m`` within the environment. + + Parameters + ---------- + module : str + The name of the module to run. + *args + Additional arguments for the module. + """ + command = INVOKE_MODULE.split() + [module] + list(args) + self.execute([command]) + + def invoke_script(self, script, *args): + """ Run a script within the environment. + + Parameters + ---------- + script : str + The name of the script to run. + *args + Additional arguments for the script. + """ + command = INVOKE_SCRIPT.split() + [script] + list(args) + self.execute([command]) + + +def locate_edm(): + """ Locate an EDM executable if it exists, else raise an exception. + + Returns the first EDM executable found on the path. On Windows, if that + executable turns out to be the "edm.bat" batch file, replaces it with the + executable that it wraps: the batch file adds another level of command-line + mangling that interferes with things like specifying version restrictions. + + Returns + ------- + edm : str + Path to the EDM executable to use. + + Raises + ------ + click.ClickException + If no EDM executable is found in the path. + """ + edm = shutil.which('edm') + if edm is None: + raise click.ClickException( + "This script requires EDM, but no EDM executable was found." + ) + + # Resolve edm.bat on Windows. + if sys.platform == "win32" and os.path.basename(edm) == "edm.bat": + edm = os.path.join(os.path.dirname(edm), "embedded", "edm.exe") + + return edm diff --git a/ets/environment.py b/ets/environment.py new file mode 100644 index 0000000..2eb729a --- /dev/null +++ b/ets/environment.py @@ -0,0 +1,313 @@ +# (C) Copyright 2005-2020 Enthought, Inc., Austin, TX +# All rights reserved. +# +# This software is provided without warranty under the terms of the BSD +# license included in LICENSE.txt and may be redistributed only under +# the conditions described in the aforementioned license. The license +# is also available online at http://www.enthought.com/licenses/BSD.txt +# +# Thanks for using Enthought open source! + +""" Code for working with Python environments. """ + +from abc import ABC, abstractmethod +import os +import shutil + +from .util import execute, do_in_existingdir, do_in_tempdir, update_os_environ + + +class Environment(ABC): + """ Abstract class for environments. + + Different concrete subclasses will use a particular environment manager, + like EDM or venv, to manage the state of the environment and execute + commands and operations within it. + + Attributes + ---------- + manager : str + A human-readable name of the environment manager. + environment : str + The name of the environment. + runtime : str + The Python runtime version for the environment. + """ + + def __init__(self, environment, runtime): + self.environment = environment + self.runtime = runtime + + @abstractmethod + def create(self, force=True): + """ Create the environment. + + Parameters + ---------- + force : bool + If force is True, any existing environment with the same name + will be removed. + """ + raise NotImplementedError() + + @abstractmethod + def clean(self): + """ Clean-up the environment. + + This attempts to destroy the environment, if possible. + """ + raise NotImplementedError() + + @abstractmethod + def install(self, packages): + """ Install packages. + + The default implementation uses pip to install. + + Parameters + ---------- + packages : list of package dicts + List of package specifications to install. A package + specification is a dictionary with items of the form:: + + manager: specification + + where manager is the name of the manager and specification is + a string which specifies a package in a way that the manager + understands. + """ + packages = [ + package['pip'] for package in packages + if 'pip' in package + ] + self.pip_install(packages) + + def install_source(self, path='.', editable=False): + """ Install a package from a source directory. + + Parameters + ---------- + path : pathlike + The path to the source directory. + editable : bool + Whether to perform an editable installation, so changes to the + source are automatically picked up. + """ + if editable: + self.invoke_module( + 'pip', 'install', '--no-dependencies', '--editable', str(path) + ) + else: + self.invoke_module( + 'pip', 'install', '--no-dependencies', str(path) + ) + + @abstractmethod + def uninstall(self, packages): + """ Uninstall packages. + + The default implementation uses pip to uninstall. + + Parameters + ---------- + packages : list of package dicts + List of package specifications to uninstall. A package + specification is a dictionary with items of the form:: + + manager: specification + + where manager is the name of the manager and specification is + a string which specifies a package in a way that the manager + understands. + """ + # default implementation uses pip to uninstall + packages = [ + package['pip'] for package in packages + if 'pip' in package + ] + self.pip_uninstall(packages) + + @abstractmethod + def invoke_module(self, module, *args): + """ Run a module using ``python -m`` within the environment. + + Parameters + ---------- + module : str + The name of the module to run. + *args + Additional arguments for the module. + """ + raise NotImplementedError() + + @abstractmethod + def invoke_script(self, script, *args): + """ Run a script within the environment. + + Parameters + ---------- + script : str + The name of the script to run. + *args + Additional arguments for the script. + """ + raise NotImplementedError() + + def pip_install(self, packages): + """ Install packages using pip. + + This should work in any well-behaved environment. + + Parameters + ---------- + packages : list of str + List of package specifications to install. + """ + self.invoke_module('pip', 'install', *packages) + + def pip_uninstall(self, packages): + """ Uninstall packages using pip. + + This should work in any well-behaved environment. + + Parameters + ---------- + packages : list of str + List of package specifications to uninstall. + """ + self.invoke_module('pip', 'uninstall', '-y', *packages) + + def run_tests(self, module, discover=True, verbose=True, + coveragerc='.coveragerc'): + """ Run unit tests with coverage. + + Parameters + ---------- + module : str + The name of the module to perform test discovery on. + discover : bool + Whether to perform test discovery. + verbose : bool + Whether to use verbose unittest output. + coveragerc : pathlike + The location of the .coveragerc file for generating + code coverage metrics. + """ + environ = {} + environ["PYTHONUNBUFFERED"] = "1" + + args = 'run -p -m -- unittest'.split() + if discover: + args.append('discover') + if verbose: + args.append('--verbose') + args.append(module) + with do_in_tempdir(files=[coveragerc], + capture_files=["./.coverage.*"]): + with update_os_environ(environ): + self.invoke_module('coverage', *args) + self.invoke_module('coverage', 'combine') + self.invoke_module('coverage', 'report', '-m') + + def flake8(self, dir): + """ Run flake8 over the codebase. + + Parameters + ---------- + dir : pathlike + The directory containing the source code. + """ + self.invoke_module('flake8', str(dir)) + + def mypy(self, dir): + """ Run mypy over the codebase. + + Parameters + ---------- + dir : pathlike + The directory containing the source code. + """ + self.invoke_module('mypy', str(dir)) + + def bandit(self, dir, level=2): + """ Run bandit over the codebase. + + Parameters + ---------- + dir : pathlike + The directory containing the source code. + """ + level = '-' + 'l'*level + self.invoke_module('bandit', level, '-r', str(dir)) + + def pydocstyle(self, dir, ): + """ Run pydocstyle over the codebase. + + Parameters + ---------- + dir : pathlike + The directory containing the source code. + """ + self.invoke_module('pydocstyle', str(dir)) + + def apidoc(self, source_dir, api_dir, exclude=[]): + """ Auto-generate API documentation. + + Parameters + ---------- + source_dir : pathlike + The directory containing the source code. + api_dir : pathlike + The directory containing the api documentation. + exclude : list of patterns + List of source file patterns to exclude from the API + documentation. + """ + if os.path.exists(api_dir): + shutil.rmtree(api_dir) + os.makedirs(api_dir) + self.invoke_script( + 'sphinx-apidoc', "-e", "-M", "-o", str(api_dir), str(source_dir), + *exclude + ) + + def build_docs(self, docs_dir='docs', error_on_warn=False): + """ Build sphinx documentation. + + Parameters + ---------- + docs_dir : pathlike + The top-level directory containing the documentation. + error_on_warn : bool + Whether to raise an error if there are any warnings generated + by Sphinx. + """ + args = ["-b", "html"] + if error_on_warn: + args.append("-W") + args += ["-d", "build/doctrees", "source", "build/html"] + + with do_in_existingdir(docs_dir): + self.invoke_script('sphinx-build', *args) + + def execute(self, commands): + """ Execute a series of commands. + + This is a wrapper around the :func:`ets.utils.execute` function which + uses the environment as a namespace for parameter substitution in + arguments. + + Parameters + ---------- + commands : list of list of str + A list of commands, each command a list of strings suitable for + use with :func:`subprocess.call` and related functions. + + Raises + ------ + SystemExit + If any command fails, :func:`sys.exit` will be called with error + code 1. + """ + parameters = self.__dict__ + execute(commands, parameters) diff --git a/ets/tests/__init__.py b/ets/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ets/tests/test_venv.py b/ets/tests/test_venv.py new file mode 100644 index 0000000..0f3fcff --- /dev/null +++ b/ets/tests/test_venv.py @@ -0,0 +1,63 @@ +# (C) Copyright 2005-2020 Enthought, Inc., Austin, TX +# All rights reserved. +# +# This software is provided without warranty under the terms of the BSD +# license included in LICENSE.txt and may be redistributed only under +# the conditions described in the aforementioned license. The license +# is also available online at http://www.enthought.com/licenses/BSD.txt +# +# Thanks for using Enthought open source! + +from pathlib import Path +import sys +from tempfile import TemporaryDirectory +from unittest import TestCase + +from ..venv import Venv +from ..util import update_os_environ + +current_runtime = '{}.{}'.format(*sys.version_info[:2]) + + +class TestVenv(TestCase): + + def test_init(self): + with TemporaryDirectory() as tempdir: + with update_os_environ({'ETS_VENV_PATH': tempdir}): + env = Venv("test", current_runtime) + + self.assertEqual(env.environment, "test") + self.assertEqual(env.runtime, current_runtime) + + def test_create_clean(self): + with TemporaryDirectory() as tempdir: + with update_os_environ({'ETS_VENV_PATH': tempdir}): + env = Venv("test", current_runtime) + env.create() + + self.assertEqual(env.environment, "test") + self.assertEqual(env.runtime, current_runtime) + + path = Path(tempdir, 'test') + + self.assertTrue(path.exists()) + self.assertTrue(path.is_dir()) + + env.clean() + + self.assertFalse(path.exists()) + + def test_install(self): + with TemporaryDirectory() as tempdir: + with update_os_environ({'ETS_VENV_PATH': tempdir}): + env = Venv("test", current_runtime) + env.create() + + path = Path(tempdir, 'test') + + self.assertTrue(path.exists()) + self.assertTrue(path.is_dir()) + + env.clean() + + self.assertFalse(path.exists()) diff --git a/ets/util.py b/ets/util.py new file mode 100644 index 0000000..6ea18ae --- /dev/null +++ b/ets/util.py @@ -0,0 +1,136 @@ +# (C) Copyright 2005-2020 Enthought, Inc., Austin, TX +# All rights reserved. +# +# This software is provided without warranty under the terms of the BSD +# license included in LICENSE.txt and may be redistributed only under +# the conditions described in the aforementioned license. The license +# is also available online at http://www.enthought.com/licenses/BSD.txt +# +# Thanks for using Enthought open source! + +""" +Utility methods for code execution and directory management. +""" + +from contextlib import contextmanager +import glob +import os +from shutil import copy, rmtree +import subprocess +import sys +from tempfile import mkdtemp + +import click + + +@contextmanager +def do_in_tempdir(files=(), capture_files=()): + """ Create a temporary directory, cleaning up after done. + + Creates the temporary directory, and changes into it. On exit returns to + original directory and removes temporary dir. + + Parameters + ---------- + files : sequence of filenames + Files to be copied across to temporary directory. + capture_files : sequence of filenames + Files to be copied back from temporary directory. + """ + path = mkdtemp() + old_path = os.getcwd() + + # send across any files we need + for filepath in files: + click.echo("copying file to tempdir: {}".format(filepath)) + copy(filepath, path) + + os.chdir(path) + try: + yield path + # retrieve any result files we want + for pattern in capture_files: + for filepath in glob.iglob(pattern): + click.echo("copying file back: {}".format(filepath)) + copy(filepath, old_path) + finally: + os.chdir(old_path) + rmtree(path) + + +@contextmanager +def do_in_existingdir(path): + """ Change directory to an existing directory, change back when done. + + Parameters + ---------- + path : str + Path of the directory to be changed into. + """ + old_path = os.getcwd() + os.chdir(path) + try: + yield path + finally: + os.chdir(old_path) + + +@contextmanager +def update_os_environ(environ): + """ Temporarily set environment variables. + + This will overwrite existing environment variable values, or + create new environment variables as needed. The original state + will be restored when the context manager exits. + + Parameters + ---------- + environ : dict + Dictionary of new environment variables to set. + """ + old_values = { + key: os.environ[key] + for key in environ + if key in os.environ + } + os.environ.update(environ) + try: + yield + finally: + for key in set(environ) - set(old_values): + del os.environ[key] + os.environ.update(old_values) + + +def execute(commands, parameters): + """ Execute a sequence of commands, substituting parameter values. + + Each command is a list of strings, each string being an argument of the + command. Python format-style substitution is performed on each argument + prior to the command being executed. + + Parameters + ---------- + commands : list of list of str + A list of commands, each command being a list of strings forming the + parts of a command to be run by :func:`subprocess.call` or similar. + functions. The strings may contain Python keyword format-style + formatting for parameter substitution. + parameters : dict + A mapping of parameter names to values for substitution into command + arguments. + + Raises + ------ + SystemExit + If the subprocess has an error, the function raises :func:`sys.exit` + with error code 1. + """ + for command in commands: + command_list = [arg.format(**parameters) for arg in command] + click.echo("[EXECUTING] {}".format(' '.join(command_list))) + try: + subprocess.check_call(command_list) # nosec + except subprocess.CalledProcessError as exc: + print(exc) + sys.exit(1) diff --git a/ets/venv.py b/ets/venv.py new file mode 100644 index 0000000..6ae2b99 --- /dev/null +++ b/ets/venv.py @@ -0,0 +1,195 @@ +# (C) Copyright 2005-2020 Enthought, Inc., Austin, TX +# All rights reserved. +# +# This software is provided without warranty under the terms of the BSD +# license included in LICENSE.txt and may be redistributed only under +# the conditions described in the aforementioned license. The license +# is also available online at http://www.enthought.com/licenses/BSD.txt +# +# Thanks for using Enthought open source! + +""" Code for working with venv virtual environments. """ + +import os +from pathlib import Path +import sys +import shutil + +import click + +from .environment import Environment + + +#: virtualenv wrapper default, good enough for a default +default_venv_path = Path('~/.virtualenvs') + + +def get_venv_path(): + """ Heuristics for the place to locate virtual environments. + + This method looks for (in order of priority): + + - an ``ETS_VENV_PATH`` enviornment variable + - a ``WORKON_HOME`` environment variable + + If neither exits, it uses ``~/.virtualenvs``. This can be overridden + by changing the module-level :obj:`default_venv_path` value. + + Returns + ------- + path : Path + The best-guess at the location of virtual environments. + """ + # virtualenv wrapper and other tools use 'WORKON_HOME' + virtualenv_wrapper_path = os.environ.get('WORKON_HOME', default_venv_path) + + # ETS-specific environment variable that overrides the above + venv_path = os.environ.get('ETS_VENV_PATH', virtualenv_wrapper_path) + + return Path(venv_path).expanduser().resolve() + + +class Venv(Environment): + """ A venv-based virtual environment. + + Attributes + ---------- + manager : str + A human-readable name of the environment manager. + environment : str + The name of the environment. + runtime : str + The Python runtime version for the environment. + base_path : Path + The root of the virtual environment in the filesystem. + script_dir : Path + The path to the script directory within the virtual environment. + python : Path + The path to the python executable within the virtual environment. + bootstrap_python : Path + The path to a python executable that can be used to create the + environment. + """ + + #: The user-facing name of the environment manager. + manager = "venv" + + def __init__(self, environment, runtime): + super().__init__(environment, runtime) + base_path = Path(environment) + if not base_path.is_absolute(): + base_path = get_venv_path().joinpath(base_path) + + self.base_path = base_path.resolve() + + if sys.platform == "win32": + self.script_dir = self.base_path / "Scripts" + else: + self.script_dir = self.base_path / "bin" + + self.python = self.script_dir / "python" + self.bootstrap_python = get_python(runtime) + + def create(self, force=True): + """ Create the environment. + + Parameters + ---------- + force : bool + If force is True, any existing environment with the same name + will be removed. + """ + install_command = "{bootstrap_python} -m venv {base_path}".split() + if force: + install_command.append("--clear") + self.execute([install_command]) + self.invoke_module('pip', 'install', '--upgrade', 'pip') + + def clean(self): + """ Clean-up the environment. + + This attempts to destroy the environment, if possible. + + Note + ---- + On windows this may fail if any file in the environment is open. + """ + # XXX may fail on Windows if files in environment are in use + shutil.rmtree(self.base_path) + + def install(self, packages): + """ Install packages. + + This class uses pip to install packages. + + Parameters + ---------- + packages : list of package dicts + List of package specifications to install. A package + specification is a dictionary with items of the form:: + + manager: specification + + where manager is the name of the manager and specification is + a string which specifies a package in a way that the manager + understands. + """ + super().install(packages) + + def uninstall(self, packages): + """ Uninstall packages. + + This class uses pip to uninstall packages. + + Parameters + ---------- + packages : list of package dicts + List of package specifications to uninstall. A package + specification is a dictionary with items of the form:: + + manager: specification + + where manager is the name of the manager and specification is + a string which specifies a package in a way that the manager + understands. + """ + super().uninstall(packages) + + def invoke_module(self, module, *args): + """ Run a module using ``python -m`` within the environment. + + Parameters + ---------- + module : str + The name of the module to run. + *args + Additional arguments for the module. + """ + command = ["{python}", "-m", module] + list(args) + self.execute([command]) + + def invoke_script(self, script, *args): + """ Run a script within the environment. + + Parameters + ---------- + script : str + The name of the script to run. + *args + Additional arguments for the script. + """ + path = self.script_dir / script + command = [str(path)] + list(args) + self.execute([command]) + + +def get_python(runtime): + """ Find the requested Python version to run the venv module. + """ + python = shutil.which('python'+runtime) + if python is None: + msg = "This script requires {python}, but no {python} executable was found." # noqa + raise click.ClickException( + msg.format(python="Python "+runtime) + ) + return python diff --git a/etstool.py b/etstool.py new file mode 100644 index 0000000..aec607f --- /dev/null +++ b/etstool.py @@ -0,0 +1,278 @@ +# (C) Copyright 2005-2020 Enthought, Inc., Austin, TX +# All rights reserved. +# +# This software is provided without warranty under the terms of the BSD +# license included in LICENSE.txt and may be redistributed only under +# the conditions described in the aforementioned license. The license +# is also available online at http://www.enthought.com/licenses/BSD.txt +# +# Thanks for using Enthought open source! + +from pathlib import Path + +import click + +from ets.click_helpers import ( + docs_option, editable_option, environment_manager_option, + environment_option, runtime_option, tests_option, verbose_option +) + +ets_repo_path = Path(__file__).parent + +dependencies = [ + { + 'pip': "bandit", + }, + { + 'pip': "click", + 'edm': "click", + }, + { + 'pip': "coverage", + 'edm': "coverage", + }, + { + 'pip': "flake8", + 'edm': "flake8", + }, + { + 'pip': "mypy", + }, + { + 'pip': "pydocstyle", + 'edm': "pydocstyle", + }, +] +test_dependencies = [] +doc_dependencies = [ + { + 'pip': "sphinx", + 'edm': "sphinx", + }, + { + 'pip': "enthought_sphinx_theme", + 'edm': "enthought_sphinx_theme", + }, +] + + +@click.group() +def cli(): + """ + Developer and CI support commands for ETS. + """ + pass + + +@cli.command() +@runtime_option +@environment_option +@environment_manager_option +@editable_option +@tests_option +@docs_option +def install(runtime, environment, environment_manager, editable, tests, docs): + """ Install the project in a new environemnt. + + Creates a clean environment using the specified environment management + system and installs all required packages for the project. Optionally + installs further dependencies required for testing and building + documentation. + """ + message = "Creating environment '{}' with {}" + click.echo( + click.style( + message.format( + environment_manager.environment, + environment_manager.manager, + ), + bold=True, + ) + ) + + packages = dependencies[:] + if tests: + packages += test_dependencies + if docs: + packages += doc_dependencies + + environment_manager.create() + environment_manager.install(packages) + environment_manager.install_source(ets_repo_path, editable=editable) + + +@cli.command() +@runtime_option +@environment_option +@environment_manager_option +@verbose_option +def test(runtime, environment, environment_manager, verbose): + module = 'ets.tests' + message = "Running tests for '{}' in '{}'" + click.echo( + click.style( + message.format(module, environment_manager.environment), + bold=True, + ) + ) + environment_manager.run_tests( + module, + coveragerc=ets_repo_path / "setup.cfg" + ) + + +@cli.command() +@runtime_option +@environment_option +@environment_manager_option +@verbose_option +def flake8(runtime, environment, environment_manager, verbose): + dir = 'ets' + message = "Running flake8 for '{}' in '{}'" + click.echo( + click.style( + message.format(dir, environment_manager.environment), + bold=True, + ) + ) + environment_manager.flake8(ets_repo_path / dir) + + +@cli.command() +@runtime_option +@environment_option +@environment_manager_option +@verbose_option +def bandit(runtime, environment, environment_manager, verbose): + dir = 'ets' + message = "Security checks for '{}' in '{}'" + click.echo( + click.style( + message.format(dir, environment_manager.environment), + bold=True, + ) + ) + environment_manager.bandit(ets_repo_path / dir) + + +@cli.command() +@runtime_option +@environment_option +@environment_manager_option +@verbose_option +def pydocstyle(runtime, environment, environment_manager, verbose): + dir = 'ets' + message = "Checking docstrings for '{}' in '{}'" + click.echo( + click.style( + message.format(dir, environment_manager.environment), + bold=True, + ) + ) + environment_manager.pydocstyle(ets_repo_path / dir) + + +@cli.command() +@runtime_option +@environment_option +@environment_manager_option +@verbose_option +def mypy(runtime, environment, environment_manager, verbose): + dir = 'ets' + message = "Running mypy for '{}' in '{}'" + click.echo( + click.style( + message.format(dir, environment_manager.environment), + bold=True, + ) + ) + environment_manager.mypy(ets_repo_path / dir) + + +@cli.command() +@runtime_option +@environment_option +@environment_manager_option +@verbose_option +def apidoc(runtime, environment, environment_manager, verbose): + package = "ets" + click.echo("Generating API documentation for '{}' in '{}'".format( + package, + environment_manager.environment)) + environment_manager.apidoc( + ets_repo_path / package, + ets_repo_path / "docs" / "source" / "api", + exclude=[str(ets_repo_path / package / "scripts")], + ) + + +@cli.command() +@runtime_option +@environment_option +@environment_manager_option +@verbose_option +def build_docs(runtime, environment, environment_manager, verbose): + click.echo("Building documentation for '{}'".format( + environment_manager.environment)) + environment_manager.build_docs(docs_dir=ets_repo_path / "docs") + + +@cli.command() +@runtime_option +@environment_option +@environment_manager_option +@click.argument('command', nargs=-1, required=True) +def run(runtime, environment, environment_manager, command): + print(command) + environment_manager.invoke_script(*command) + + +@cli.command() +@runtime_option +@environment_option +@environment_manager_option +def clean(runtime, environment, environment_manager): + click.echo("Cleaning environment '{}'".format( + environment_manager.environment)) + environment_manager.clean() + + +@cli.command() +@runtime_option +@click.option( + "--environment-manager", + default='venv', + help="Environment management tool to use", +) +@docs_option +def test_clean(runtime, environment_manager, docs): + """ Run tests and build documentation in a clean environment. + + A clean environment is created for the test run, and removed + again afterwards. + """ + args = [ + "--runtime={}".format(runtime), + "--environment-manager={}".format(environment_manager), + ] + if docs: + install_args = args + ['--docs'] + else: + install_args = args + ['--no-docs'] + + try: + install(args=install_args, standalone_mode=False) + flake8(args=args, standalone_mode=False) + mypy(args=args, standalone_mode=False) + bandit(args=args, standalone_mode=False) + pydocstyle(args=args, standalone_mode=False) + test(args=args, standalone_mode=False) + if docs: + apidoc(args=args, standalone_mode=False) + build_docs(args=args, standalone_mode=False) + finally: + clean(args=args, standalone_mode=False) + + +if __name__ == "__main__": + cli(prog_name="python etstool.py") diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ets.py b/scripts/ets.py similarity index 90% rename from ets.py rename to scripts/ets.py index 41a4d7c..5c3be67 100644 --- a/ets.py +++ b/scripts/ets.py @@ -1,5 +1,18 @@ #! /usr/bin/env python -"""A thin replacement for ETSProjectTools. Performs checkout, update, install, +#nosec + +# (C) Copyright 2005-2020 Enthought, Inc., Austin, TX +# All rights reserved. +# +# This software is provided without warranty under the terms of the BSD +# license included in LICENSE.txt and may be redistributed only under +# the conditions described in the aforementioned license. The license +# is also available online at http://www.enthought.com/licenses/BSD.txt +# +# Thanks for using Enthought open source! + +""" +A thin replacement for ETSProjectTools. Performs checkout, update, install, build, etc, of all actively maintained ETS packages, and allows arbitrary shell commands to be run on all packages. """ diff --git a/ets_docs.py b/scripts/ets_docs.py similarity index 67% rename from ets_docs.py rename to scripts/ets_docs.py index 5fd00a1..9122aba 100644 --- a/ets_docs.py +++ b/scripts/ets_docs.py @@ -1,4 +1,17 @@ #! /usr/bin/env python +# flake8: noqa +# type: ignore +# nosec + +# (C) Copyright 2005-2020 Enthought, Inc., Austin, TX +# All rights reserved. +# +# This software is provided without warranty under the terms of the BSD +# license included in LICENSE.txt and may be redistributed only under +# the conditions described in the aforementioned license. The license +# is also available online at http://www.enthought.com/licenses/BSD.txt +# +# Thanks for using Enthought open source! """A thin replacement for SetupDocs (which is no longer part of ETS). Performs documentation building, check in, updating, etc of all actively maintained ETS packages. @@ -62,7 +75,7 @@ def main(): if len(sys.argv) < 2 or sys.argv[1].startswith('-'): - print usage % (aliases, ets_package_names) + print(usage % (aliases, ets_package_names)) return arg1 = sys.argv[1] @@ -75,7 +88,7 @@ def main(): ets_packages = ets_package_names.split() for ets_pkg_name in ets_packages: - print "Updating documentation branch for {0}...".format(ets_pkg_name) + print("Updating documentation branch for {0}...".format(ets_pkg_name)) # Find the current branch, so that we may return to it branches = subprocess.check_output(['git', 'branch'], cwd=ets_pkg_name) @@ -85,13 +98,13 @@ def main(): # Checkout the gh-pages branch try: subprocess.check_call(['git', 'checkout', 'gh-pages'], cwd=ets_pkg_name) - except (OSError, subprocess.CalledProcessError), detail: - print " Error running command in package %s:\n %s" % (ets_pkg_name, detail) - raw_input(" Press enter to process remaining packages.") + except (OSError, subprocess.CalledProcessError) as detail: + print(" Error running command in package %s:\n %s" % (ets_pkg_name, detail)) + input(" Press enter to process remaining packages.") continue # Copy the files over - print "Copying files for {0}".format(ets_pkg_name) + print("Copying files for {0}".format(ets_pkg_name)) if ets_pkg_name == 'mayavi': copy_tree(ets_pkg_name + '/docs/build/tvtk/html/', ets_pkg_name + '/tvtk/') copy_tree(ets_pkg_name + '/docs/build/mayavi/html/', ets_pkg_name + '/mayavi/') @@ -101,33 +114,33 @@ def main(): # Add everything to the repository try: subprocess.check_call(['git', 'add', '.'], cwd=ets_pkg_name) - except (OSError, subprocess.CalledProcessError), detail: - print " Error running command in package %s:\n %s" % (ets_pkg_name, detail) - raw_input(" Press enter to process remaining packages.") + except (OSError, subprocess.CalledProcessError) as detail: + print(" Error running command in package %s:\n %s" % (ets_pkg_name, detail)) + input(" Press enter to process remaining packages.") continue # Commit to the repo. try: subprocess.check_call(['git', 'commit', '-a', '-m', '"Updated docs."'], cwd=ets_pkg_name) - except (OSError, subprocess.CalledProcessError), detail: - print " Error running command in package %s:\n %s" % (ets_pkg_name, detail) - raw_input(" Press enter to process remaining packages.") + except (OSError, subprocess.CalledProcessError) as detail: + print(" Error running command in package %s:\n %s" % (ets_pkg_name, detail)) + input(" Press enter to process remaining packages.") continue # Push these changes. try: subprocess.check_call(['git', 'push'], cwd=ets_pkg_name) - except (OSError, subprocess.CalledProcessError), detail: - print " Error running command in package %s:\n %s" % (ets_pkg_name, detail) - raw_input(" Press enter to process remaining packages.") + except (OSError, subprocess.CalledProcessError) as detail: + print(" Error running command in package %s:\n %s" % (ets_pkg_name, detail)) + input(" Press enter to process remaining packages.") continue # Return to the current branch try: subprocess.check_call(['git', 'checkout', current_branch], cwd=ets_pkg_name) - except (OSError, subprocess.CalledProcessError), detail: - print " Error running command in package %s:\n %s" % (ets_pkg_name, detail) - raw_input(" Press enter to process remaining packages.") + except (OSError, subprocess.CalledProcessError) as detail: + print(" Error running command in package %s:\n %s" % (ets_pkg_name, detail)) + input(" Press enter to process remaining packages.") continue print @@ -144,14 +157,14 @@ def main(): # Run the command in each project directory for ets_pkg_name in ets_package_names.split(): - print "Running command %r in package %s" % (' '.join(cmd), ets_pkg_name) + print("Running command %r in package %s" % (' '.join(cmd), ets_pkg_name)) try: subprocess.check_call(cmd, cwd=ets_pkg_name + '/docs/') print - except (OSError, subprocess.CalledProcessError), detail: - print " Error running command in package %s:\n %s" % (ets_pkg_name, detail) - raw_input(" Press enter to process remaining packages.") + except (OSError, subprocess.CalledProcessError) as detail: + print(" Error running command in package %s:\n %s" % (ets_pkg_name, detail)) + input(" Press enter to process remaining packages.") if __name__ == "__main__": diff --git a/setup.py b/setup.py index 548bfa2..75614e6 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,19 @@ -# Copyright (c) 2008-2012 by Enthought, Inc. +# (C) Copyright 2005-2020 Enthought, Inc., Austin, TX # All rights reserved. +# +# This software is provided without warranty under the terms of the BSD +# license included in LICENSE.txt and may be redistributed only under +# the conditions described in the aforementioned license. The license +# is also available online at http://www.enthought.com/licenses/BSD.txt +# +# Thanks for using Enthought open source! import sys -from setuptools import setup +from setuptools import setup, find_packages -setup_data = {} -execfile('setup_data.py', setup_data) +setup_data = {'requirements': []} +exec(open('setup_data.py').read(), setup_data) INFO = setup_data['INFO'] if 'develop' in sys.argv: @@ -14,38 +21,17 @@ # The actual setup call. setup( - name = 'ets', - version = INFO['version'], - author = 'Enthought, Inc.', - download_url = ('http://www.enthought.com/repo/ets/ets-%s.tar.gz' % - INFO['version']), - author_email = 'info@enthought.com', - classifiers = [c.strip() for c in """\ - Development Status :: 4 - Beta - Intended Audience :: Developers - Intended Audience :: Science/Research - License :: OSI Approved :: BSD License - Operating System :: MacOS - Operating System :: Microsoft :: Windows - Operating System :: OS Independent - Operating System :: POSIX - Operating System :: Unix - Programming Language :: Python - Topic :: Scientific/Engineering - Topic :: Software Development - Topic :: Software Development :: Libraries - """.splitlines() if len(c.strip()) > 0], - description = 'Enthought Tool Suite meta-project', - long_description = open('README.rst').read(), - install_requires = INFO['install_requires'], - license = 'BSD', - maintainer = 'ETS Developers', - maintainer_email = 'enthought-dev@enthought.com', - py_modules = ["ets", "ets_docs"], - entry_points = dict(console_scripts=[ - "ets = ets:main", - "ets-docs = ets_docs:main", - ]), - platforms = ["Windows", "Linux", "Mac OS-X", "Unix", "Solaris"], - url = 'http://code.enthought.com/projects/tool-suite.php', + version=INFO['version'], + install_requires=INFO['install_requires'], + packages=find_packages(), + entry_points={ + # 'console_scripts': [ + # "ets = ets.scripts.ets:main", + # "ets-docs = ets.scripts.ets_docs:main", + # ], + "flake8.extension": [ + "H = ets.copyright_header:CopyrightHeaderExtension", + ], + }, +# platforms=["Windows", "Linux", "Mac OS-X", "Unix", "Solaris"], ) diff --git a/setup_data.py b/setup_data.py index 50bc645..8d4f90e 100644 --- a/setup_data.py +++ b/setup_data.py @@ -1,23 +1,23 @@ requirements = [ - ('apptools', '4.2.1'), - ('blockcanvas', '4.0.3'), - ('chaco', '4.5.0'), - ('codetools', '4.2.0'), - ('enable', '4.4.1'), - ('encore', '0.6.0'), - ('envisage', '4.4.0'), - ('etsdevtools', '4.0.2'), - ('etsproxy', '0.1.2'), - ('graphcanvas', '4.0.2'), - ('mayavi', '4.3.1'), - ('pyface', '4.4.0'), - ('scimath', '4.1.2'), - ('traits', '4.5.0'), - ('traitsui', '4.4.0'), +# ('apptools', '4.2.1'), +# ('blockcanvas', '4.0.3'), +# ('chaco', '4.5.0'), +# ('codetools', '4.2.0'), +# ('enable', '4.4.1'), +# ('encore', '0.6.0'), +# ('envisage', '4.4.0'), +# ('etsdevtools', '4.0.2'), +# ('etsproxy', '0.1.2'), +# ('graphcanvas', '4.0.2'), +# ('mayavi', '4.3.1'), +# ('pyface', '4.4.0'), +# ('scimath', '4.1.2'), +# ('traits', '4.5.0'), +# ('traitsui', '4.4.0'), ] INFO = { 'name': 'ets', - 'version': '4.4.4', - 'install_requires': ['%s >= %s.dev' % nv for nv in requirements], + 'version': '5.0.0', + 'install_requires': ['click'] + ['%s >= %s.dev' % nv for nv in requirements], }