From eed971d704a771640f18e3db607388e070d360fd Mon Sep 17 00:00:00 2001 From: Richard Maynard Date: Mon, 26 May 2025 12:24:21 -0500 Subject: [PATCH 1/5] replace README --- README.md | 303 +++++++++++++++++++++++++++--------------------------- 1 file changed, 151 insertions(+), 152 deletions(-) diff --git a/README.md b/README.md index c0ceb35..bcf10a7 100644 --- a/README.md +++ b/README.md @@ -1,224 +1,223 @@ # terraform-worker -`terraform-worker` is a command line tool for pipelining terraform operations while sharing state between them. The worker consumese a yaml configuration file which is broken up into two sections, definitions (which were really just top level modules) and sub-modules. The definitions are put into a worker config in order, with the terraform variables, and remote state variables. Following is a sample configuration file and command: +**terraform-worker** is a command-line tool purpose-built for orchestrating large-scale, modular Terraform deployments. It enables precise control over provider versions, remote state scaffolding, and shared configuration logic β€” ideal for enterprise infrastructure teams managing complex environments across multiple workspaces and deployment stages. + +Unlike tools such as `terragrunt`, this is not a drop-in wrapper β€” `terraform-worker` introduces a new structure and workflow model, optimized for maintainability, scalability, and declarative control, and integration with CI/CD pipelines. + +--- + +## πŸš€ Features + +- πŸ”§ Modularized state management across nested or sequential Terraform operations +- 🧱 Automated scaffolding of provider blocks, remote state configs, and input variables +- ✨ Jinja2 templating for flexible, environment-specific rendering +- πŸ”’ Enforces version consistency of modules and providers +- 🧰 Supports YAML, JSON, and HCL2 for configuration definitions +- πŸ“¦ Clean CLI with pluggable backend and runtime overrides +- πŸ” Designed from the ground up for enterprise environments + +--- + +## πŸ”„ Example Workflow + +Here's a minimal example using `worker.yaml` to scaffold a VPC and RDS database with shared state: -*./worker.yaml* ```yaml terraform: providers: aws: vars: region: {{ aws_region }} - version: "~> 2.61.0" + version: "~> 4.0" - # global level variables terraform_vars: region: {{ aws_region }} environment: dev definitions: - # Either setup a VPC and resources, or deploy into an existing one network: path: /definitions/aws/network-existing database: path: /definitions/aws/rds + remote_vars: + subnet: network.outputs.subnet_id ``` -```sh -% worker --aws-profile default --backend s3 terraform example1 -``` -**NOTE:** When adding a provider from a non-hashicorp source, use a `source` field, as follows -(_the `source` field is only valid for terraform 13+ and is not emitted when using 12_): +Run the deployment: -```yaml -providers: -... - kubectl: - vars: - version: "~> 1.9" - source: "gavinbunney/kubectl" +```sh +% worker --aws-profile default --config-var aws_region=us-west-2 terraform deploy ``` -In addition to using command line options, worker configuration can be specified using a `worker_options` section in -the worker configuration. +Use custom worker options for flexibility: ```yaml terraform: worker_options: backend: s3 backend_prefix: tfstate - terraform_bin: /home/user/bin/terraform - - providers: -... + terraform_bin: /usr/local/bin/terraform ``` -**terraform-worker** requires a configuration file. By default, it will looks for a file named "worker.yaml" in the -current working directory. Together with the `worker_options` listed above, it's possible to specify all options -either in the environment or in the configuration file and simply call the worker command by itself. +--- -```sh - % env | grep AWS - AWS_ACCESS_KEY_ID=somekey - AWS_SECRET_ACCESS_KEY=somesecret - % head ./worker.yaml -terraform: - worker_options: - backend: s3 - backend_prefix: tfstate - terraform_bin: /home/user/bin/terraform - % worker terraform my-deploy +## πŸ“Œ Configuration Notes + +- Config files may be written in YAML, JSON, or HCL2 +- Configs support templating via [Jinja2](https://jinja.palletsprojects.com/) +- `terraform-worker` reads `worker.yaml` by default from the current directory +- Variables like `{{ aws_region }}` are injected via `--config-var` CLI args + +--- + +## 🧠 How It Works + +1. Parses your configuration into **definitions** (state modules) +2. Injects provider/version/variable blocks automatically +3. Handles remote state linking across module outputs +4. Creates ephemeral execution directories (unless `--no-clean` is used) +5. Delegates to native Terraform for execution (`init`, `plan`, `apply`, etc.) + +--- + +## πŸ§ͺ Development & Testing + +Initialize the environment: + +```bash +poetry install +make init ``` -## Assuming a Role +Run tests: + +```bash +pytest +``` -The first step in assuming a role is to create the role to be assumed as documented in [Creating a role to delegate permissions to an IAM user ](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user.html) and then granting permissions to assume the role as documented in [Granting a user permissions to switch roles ](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_permissions-to-switch.html). +--- -To have the worker assume a role once the role and permissions are configured the `--aws-role-arn` and `--aws-external-id` flags need to be provided to the worker along with the credentials for the trusted account. Since neither the role ARN nor the ExternalId are secret, this allows running under another set of credentials without providing any additional secrets. +## πŸ“€ Releasing -## Development +Update the version and publish to PyPI: -```sh - # virtualenv setup stuff... and then: - % pip install poetry && make init +```bash +poetry version +poetry publish --build ``` -## Releasing +Configure credentials for Poetry using [these instructions](https://python-poetry.org/docs/repositories/#configuring-credentials). + +--- -Publishing a release to PYPI is done locally through poetry. Instructions on how to configure credentials for poetry can be found [here](https://python-poetry.org/docs/repositories/#configuring-credentials). +## πŸ›  Troubleshooting + +Use `--no-clean` to retain generated terraform files: -Bump the version of the worker and commit the change: ```sh - % poetry version +worker --no-clean terraform deploy ``` -Build and publish the package to PYPI: -```sh - % poetry publish --build +The tool will print the temporary directory it used, e.g.: + +```text +using temporary Directory: /tmp/tmpew44uopp ``` -## Configuration +You can `cd` into the generated definition to run manual Terraform commands for debugging. -A project is configured through a worker config, a yaml, json, or hcl2 file that specifies the definitions, inputs, outputs, providers and all other necessary configuration. The worker config is what specifies how state is shared among your definitions. The config support jinja templating that can be used to conditionally pass state or pass in env variables through the command line via the `--config-var` option. +--- -*./worker.yaml* -```yaml -terraform: - providers: - aws: - vars: - region: {{ aws_region }} - version: "~> 2.61.1" +## πŸͺ Hook Scripts - # global level variables - terraform_vars: - region: {{ aws_region }} - environment: dev +Each Terraform action (`init`, `plan`, `apply`) can be extended via optional lifecycle hook scripts, enabling advanced custom logic around deployments. - definitions: - # Either setup a VPC and resources, or deploy into an existing one - network: - path: /definitions/aws/network-existing +### Supported Hook Types - database: - path: /definitions/aws/rds - remote_vars: - subnet: network.outputs.subnet_id -``` +For every definition, the following scripts are recognized and executed if present: -```json -{ - "terraform": { - "providers": { - "aws": { - "vars": { - "region": "{{ aws_region }}", - "version": "~> 2.61" - } - } - }, - "terraform_vars": { - "region": "{{ aws_region }}", - "environment": "dev" - }, - "definitions": { - "network": { - "path": "/definitions/aws/network-existing" - }, - "database": { - "path": "/definitions/aws/rds", - "remote_vars": { - "subnet": "network.outputs.subnet_id" - } - } - } - } -} -``` +- `pre_init`, `post_init` +- `pre_plan`, `post_plan` +- `pre_apply`, `post_apply` + +Scripts must be placed within the corresponding definition directory. + +### Language & Execution + +Hook scripts can be written in **any language** β€” they are executed as standalone shell commands. Ensure the script has execution permissions (e.g., `chmod +x`). -```hcl -terraform { - providers { - aws = { - vars = { - region = "{{ aws_region }}" - version = "2.63.0" - } - } - } - - terraform_vars { - environment = "dev" - region = "{{ aws_region }}" - } - - definitions { - network = { - path = "/definitions/aws/network-existing" - } - - database = { - path = "/definitions/aws/rds" - - remote_vars = { - subnet = "network.outputs.subnet_id" - } - } - } -} +Example: +```bash +#!/usr/bin/env python3 +import os +print("Pre-apply hook running for", os.environ.get("TF_WORKER_DEFINITION")) ``` -In this config, the worker manages two separate terraform modules, a `network` and a `database` definition, and shares an output from the network definition with the database definition. This is made available inside of the `database` definition through the `local.subnet` value. +### Environment Variable Access -`aws_region` is substituted at runtime for the value of `--aws-region` passed through the command line. +The following environment variables are automatically injected into all hook scripts: -## Troubleshooting +- `TF_`-prefixed variables for: + - Terraform input variables (`TF_REGION`, `TF_ENVIRONMENT`, etc.) + - Rendered template variables (e.g., Jinja context) + - Remote state outputs +- Authentication credentials (e.g., `AWS_ACCESS_KEY_ID`, `GOOGLE_APPLICATION_CREDENTIALS`, etc.) -Running the worker with the `--no-clean` option will keep around the terraform files that the worker generates. You can use these generated files to directly run terraform commands for that definition. This is useful for when you need to do things like troubleshoot or delete items from the remote state. After running the worker with --no-clean, cd into the definition directory where the terraform-worker generates it's tf files. The worker should tell you where it's putting these for example: +This allows hook scripts to dynamically read and interact with Terraform configuration and authentication context. -``` -... -building deployment mfaitest -using temporary Directory: /tmp/tmpew44uopp -... -``` +### Use Cases + +- Pre-checks or assertions before deploys +- Dynamic secrets injection +- Notifications or logging +- External system integration (e.g., Slack, monitoring) -In order to troubleshoot this definition, you would cd /tmp/tmpew44uopp/definitions/my_definition/ and then perform any terraform commands from there. +Hook scripts provide a powerful extension point for customizing behavior around each Terraform stage without modifying core logic. -## Background +--- + +## πŸ”’ Assuming a Role (AWS) + +To execute with a specific IAM role: + +```sh +worker --aws-role-arn arn:aws:iam::1234567890:role/example \ + --aws-external-id my-id \ + terraform deploy +``` -The terraform worker was a weekend project to run terraform against a series of definitions (modules). The idea was the configuration vars, provider configuration, remote state, and variables from remote state would all be dynamically generated. The purpose was for building kubernetes deployments, and allowing all of the configuration information to be stored as either yamnl files in github, or have the worker configuration generated by an API which stored all of the deployment configurations in a database. +Configure the role and trust relationship per [AWS IAM guidelines](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user.html). -## Documentation +--- -Documentation uses the [Sphinx](https://www.sphinx-doc.org/en/master/index.html) documentation fromework. +## πŸ“š Documentation -To build HTML documentation: +Full documentation is built using [Sphinx](https://www.sphinx-doc.org/en/master/): ```bash -% cd docs -% make clean && make html +cd docs +make html ``` -The documentation can be viewed locally by open `./docs/build/index.html` in a browser. +Open the local docs via: + +```sh +open ./docs/build/index.html +``` + +--- + +## 🧭 Design Philosophy + +**terraform-worker** was created from the ground up for production-first use in enterprise CI/CD systems, including: +- Large deployments with complex dependency graphs +- Strict version controls and reproducibility +- Integration with GitHub or API-driven configuration management + +While it excels at scale, it’s also a powerful CLI for individuals who need structure without sacrificing flexibility. + +--- +## πŸ”— License + +Apache License 2.0 β€” see [LICENSE](./LICENSE) for details. From 4b2b4d1a16f6ea5080afb0def300670c763edc91 Mon Sep 17 00:00:00 2001 From: Richard Maynard Date: Mon, 26 May 2025 12:49:59 -0500 Subject: [PATCH 2/5] add license information --- README.md | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bcf10a7..df9a057 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ # terraform-worker +[![License: Apache-2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](./LICENSE) + **terraform-worker** is a command-line tool purpose-built for orchestrating large-scale, modular Terraform deployments. It enables precise control over provider versions, remote state scaffolding, and shared configuration logic β€” ideal for enterprise infrastructure teams managing complex environments across multiple workspaces and deployment stages. @@ -6,6 +8,21 @@ Unlike tools such as `terragrunt`, this is not a drop-in wrapper β€” `terraform- --- +## πŸ“– Table of Contents + +- [πŸš€ Features](#-features) +- [πŸ”„ Example Workflow](#-example-workflow) +- [πŸ“Œ Configuration Notes](#-configuration-notes) +- [🧠 How It Works](#-how-it-works) +- [πŸ§ͺ Development & Testing](#-development--testing) +- [πŸ“€ Releasing](#-releasing) +- [πŸ›  Troubleshooting](#-troubleshooting) +- [πŸͺ Hook Scripts](#-hook-scripts) +- [πŸ”’ Assuming a Role (AWS)](#-assuming-a-role-aws) +- [πŸ“š Documentation](#-documentation) +- [🧭 Design Philosophy](#-design-philosophy) +- [πŸ”— License](#-license) + ## πŸš€ Features - πŸ”§ Modularized state management across nested or sequential Terraform operations @@ -218,6 +235,17 @@ open ./docs/build/index.html While it excels at scale, it’s also a powerful CLI for individuals who need structure without sacrificing flexibility. --- -## πŸ”— License +## πŸ” Legal Summary β€” Apache License 2.0 Apache License 2.0 β€” see [LICENSE](./LICENSE) for details. + +This project is licensed under the Apache License 2.0, which means: + +- βœ… You can use the code freely, including in commercial applications +- βœ… You can modify it, fork it, and redistribute it +- βœ… You **do not** have to open source your own modifications +- βœ… It includes a patent grant to protect you from contributors asserting patent claims +- ❗ You **must** include a copy of the license and provide proper attribution +- ❗ You **must** note significant changes if you modify the code + +This license is designed to encourage broad use while protecting both users and contributors. From 818f09adeb66bbec0f6dd89ba87d7edf97cd8447 Mon Sep 17 00:00:00 2001 From: Richard Maynard Date: Mon, 26 May 2025 15:14:46 -0500 Subject: [PATCH 3/5] Rewrite readme, add basic pipeline actions --- .github/workflows/ci.yml | 40 ++++++++++++++++++++++++++++++++++++++++ Makefile | 16 ++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c1e1e45 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +name: CI + +on: + push: + branches: ['*'] + pull_request: + branches: [master] + +jobs: + lint-format-test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install Poetry + run: curl -sSL https://install.python-poetry.org | python3 - + + - name: Configure Poetry Path + run: echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install Dependencies + run: poetry install --with dev + + - name: Check Code Format (black) + run: poetry run black --check tfworker tests + + - name: Check Imports Sorted (isort) + run: poetry run isort --check-only tfworker tests + + - name: Run Lint (flake8) + run: poetry run flake8 --ignore E501,W503 tfworker tests + + - name: Run Tests + run: poetry run pytest -p no:warnings --disable-socket diff --git a/Makefile b/Makefile index 12ddd81..a171eb2 100644 --- a/Makefile +++ b/Makefile @@ -31,3 +31,19 @@ clean: @rm -rf build dist .eggs terraform_worker.egg-info @find . -name *.pyc -exec rm {} \; @find . -name __pycache__ -type d -exec rmdir {} \; + +triage-export: + @echo "πŸ“₯ Exporting open, untriaged issues for AI review..." + gh issue list --repo ephur/terraform-worker --state open --limit 1000 --json number,title,body,labels | \ + jq '[.[] | select(.labels | all(.name != "triaged"))]' > open_issues.json + @echo "βœ… Issues written to open_issues.json" + +triage-preview: + @echo "πŸ” Previewing untriaged issue titles..." + @if [ ! -f open_issues.json ]; then \ + $(MAKE) triage-export; \ + fi + jq -r '.[] | "\(.number): \(.title)"' open_issues.json + +ready: lint format test + @echo "βœ… All checks passed. You are ready to commit or push." From 5292c62fd7826f8df3280b93564fca89306b7dd0 Mon Sep 17 00:00:00 2001 From: Richard Maynard Date: Mon, 26 May 2025 15:18:14 -0500 Subject: [PATCH 4/5] Update setuptools version --- README.md | 9 ++++++++- poetry.lock | 19 ++++++++++++------- pyproject.toml | 2 +- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index df9a057..85a7ca5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,13 @@ # terraform-worker -[![License: Apache-2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](./LICENSE) +[![License: Apache-2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](./LICENSE) +![CI](https://github.com/ephur/terraform-worker/actions/workflows/ci.yml/badge.svg) +![Coverage](https://img.shields.io/badge/coverage-60%25-yellow) +![Python](https://img.shields.io/badge/python-3.11-blue) +![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg) +![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg) +![Linted: flake8](https://img.shields.io/badge/linted-flake8-green) +![Built with Poetry](https://img.shields.io/badge/built%20with-poetry-blue) **terraform-worker** is a command-line tool purpose-built for orchestrating large-scale, modular Terraform deployments. It enables precise control over provider versions, remote state scaffolding, and shared configuration logic β€” ideal for enterprise infrastructure teams managing complex environments across multiple workspaces and deployment stages. diff --git a/poetry.lock b/poetry.lock index 1fb2ce7..97fcca8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1948,18 +1948,23 @@ files = [ [[package]] name = "setuptools" -version = "70.3.0" +version = "78.1.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "setuptools-70.3.0-py3-none-any.whl", hash = "sha256:fe384da74336c398e0d956d1cae0669bc02eed936cdb1d49b57de1990dc11ffc"}, - {file = "setuptools-70.3.0.tar.gz", hash = "sha256:f171bab1dfbc86b132997f26a119f6056a57950d058587841a0082e8830f9dc5"}, + {file = "setuptools-78.1.1-py3-none-any.whl", hash = "sha256:c3a9c4211ff4c309edb8b8c4f1cbfa7ae324c4ba9f91ff254e3d305b9fd54561"}, + {file = "setuptools-78.1.1.tar.gz", hash = "sha256:fcc17fd9cd898242f6b4adfaca46137a9edef687f43e6f78469692a5e70d851d"}, ] [package.extras] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.8.0)"] +core = ["importlib_metadata (>=6)", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.14.*)", "pytest-mypy"] [[package]] name = "six" @@ -2342,4 +2347,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "abdaa04cd2215b9c2969684444987d76d90af7e7a01489acdbf0e43ea4196666" +content-hash = "62ac9918e041ffb709b6cf4ef040938f826b4d151e83aee0d610dae7569e7bf2" diff --git a/pyproject.toml b/pyproject.toml index 26340a2..62d4ccd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ mergedeep = "^1.3" pydantic = "^2.7" python-hcl2 = "^4.3" pyyaml = "^6.0" -setuptools = "^70.0" +setuptools = "^78.1" pydantic-settings = "^2.3.4" packaging = "^24.1" From c8ab6a09714af147d20d5778c33a4f36409fa332 Mon Sep 17 00:00:00 2001 From: Richard Maynard Date: Mon, 26 May 2025 22:57:13 -0500 Subject: [PATCH 5/5] Address linting and formatting issues --- .isort.cfg | 2 +- tests/authenticators/test_auth_aws.py | 1 + .../test_authenticators_collection.py | 1 + tests/backends/test_backends_base.py | 1 + tests/backends/test_backends_s3.py | 3 +- tests/conftest.py | 1 + tests/copier/test_copier_factory.py | 1 + tests/copier/test_copier_fs.py | 1 + tests/copier/test_copier_git.py | 1 + .../test_definitions_collection.py | 1 + tests/definitions/test_definitions_model.py | 1 + tests/definitions/test_definitions_prepare.py | 1 + tests/handlers/test_snyk_scan.py | 150 +++++++++--------- tests/test_app_state.py | 1 + tests/test_cli_options.py | 1 + tests/util/test_util_cli.py | 1 + tests/util/test_util_hooks.py | 1 + tests/util/test_util_log.py | 1 + tests/util/test_util_system.py | 1 + tests/util/test_util_terraform_helpers.py | 1 + tfworker/app_state.py | 1 + tfworker/authenticators/aws.py | 3 +- tfworker/authenticators/collection.py | 3 +- tfworker/backends/gcs.py | 3 +- tfworker/backends/s3.py | 2 + tfworker/cli.py | 3 +- tfworker/cli_options.py | 3 +- tfworker/commands/base.py | 3 +- tfworker/commands/config.py | 3 +- tfworker/commands/root.py | 1 + tfworker/commands/terraform.py | 2 +- tfworker/definitions/collection.py | 3 +- tfworker/definitions/model.py | 3 +- tfworker/definitions/plan.py | 1 + tfworker/definitions/prepare.py | 1 + tfworker/handlers/__init__.py | 2 +- tfworker/handlers/bitbucket.py | 1 + tfworker/handlers/registry.py | 1 + tfworker/handlers/s3.py | 1 + tfworker/handlers/snyk.py | 15 +- tfworker/handlers/trivy.py | 3 +- tfworker/providers/collection.py | 3 +- tfworker/providers/model.py | 1 + tfworker/types/freezable_basemodel.py | 1 + tfworker/util/cli.py | 3 +- tfworker/util/log.py | 2 +- tfworker/util/terraform.py | 1 + tfworker/util/terraform_helpers.py | 3 +- 48 files changed, 148 insertions(+), 96 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index bbba6d5..11ea026 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,4 +1,4 @@ [settings] -known_third_party = atlassian,boto3,botocore,click,google,hcl2,jinja2,lark,moto,packaging,pydantic,pydantic_core,pydantic_settings,pytest,tfworker,yaml +known_third_party = atlassian,boto3,botocore,click,google,hcl2,jinja2,lark,moto,packaging,pydantic,pydantic_core,pydantic_settings,pytest,yaml profile = black skip = ["*/__init__.py"] diff --git a/tests/authenticators/test_auth_aws.py b/tests/authenticators/test_auth_aws.py index 8695368..ea5449e 100644 --- a/tests/authenticators/test_auth_aws.py +++ b/tests/authenticators/test_auth_aws.py @@ -5,6 +5,7 @@ from botocore.exceptions import NoCredentialsError from moto import mock_aws from pydantic import ValidationError + from tfworker.authenticators import AWSAuthenticator, AWSAuthenticatorConfig from tfworker.authenticators.aws import ( _assume_role_session, diff --git a/tests/authenticators/test_authenticators_collection.py b/tests/authenticators/test_authenticators_collection.py index 82af026..2de07f3 100644 --- a/tests/authenticators/test_authenticators_collection.py +++ b/tests/authenticators/test_authenticators_collection.py @@ -3,6 +3,7 @@ import pytest from pydantic import ValidationError from pydantic_core import InitErrorDetails + from tfworker.authenticators.base import BaseAuthenticator, BaseAuthenticatorConfig from tfworker.authenticators.collection import AuthenticatorsCollection from tfworker.exceptions import FrozenInstanceError, UnknownAuthenticator diff --git a/tests/backends/test_backends_base.py b/tests/backends/test_backends_base.py index 2afe806..fa908fd 100644 --- a/tests/backends/test_backends_base.py +++ b/tests/backends/test_backends_base.py @@ -1,6 +1,7 @@ import json import pytest + from tfworker.backends.base import validate_backend_empty from tfworker.exceptions import BackendError diff --git a/tests/backends/test_backends_s3.py b/tests/backends/test_backends_s3.py index fe99097..80cf257 100644 --- a/tests/backends/test_backends_s3.py +++ b/tests/backends/test_backends_s3.py @@ -3,8 +3,9 @@ import boto3 import botocore import pytest -import tfworker.util.log as log from moto import mock_aws + +import tfworker.util.log as log from tfworker.backends.s3 import S3Backend from tfworker.exceptions import BackendError diff --git a/tests/conftest.py b/tests/conftest.py index c5b32f1..1ba222c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,7 @@ import click import pytest from moto import mock_aws + from tfworker.app_state import AppState from tfworker.authenticators import AuthenticatorsCollection from tfworker.cli_options import CLIOptionsClean, CLIOptionsRoot, CLIOptionsTerraform diff --git a/tests/copier/test_copier_factory.py b/tests/copier/test_copier_factory.py index 682b139..64454aa 100644 --- a/tests/copier/test_copier_factory.py +++ b/tests/copier/test_copier_factory.py @@ -4,6 +4,7 @@ from unittest.mock import patch import pytest + from tfworker.copier.factory import Copier, CopyFactory C_CONFLICTS = ["test.txt", "foo", "test.tf"] diff --git a/tests/copier/test_copier_fs.py b/tests/copier/test_copier_fs.py index 7fd73ac..076f1a2 100644 --- a/tests/copier/test_copier_fs.py +++ b/tests/copier/test_copier_fs.py @@ -2,6 +2,7 @@ import tempfile import pytest + from tfworker.copier import FileSystemCopier diff --git a/tests/copier/test_copier_git.py b/tests/copier/test_copier_git.py index fde1c60..54d5406 100644 --- a/tests/copier/test_copier_git.py +++ b/tests/copier/test_copier_git.py @@ -6,6 +6,7 @@ from unittest import mock import pytest + from tfworker.copier import GitCopier C_CONFLICTS = ["test.txt", "foo", "test.tf"] diff --git a/tests/definitions/test_definitions_collection.py b/tests/definitions/test_definitions_collection.py index 7c1ed85..4d5defb 100644 --- a/tests/definitions/test_definitions_collection.py +++ b/tests/definitions/test_definitions_collection.py @@ -1,4 +1,5 @@ import pytest + from tfworker.definitions import Definition, DefinitionsCollection from tfworker.exceptions import FrozenInstanceError diff --git a/tests/definitions/test_definitions_model.py b/tests/definitions/test_definitions_model.py index 374930b..3aa7ebf 100644 --- a/tests/definitions/test_definitions_model.py +++ b/tests/definitions/test_definitions_model.py @@ -1,4 +1,5 @@ import pytest + from tfworker.definitions.model import Definition, DefinitionRemoteOptions diff --git a/tests/definitions/test_definitions_prepare.py b/tests/definitions/test_definitions_prepare.py index 3fe0ccf..c1f50b0 100644 --- a/tests/definitions/test_definitions_prepare.py +++ b/tests/definitions/test_definitions_prepare.py @@ -1,4 +1,5 @@ import pytest + from tfworker.definitions import Definition, DefinitionsCollection from tfworker.definitions.prepare import DefinitionPrepare from tfworker.exceptions import ReservedFileError, TFWorkerException diff --git a/tests/handlers/test_snyk_scan.py b/tests/handlers/test_snyk_scan.py index d99a4cb..e10c7d1 100644 --- a/tests/handlers/test_snyk_scan.py +++ b/tests/handlers/test_snyk_scan.py @@ -1,12 +1,14 @@ -import pytest import shutil from pathlib import Path -from unittest.mock import Mock, patch, MagicMock +from unittest.mock import MagicMock, Mock, patch + +import pytest + +from tfworker.commands.terraform import TerraformResult from tfworker.definitions import Definition from tfworker.exceptions import HandlerError -from tfworker.commands.terraform import TerraformResult -from tfworker.types import TerraformAction, TerraformStage from tfworker.handlers import SnykConfig, SnykHandler +from tfworker.types import TerraformAction, TerraformStage mock_path_exists = Mock() mock_path_exists_fn = Mock(return_value=True) @@ -16,15 +18,15 @@ mock_path_not_exists_fn = Mock(return_value=False) mock_path_not_exists.exists = mock_path_not_exists_fn -mock_access_allowed = Mock(return_value=True) +mock_access_allowed = Mock(return_value=True) mock_access_denied = Mock(return_value=False) mock_environ = MagicMock() -mock_environ_dict = {'SNYK_TOKEN': "1234567890"} +mock_environ_dict = {"SNYK_TOKEN": "1234567890"} mock_environ.__getitem__.side_effect = mock_environ_dict.__getitem__ mock_environ_no_token = MagicMock() -mock_environ_no_token.__getitem__.side_effect = {'SNYK_TOKEN': None}.__getitem__ +mock_environ_no_token.__getitem__.side_effect = {"SNYK_TOKEN": None}.__getitem__ tmp_dir = f"{Path.cwd()}/.pytest-tmp" snyk_path = f"{tmp_dir}/snyk" @@ -37,19 +39,22 @@ # Copy success/failure scripts in its place to simulate mock_snyk_success_path = f"{scripts_dir}/mock_snyk_success.sh" mock_snyk_failure_path = f"{scripts_dir}/mock_snyk_failure.sh" + + @pytest.fixture(autouse=True) def required_file_system_paths(): - Path.mkdir(Path(tmp_dir), 0o777, exist_ok=True) - Path.mkdir(f"{tmp_dir}/plans", 0o777, parents=False, exist_ok=True) - Path(plan_path).touch(0o777) - Path.mkdir(Path(definition_dir), mode=0o777, parents=True, exist_ok=True) - yield - shutil.rmtree("./.pytest-tmp") + Path.mkdir(Path(tmp_dir), 0o777, exist_ok=True) + Path.mkdir(f"{tmp_dir}/plans", 0o777, parents=False, exist_ok=True) + Path(plan_path).touch(0o777) + Path.mkdir(Path(definition_dir), mode=0o777, parents=True, exist_ok=True) + yield + shutil.rmtree("./.pytest-tmp") + class TestSnykHandler: - @patch('os.path', mock_path_exists) - @patch('os.access', mock_access_allowed) - @patch('os.environ', mock_environ) + @patch("os.path", mock_path_exists) + @patch("os.access", mock_access_allowed) + @patch("os.environ", mock_environ) def test_init(self): config = Mock() config.model_fields = ["path", "my_custom_field", "required"] @@ -60,12 +65,12 @@ def test_init(self): handler = SnykHandler(config) assert handler._path == "/usr/bin/snyk" assert handler._my_custom_field == "abc123" - assert handler._required == True + assert handler._required is True mock_path_exists_fn.assert_called mock_access_allowed.assert_called - @patch('os.path', mock_path_not_exists) - @patch('os.access', mock_access_allowed) + @patch("os.path", mock_path_not_exists) + @patch("os.access", mock_access_allowed) def test_init_path_not_exists(self): config = Mock() config.model_fields = ["path", "required"] @@ -73,14 +78,14 @@ def test_init_path_not_exists(self): config.required = True with pytest.raises(HandlerError): - handler = SnykHandler(config) - assert handler._path == "/usr/bin/snyk" - assert handler._required == True - mock_path_not_exists_fn.assert_called - mock_access_allowed.assert_not_called - - @patch('os.path', mock_path_exists) - @patch('os.access', mock_access_denied) + handler = SnykHandler(config) + assert handler._path == "/usr/bin/snyk" + assert handler._required is True + mock_path_not_exists_fn.assert_called + mock_access_allowed.assert_not_called + + @patch("os.path", mock_path_exists) + @patch("os.access", mock_access_denied) def test_init_path_not_allowed(self): config = Mock() config.model_fields = ["path", "required"] @@ -88,15 +93,15 @@ def test_init_path_not_allowed(self): config.required = True with pytest.raises(HandlerError): - handler = SnykHandler(config) - assert handler._path == "/usr/bin/snyk" - assert handler._required == True - mock_path_exists_fn.assert_called - mock_access_denied.assert_called - - @patch('os.path', mock_path_exists) - @patch('os.access', mock_access_allowed) - @patch('os.environ', mock_environ_no_token) + handler = SnykHandler(config) + assert handler._path == "/usr/bin/snyk" + assert handler._required is True + mock_path_exists_fn.assert_called + mock_access_denied.assert_called + + @patch("os.path", mock_path_exists) + @patch("os.access", mock_access_allowed) + @patch("os.environ", mock_environ_no_token) def test_init_missing_token(self): config = Mock() config.model_fields = ["path", "required"] @@ -104,32 +109,32 @@ def test_init_missing_token(self): config.required = True with pytest.raises(HandlerError): - handler = SnykHandler(config) - assert handler._path == "/usr/bin/snyk" - assert handler._required == True - mock_path_exists_fn.assert_called - mock_access_allowed.assert_called + handler = SnykHandler(config) + assert handler._path == "/usr/bin/snyk" + assert handler._required is True + mock_path_exists_fn.assert_called + mock_access_allowed.assert_called - @patch('os.environ', mock_environ) + @patch("os.environ", mock_environ) def test_execute_pre_plan(self): # Mock out the snyk call with a successful script shutil.copy(Path(mock_snyk_success_path), Path(snyk_path)) - + config = SnykConfig(path=snyk_path, required=True) handler = SnykHandler(config) handler.execute( action=TerraformAction.PLAN, stage=TerraformStage.PRE, deployment="apps/dev", - definition=Definition(name='test_def', path="apps-dev/test-def"), + definition=Definition(name="test_def", path="apps-dev/test-def"), working_dir=working_dir, ) - @patch('os.environ', mock_environ) + @patch("os.environ", mock_environ) def test_execute_pre_plan_fail(self): # Mock out the snyk call with a failure script shutil.copy(Path(mock_snyk_failure_path), Path(snyk_path)) - + config = SnykConfig(path=snyk_path, required=True) with pytest.raises(HandlerError): @@ -137,35 +142,39 @@ def test_execute_pre_plan_fail(self): action=TerraformAction.PLAN, stage=TerraformStage.PRE, deployment="apps/dev", - definition=Definition(name='test_def', path="apps-dev/test-def"), + definition=Definition(name="test_def", path="apps-dev/test-def"), working_dir=working_dir, ) - @patch('os.environ', mock_environ) + @patch("os.environ", mock_environ) def test_execute_post_plan_no_changes(self): # Mock out the snyk call with a failre script # We shouldn't end up calling it, so it should fail if we do shutil.copy(Path(mock_snyk_failure_path), Path(snyk_path)) - - result = TerraformResult(0, "Plan has no changes".encode("utf-8"), "".encode("utf-8")) + + result = TerraformResult( + 0, "Plan has no changes".encode("utf-8"), "".encode("utf-8") + ) config = SnykConfig(path=snyk_path, required=True) handler = SnykHandler(config) handler.execute( action=TerraformAction.PLAN, stage=TerraformStage.POST, deployment="apps/dev", - definition=Definition(name='test_def', path="apps-dev/test-def"), + definition=Definition(name="test_def", path="apps-dev/test-def"), working_dir=working_dir, - result=result + result=result, ) - @patch('os.environ', mock_environ) + @patch("os.environ", mock_environ) def test_execute_post_plan_changes_fail(self): # Mock out the snyk call with a faliure script # We want to ensure it got called, so we'll expect the failure as an exception shutil.copy(Path(mock_snyk_failure_path), Path(snyk_path)) - - result = TerraformResult(2, "Plan has changes".encode("utf-8"), "".encode("utf-8")) + + result = TerraformResult( + 2, "Plan has changes".encode("utf-8"), "".encode("utf-8") + ) config = SnykConfig(path=snyk_path, required=True) handler = SnykHandler(config) @@ -174,24 +183,21 @@ def test_execute_post_plan_changes_fail(self): action=TerraformAction.PLAN, stage=TerraformStage.POST, deployment="apps/dev", - definition=Definition(name='test_def', path="apps-dev/test-def", plan_file=plan_path), + definition=Definition( + name="test_def", path="apps-dev/test-def", plan_file=plan_path + ), working_dir=working_dir, - result=result + result=result, ) - @patch('os.environ', mock_environ) - @patch('os.access', mock_access_allowed) - @patch('os.environ', mock_environ_no_token) + @patch("os.environ", mock_environ) + @patch("os.access", mock_access_allowed) + @patch("os.environ", mock_environ_no_token) def test_verify_snyk_args(self): - # Mock out the snyk call with a successful script - shutil.copy(Path(mock_snyk_success_path), Path(snyk_path)) - config = SnykConfig(path=snyk_path) - handler = SnykHandler(config) - test_def_dir = f"{definition_dir}/my-test-def" - args = handler._build_snyk_args(test_def_dir) - assert args == [ - snyk_path, - "iac", - "test", - test_def_dir - ] \ No newline at end of file + # Mock out the snyk call with a successful script + shutil.copy(Path(mock_snyk_success_path), Path(snyk_path)) + config = SnykConfig(path=snyk_path) + handler = SnykHandler(config) + test_def_dir = f"{definition_dir}/my-test-def" + args = handler._build_snyk_args(test_def_dir) + assert args == [snyk_path, "iac", "test", test_def_dir] diff --git a/tests/test_app_state.py b/tests/test_app_state.py index fa9a867..ce315e9 100644 --- a/tests/test_app_state.py +++ b/tests/test_app_state.py @@ -1,4 +1,5 @@ import pytest + from tfworker.app_state import AppState from tfworker.cli_options import CLIOptionsTerraform from tfworker.exceptions import FrozenInstanceError diff --git a/tests/test_cli_options.py b/tests/test_cli_options.py index 3403b40..dded6be 100644 --- a/tests/test_cli_options.py +++ b/tests/test_cli_options.py @@ -1,4 +1,5 @@ import pytest + from tfworker import cli_options as c diff --git a/tests/util/test_util_cli.py b/tests/util/test_util_cli.py index 94be0ab..5d88c19 100644 --- a/tests/util/test_util_cli.py +++ b/tests/util/test_util_cli.py @@ -4,6 +4,7 @@ import click import pytest from pydantic import BaseModel + from tfworker.util.cli import pydantic_to_click, validate_host diff --git a/tests/util/test_util_hooks.py b/tests/util/test_util_hooks.py index 107f6cf..f664d22 100644 --- a/tests/util/test_util_hooks.py +++ b/tests/util/test_util_hooks.py @@ -1,6 +1,7 @@ from unittest import mock import pytest + import tfworker.util.hooks as hooks from tfworker.exceptions import HookError from tfworker.types.terraform import TerraformAction, TerraformStage diff --git a/tests/util/test_util_log.py b/tests/util/test_util_log.py index f5cddfb..1b33f75 100644 --- a/tests/util/test_util_log.py +++ b/tests/util/test_util_log.py @@ -1,6 +1,7 @@ from unittest.mock import patch import pytest + import tfworker.util.log as log REDACTED_ITEMS = ["aws_secret_access_key", "aws_session_token", "aws_profile"] diff --git a/tests/util/test_util_system.py b/tests/util/test_util_system.py index e1b7931..faae8d9 100644 --- a/tests/util/test_util_system.py +++ b/tests/util/test_util_system.py @@ -1,6 +1,7 @@ from unittest import mock import pytest + from tfworker.util.system import get_platform, pipe_exec, strip_ansi diff --git a/tests/util/test_util_terraform_helpers.py b/tests/util/test_util_terraform_helpers.py index 863e5f3..0ede7c6 100644 --- a/tests/util/test_util_terraform_helpers.py +++ b/tests/util/test_util_terraform_helpers.py @@ -5,6 +5,7 @@ import pytest from packaging.specifiers import SpecifierSet + from tfworker.exceptions import TFWorkerException from tfworker.providers import ProviderGID from tfworker.providers.collection import ProvidersCollection diff --git a/tfworker/app_state.py b/tfworker/app_state.py index 8188384..f2917c7 100644 --- a/tfworker/app_state.py +++ b/tfworker/app_state.py @@ -1,6 +1,7 @@ from pathlib import Path from pydantic import ConfigDict, Field + from tfworker import cli_options from tfworker.authenticators import AuthenticatorsCollection from tfworker.backends import BaseBackend diff --git a/tfworker/authenticators/aws.py b/tfworker/authenticators/aws.py index daa7109..3c307a9 100644 --- a/tfworker/authenticators/aws.py +++ b/tfworker/authenticators/aws.py @@ -2,9 +2,10 @@ from typing import Dict import boto3 -import tfworker.util.log as log from botocore.credentials import Credentials from pydantic import model_validator + +import tfworker.util.log as log from tfworker.exceptions import TFWorkerException from .base import BaseAuthenticator, BaseAuthenticatorConfig diff --git a/tfworker/authenticators/collection.py b/tfworker/authenticators/collection.py index 8ccac7a..9821012 100644 --- a/tfworker/authenticators/collection.py +++ b/tfworker/authenticators/collection.py @@ -2,8 +2,9 @@ import threading from typing import TYPE_CHECKING -import tfworker.util.log as log from pydantic import ValidationError + +import tfworker.util.log as log from tfworker.exceptions import FrozenInstanceError, UnknownAuthenticator from .aws import AWSAuthenticator # noqa diff --git a/tfworker/backends/gcs.py b/tfworker/backends/gcs.py index f1075e9..9795ca3 100644 --- a/tfworker/backends/gcs.py +++ b/tfworker/backends/gcs.py @@ -2,11 +2,12 @@ from typing import TYPE_CHECKING import click -import tfworker.util.log as log from google.api_core import page_iterator from google.auth.exceptions import DefaultCredentialsError from google.cloud import storage from google.cloud.exceptions import Conflict, NotFound + +import tfworker.util.log as log from tfworker.exceptions import BackendError from tfworker.types import JSONType diff --git a/tfworker/backends/s3.py b/tfworker/backends/s3.py index 278be51..d6b7190 100644 --- a/tfworker/backends/s3.py +++ b/tfworker/backends/s3.py @@ -8,6 +8,7 @@ import botocore.errorfactory import botocore.paginate import click + import tfworker.util.log as log from tfworker.exceptions import BackendError from tfworker.types import JSONType @@ -16,6 +17,7 @@ if TYPE_CHECKING: import boto3 # pragma: no cover # noqa + from tfworker.app_state import AppState # pragma: no cover # noqa from tfworker.authenticators import ( # pragma: no cover # noqa AuthenticatorsCollection, diff --git a/tfworker/cli.py b/tfworker/cli.py index 0038a54..5ca22ae 100644 --- a/tfworker/cli.py +++ b/tfworker/cli.py @@ -1,7 +1,8 @@ #!/usr/bin/env python import click -import tfworker.util.log as log from pydantic import ValidationError + +import tfworker.util.log as log from tfworker.app_state import AppState from tfworker.cli_options import CLIOptionsClean, CLIOptionsRoot, CLIOptionsTerraform from tfworker.commands.clean import CleanCommand diff --git a/tfworker/cli_options.py b/tfworker/cli_options.py index 2d8d9ba..048b997 100644 --- a/tfworker/cli_options.py +++ b/tfworker/cli_options.py @@ -3,7 +3,6 @@ from typing import List, Optional, Union import click -import tfworker.util.log as log from pydantic import ( ConfigDict, Field, @@ -12,6 +11,8 @@ model_validator, ) from pydantic_core import InitErrorDetails + +import tfworker.util.log as log from tfworker import constants as const from tfworker.backends import Backends from tfworker.types import FreezableBaseModel diff --git a/tfworker/commands/base.py b/tfworker/commands/base.py index bcadc58..84be81d 100644 --- a/tfworker/commands/base.py +++ b/tfworker/commands/base.py @@ -1,9 +1,10 @@ from typing import TYPE_CHECKING, Any, Dict, Union import click +from pydantic import BaseModel, ValidationError + import tfworker.commands.config as c import tfworker.util.log as log -from pydantic import BaseModel, ValidationError from tfworker.exceptions import BackendError, HandlerError, TFWorkerException from tfworker.util.cli import handle_config_error diff --git a/tfworker/commands/config.py b/tfworker/commands/config.py index 615fca4..e0182c4 100644 --- a/tfworker/commands/config.py +++ b/tfworker/commands/config.py @@ -8,10 +8,11 @@ import click import hcl2 import jinja2 -import tfworker.util.log as log import yaml from jinja2.runtime import StrictUndefined from pydantic import BaseModel, ValidationError + +import tfworker.util.log as log from tfworker.app_state import AppState from tfworker.types.config_file import ConfigFile from tfworker.util.cli import handle_config_error diff --git a/tfworker/commands/root.py b/tfworker/commands/root.py index 9d67641..508fdc2 100644 --- a/tfworker/commands/root.py +++ b/tfworker/commands/root.py @@ -3,6 +3,7 @@ from typing import Any, Dict import click + import tfworker.util.log as log from tfworker.cli_options import CLIOptionsRoot diff --git a/tfworker/commands/terraform.py b/tfworker/commands/terraform.py index 053ad51..ce47e47 100644 --- a/tfworker/commands/terraform.py +++ b/tfworker/commands/terraform.py @@ -296,7 +296,7 @@ def _generate_plan_output_json(self, name) -> None: f"{self.app_state.terraform_options.terraform_bin} show -json {definition.plan_file}", cwd=working_dir, env=self.terraform_config.env, - stream_output=False + stream_output=False, ) ) diff --git a/tfworker/definitions/collection.py b/tfworker/definitions/collection.py index f468e1b..9ab425a 100644 --- a/tfworker/definitions/collection.py +++ b/tfworker/definitions/collection.py @@ -2,9 +2,10 @@ from collections.abc import Mapping from typing import Dict, List -import tfworker.util.log as log from pydantic import GetCoreSchemaHandler, ValidationError from pydantic_core import CoreSchema, core_schema + +import tfworker.util.log as log from tfworker.exceptions import FrozenInstanceError from tfworker.util.cli import handle_config_error diff --git a/tfworker/definitions/model.py b/tfworker/definitions/model.py index 8981bf5..e4fee15 100644 --- a/tfworker/definitions/model.py +++ b/tfworker/definitions/model.py @@ -1,9 +1,10 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Union -import tfworker.util.log as log from pydantic import BaseModel, ConfigDict, Field +import tfworker.util.log as log + class DefinitionRemoteOptions(BaseModel): """ diff --git a/tfworker/definitions/plan.py b/tfworker/definitions/plan.py index 99cf84c..fde7eef 100644 --- a/tfworker/definitions/plan.py +++ b/tfworker/definitions/plan.py @@ -5,6 +5,7 @@ if TYPE_CHECKING: from click import Context # pragma: no cover # noqa: F401 + from tfworker.app_state import AppState # pragma: no cover # noqa: F401 from tfworker.definitions.model import Definition # pragma: no cover # noqa: F401 diff --git a/tfworker/definitions/prepare.py b/tfworker/definitions/prepare.py index d76e7e6..3deb3ea 100644 --- a/tfworker/definitions/prepare.py +++ b/tfworker/definitions/prepare.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Dict, Union import jinja2 + import tfworker.util.log as log from tfworker.constants import ( RESERVED_FILES, diff --git a/tfworker/handlers/__init__.py b/tfworker/handlers/__init__.py index acb77f2..22d1aef 100644 --- a/tfworker/handlers/__init__.py +++ b/tfworker/handlers/__init__.py @@ -2,5 +2,5 @@ from .bitbucket import BitbucketConfig, BitbucketHandler # pragma: no cover # noqa from .collection import HandlersCollection # pragma: no cover # noqa from .s3 import S3Handler # pragma: no cover # noqa +from .snyk import SnykConfig, SnykHandler # pragma: no cover # noqa from .trivy import TrivyConfig, TrivyHandler # pragma: no cover # noqa -from .snyk import SnykConfig, SnykHandler # pragma: no cover # noqa diff --git a/tfworker/handlers/bitbucket.py b/tfworker/handlers/bitbucket.py index 8b7af09..a339233 100644 --- a/tfworker/handlers/bitbucket.py +++ b/tfworker/handlers/bitbucket.py @@ -3,6 +3,7 @@ from atlassian.bitbucket import Cloud from pydantic import BaseModel, Field from pydantic_settings import SettingsConfigDict + from tfworker.exceptions import HandlerError from tfworker.types.terraform import TerraformAction, TerraformStage diff --git a/tfworker/handlers/registry.py b/tfworker/handlers/registry.py index 3cf9885..f8174c9 100644 --- a/tfworker/handlers/registry.py +++ b/tfworker/handlers/registry.py @@ -1,6 +1,7 @@ from typing import Callable, Dict, List from pydantic import BaseModel + from tfworker.exceptions import HandlerError from .base import BaseHandler diff --git a/tfworker/handlers/s3.py b/tfworker/handlers/s3.py index 5aaebfb..27dda70 100644 --- a/tfworker/handlers/s3.py +++ b/tfworker/handlers/s3.py @@ -7,6 +7,7 @@ import boto3 import botocore import click + import tfworker.util.log as log from tfworker.backends import Backends from tfworker.exceptions import HandlerError diff --git a/tfworker/handlers/snyk.py b/tfworker/handlers/snyk.py index d05d6ff..2c4791c 100644 --- a/tfworker/handlers/snyk.py +++ b/tfworker/handlers/snyk.py @@ -1,6 +1,7 @@ import os from pathlib import Path from typing import TYPE_CHECKING, Union + from pydantic import BaseModel import tfworker.util.log as log @@ -15,6 +16,7 @@ from tfworker.commands.terraform import TerraformResult from tfworker.definitions.model import Definition + class SnykConfig(BaseModel): args: dict = {} cache_dir: str = "/tmp/snyk_cache" @@ -31,6 +33,7 @@ class SnykConfig(BaseModel): stream_output: bool = True exempt_definitions: list = [] + @HandlerRegistry.register("snyk") class SnykHandler(BaseHandler): """ @@ -80,19 +83,19 @@ def execute( None """ # pre plan; snyk scan the generated definition source - if (action == TerraformAction.PLAN and stage == TerraformStage.PRE): + if action == TerraformAction.PLAN and stage == TerraformStage.PRE: definition_path = definition.get_target_path(working_dir=working_dir) if definition_path is None: raise HandlerError( "definition_path is not provided, can't scan", terminate=self._required, ) - + definition_name = definition.name if self._skip_definition or definition_name in self._exempt_definitions: log.info(f"Skipping snyk scan of definition: {definition_name}") return None - + log.info(f"scanning definition with snyk: {definition_path}") self._scan(definition, Path(definition_path)) @@ -111,12 +114,12 @@ def execute( if self._skip_planfile: log.info(f"Skipping snyk scan of planfile: {planfile}") return None - + definition_name = definition.name if self._skip_definition or definition_name in self._exempt_definitions: log.info(f"Skipping snyk scan of definition: {definition_name}") return None - + jsonfile = Path(planfile).with_suffix(".tfplan.json") log.info(f"scanning planfile with snyk: {jsonfile}") self._scan(definition, Path(jsonfile)) @@ -154,7 +157,7 @@ def _build_snyk_args(self, target_path): snyk_args.append("test") snyk_args.append(str(Path(target_path).resolve())) return snyk_args - + def _handle_results(self, exit_code, stdout, stderr, definition): """Handle the results of the snyk scan diff --git a/tfworker/handlers/trivy.py b/tfworker/handlers/trivy.py index 101a169..171a599 100644 --- a/tfworker/handlers/trivy.py +++ b/tfworker/handlers/trivy.py @@ -2,8 +2,9 @@ from pathlib import Path from typing import TYPE_CHECKING, Union -import tfworker.util.log as log from pydantic import BaseModel + +import tfworker.util.log as log from tfworker.exceptions import HandlerError from tfworker.types.terraform import TerraformAction, TerraformStage diff --git a/tfworker/providers/collection.py b/tfworker/providers/collection.py index 0d579ca..d78de9b 100644 --- a/tfworker/providers/collection.py +++ b/tfworker/providers/collection.py @@ -3,9 +3,10 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Dict, List -import tfworker.util.log as log from pydantic import GetCoreSchemaHandler, ValidationError from pydantic_core import CoreSchema, core_schema + +import tfworker.util.log as log from tfworker.exceptions import FrozenInstanceError if TYPE_CHECKING: diff --git a/tfworker/providers/model.py b/tfworker/providers/model.py index d2288ac..c191d34 100644 --- a/tfworker/providers/model.py +++ b/tfworker/providers/model.py @@ -1,6 +1,7 @@ from typing import Any, Dict, Optional from pydantic import BaseModel, ConfigDict, model_validator + from tfworker.constants import ( TF_PROVIDER_DEFAULT_HOSTNAME, TF_PROVIDER_DEFAULT_NAMESPACE, diff --git a/tfworker/types/freezable_basemodel.py b/tfworker/types/freezable_basemodel.py index ca6343f..cca1f06 100644 --- a/tfworker/types/freezable_basemodel.py +++ b/tfworker/types/freezable_basemodel.py @@ -1,4 +1,5 @@ from pydantic import BaseModel, PrivateAttr + from tfworker.exceptions import FrozenInstanceError diff --git a/tfworker/util/cli.py b/tfworker/util/cli.py index 9cb42de..9aa5df2 100644 --- a/tfworker/util/cli.py +++ b/tfworker/util/cli.py @@ -2,9 +2,10 @@ from enum import Enum import click -import tfworker.util.log as log from pydantic import BaseModel, ValidationError from pydantic.fields import PydanticUndefined + +import tfworker.util.log as log from tfworker.util.system import get_platform diff --git a/tfworker/util/log.py b/tfworker/util/log.py index 43b12de..bacc4a0 100644 --- a/tfworker/util/log.py +++ b/tfworker/util/log.py @@ -4,6 +4,7 @@ from typing import Any, Dict, List, Union from click import secho + from tfworker.constants import REDACTED_ITEMS @@ -29,7 +30,6 @@ def log( Args: msg () """ - global log_level level_colors = { log_level.TRACE: "cyan", log_level.DEBUG: "blue", diff --git a/tfworker/util/terraform.py b/tfworker/util/terraform.py index cbe916d..1dfe219 100644 --- a/tfworker/util/terraform.py +++ b/tfworker/util/terraform.py @@ -6,6 +6,7 @@ from typing import Dict, List, Union import click + import tfworker.util.log as log import tfworker.util.terraform_helpers as tfhelpers from tfworker.constants import ( diff --git a/tfworker/util/terraform_helpers.py b/tfworker/util/terraform_helpers.py index 628a9d5..da08545 100644 --- a/tfworker/util/terraform_helpers.py +++ b/tfworker/util/terraform_helpers.py @@ -5,9 +5,10 @@ from typing import TYPE_CHECKING, Dict, List import hcl2 -import tfworker.util.log as log from lark.exceptions import UnexpectedToken from packaging.specifiers import InvalidSpecifier, SpecifierSet + +import tfworker.util.log as log from tfworker.exceptions import TFWorkerException from tfworker.util.system import get_platform