diff --git a/.gitignore b/.gitignore index d8dfc33..1fe2631 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ registry.terraform.io/ coverage.xml reports/* plans* + +# go tool related build files +tfworker-hcl2json diff --git a/.isort.cfg b/.isort.cfg index ee90e80..884e4b2 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,4 +1,4 @@ [settings] -known_third_party = atlassian,boto3,botocore,click,google,hcl2,jinja2,lark,mergedeep,moto,openai,packaging,pydantic,pydantic_core,pydantic_settings,pytest,yaml +known_third_party = atlassian,boto3,botocore,click,google,jinja2,mergedeep,moto,openai,packaging,pydantic,pydantic_core,pydantic_settings,pytest,yaml profile = black skip = ["*/__init__.py"] diff --git a/Makefile b/Makefile index da7d618..652745c 100644 --- a/Makefile +++ b/Makefile @@ -14,11 +14,11 @@ format: init-dev @poetry run seed-isort-config || echo "known_third_party setting changed. Please commit pyproject.toml" poetry run isort tfworker tests -test: init-dev +test: init-dev go-build go-test poetry run pytest -p no:warnings --disable-socket poetry run coverage report --fail-under=60 -m --skip-empty -ci-test: init-dev +ci-test: init-dev go-test poetry run pytest --disable-socket --junitxml=reports/junit.xml poetry run coverage xml -o reports/coverage.xml @@ -37,3 +37,36 @@ clean: @rm -rf build dist .eggs terraform_worker.egg-info @find . -name *.pyc -exec rm {} \; @find . -name __pycache__ -type d -exec rmdir {} \; + @rm -f tfworker-hcl2json + +# --- Go integration --- + +.PHONY: go-build go-test + +go-build: + @if which go >/dev/null 2>&1; then \ + if [ ! -f tools/hcl2json/go.sum ]; then \ + echo "[go-build] Missing tools/hcl2json/go.sum"; \ + echo " Run: (cd tools/hcl2json && go mod tidy) and commit go.sum"; \ + echo " Skipping Go build"; \ + else \ + echo "Building Go HCL helper..."; \ + (cd tools/hcl2json && go build -o ../../tfworker-hcl2json); \ + fi; \ + else \ + echo "Go not installed, skipping go-build"; \ + fi + +go-test: + @if which go >/dev/null 2>&1; then \ + if [ ! -f tools/hcl2json/go.sum ]; then \ + echo "[go-test] Missing tools/hcl2json/go.sum"; \ + echo " Run: (cd tools/hcl2json && go mod tidy) and commit go.sum"; \ + echo " Skipping Go tests"; \ + else \ + echo "Running Go tests..."; \ + (cd tools/hcl2json && go test ./...); \ + fi; \ + else \ + echo "Go not installed, skipping go-test"; \ + fi diff --git a/README.md b/README.md index c0ceb35..5a770be 100644 --- a/README.md +++ b/README.md @@ -221,4 +221,35 @@ To build HTML documentation: % make clean && make html ``` +### Optional: Native HCL (Go) Parser + +For improved compatibility with HCL2/Terraform syntax, tfworker can use a small Go helper that leverages HashiCorp's native HCL parser. + +- Build the helper (requires Go 1.20+): + - `go build -o tfworker-hcl2json ./tools/hcl2json` + - Put `tfworker-hcl2json` on your PATH or set `TFWORKER_HCL_BIN` to its full path. + +- Engine selection: + - Default (auto): If the helper is found via `TFWORKER_HCL_BIN` or your `PATH`, tfworker uses the Go parser; otherwise it uses `python-hcl2`. + - Forced: Set `TFWORKER_HCL_ENGINE=go` or `TFWORKER_HCL_ENGINE=python` to force a specific engine. If set to `go` but the helper is missing, tfworker errors out. + +- Batch parsing: tfworker batches `.tf` files when discovering required providers and sends them to the Go helper in one process (`--multi`), improving performance on large repos. Python fallback parses files individually. + +### Go Development + +For the small Go helper in `tools/hcl2json/`: + +- Commit module files: Always commit both `go.mod` and `go.sum`. +- First-time setup: If `go.sum` is missing, run: + - `(cd tools/hcl2json && go mod tidy)` + - Commit the generated `go.sum`. +- Make targets: + - `make go-build` builds the helper into `./tfworker-hcl2json`. + - `make go-test` runs `go test` under `tools/hcl2json`. + - If Go isn’t installed or `go.sum` is missing, the Makefile prints guidance and skips Go steps. +- One system Go: Use a single up-to-date Go (1.22+) on your machine. +- Optional vendoring (air-gapped builds): + - `cd tools/hcl2json && go mod vendor` + - Build with `-mod=vendor` if needed; otherwise vendoring is not required. + The documentation can be viewed locally by open `./docs/build/index.html` in a browser. diff --git a/tests/definitions/test_definitions_prepare_extra.py b/tests/definitions/test_definitions_prepare_extra.py index 2c1c7b9..ddc2f49 100644 --- a/tests/definitions/test_definitions_prepare_extra.py +++ b/tests/definitions/test_definitions_prepare_extra.py @@ -137,12 +137,14 @@ def test_create_local_vars(mocker, def_prepare, definition): def test_create_worker_tf_calls_helpers(mocker, def_prepare): m1 = mocker.patch.object(def_prepare, "_get_remotes", return_value=["r"]) - m2 = mocker.patch.object(def_prepare, "_get_provider_content", return_value="PCONT") - m3 = mocker.patch.object(def_prepare, "_write_worker_tf") + m2 = mocker.patch.object(def_prepare, "_get_used_providers", return_value=["p"]) + m3 = mocker.patch.object(def_prepare, "_get_provider_content", return_value="PCONT") + m4 = mocker.patch.object(def_prepare, "_write_worker_tf") def_prepare.create_worker_tf("def1") m1.assert_called_once_with("def1") m2.assert_called_once_with("def1") - m3.assert_called_once_with("def1", ["r"], "PCONT") + m3.assert_called_once_with("def1", ["p"]) + m4.assert_called_once_with("def1", ["r"], "PCONT", ["p"]) def test_create_terraform_vars(mocker, def_prepare, definition): @@ -222,14 +224,21 @@ def test_download_modules_failure(mocker, def_prepare, definition): def_prepare.download_modules("def1") -def test_get_provider_content(def_prepare, mocker, definition): - mocker.patch.object(Definition, "get_used_providers", return_value=None) +def test_get_provider_content(def_prepare): def_prepare._app_state.providers.required_hcl.return_value = "REQ" - assert def_prepare._get_provider_content("def1") == "REQ" + assert def_prepare._get_provider_content("def1", None) == "REQ" def_prepare._app_state.providers.required_hcl.assert_called_once_with(None) - mocker.patch.object(Definition, "get_used_providers", return_value=["p"]) - assert def_prepare._get_provider_content("def1") == "" + assert def_prepare._get_provider_content("def1", ["p"]) == "" + + +def test_get_used_providers_caches(def_prepare, mocker): + mock_method = mocker.patch.object( + Definition, "get_used_providers", side_effect=[["p"], ["other"]] + ) + assert def_prepare._get_used_providers("def1") == ["p"] + assert def_prepare._get_used_providers("def1") == ["p"] + assert mock_method.call_count == 1 def test_get_remotes(def_prepare, mocker, definition): @@ -271,8 +280,7 @@ def test_write_worker_tf(def_prepare, definition, mocker): def_prepare._app_state.providers.provider_hcl.return_value = "PROV" def_prepare._app_state.backend.hcl.return_value = "BACKEND" def_prepare._app_state.backend.data_hcl.return_value = "DATA" - mocker.patch.object(Definition, "get_used_providers", return_value=["p1"]) - def_prepare._write_worker_tf("def1", ["r"], "REQ") + def_prepare._write_worker_tf("def1", ["r"], "REQ", ["p1"]) tf_path = ( Path(definition.get_target_path(def_prepare._app_state.working_dir)) / WORKER_TF_FILENAME diff --git a/tests/util/test_hcl_engine_parity.py b/tests/util/test_hcl_engine_parity.py new file mode 100644 index 0000000..6b263c5 --- /dev/null +++ b/tests/util/test_hcl_engine_parity.py @@ -0,0 +1,49 @@ +import os +import shutil +from pathlib import Path + +import pytest + +from tfworker.util.terraform_helpers import _find_required_providers + + +def _helper_on_path() -> str | None: + # Respect explicit env if provided + explicit = os.getenv("TFWORKER_HCL_BIN") + if explicit and Path(explicit).exists(): + return explicit + # Check PATH + found = shutil.which("tfworker-hcl2json") + if found: + return found + # Check repo root build location (make go-build outputs here) + repo_root = Path(__file__).parents[2] + candidate = repo_root / "tfworker-hcl2json" + if candidate.exists(): + return str(candidate) + return None + + +@pytest.mark.integration +def test_required_providers_engine_parity(monkeypatch): + """ + Ensure python and go engines return identical provider requirements + for a known fixture directory. + """ + fixtures_dir = Path(__file__).parents[1] / "fixtures" / "definitions" / "test_a" + + # Baseline using Python engine + monkeypatch.setenv("TFWORKER_HCL_ENGINE", "python") + providers_py = _find_required_providers(str(fixtures_dir)) + + # If the Go helper isn't available, skip parity check gracefully + helper = _helper_on_path() + if not helper: + pytest.skip("Go HCL helper not found; skipping engine parity test") + + # Compare with Go engine + monkeypatch.setenv("TFWORKER_HCL_ENGINE", "go") + monkeypatch.setenv("TFWORKER_HCL_BIN", helper) + providers_go = _find_required_providers(str(fixtures_dir)) + + assert providers_go == providers_py diff --git a/tests/util/test_hcl_parser.py b/tests/util/test_hcl_parser.py new file mode 100644 index 0000000..b32ed23 --- /dev/null +++ b/tests/util/test_hcl_parser.py @@ -0,0 +1,68 @@ +import json +import stat +from pathlib import Path + +import pytest + +from tfworker.util import hcl_parser + + +class TestHCLParser: + def test_parse_string_python_engine(self, monkeypatch): + # Force python engine + monkeypatch.setenv("TFWORKER_HCL_ENGINE", "python") + hcl = """ + terraform { + worker_options { x = 1 } + } + """ + d = hcl_parser.parse_string(hcl) + assert "terraform" in d + + def test_parse_with_go_binary(self, tmp_path: Path, monkeypatch): + # Create a fake go helper that just prints a known JSON + payload = {"terraform": [{"worker_options": [{"x": 1}]}]} + fake = tmp_path / "tfworker-hcl2json" + fake.write_text("#!/bin/sh\n" + f"echo '{json.dumps(payload)}'\n") + fake.chmod(fake.stat().st_mode | stat.S_IEXEC) + + monkeypatch.setenv("TFWORKER_HCL_ENGINE", "go") + monkeypatch.setenv("TFWORKER_HCL_BIN", str(fake)) + + d = hcl_parser.parse_string("terraform {}") + assert d == payload + + def test_go_binary_error_surface(self, tmp_path: Path, monkeypatch): + # Fake helper exits non-zero + fake = tmp_path / "tfworker-hcl2json" + fake.write_text("#!/bin/sh\nexit 3\n") + fake.chmod(fake.stat().st_mode | stat.S_IEXEC) + + monkeypatch.setenv("TFWORKER_HCL_ENGINE", "go") + monkeypatch.setenv("TFWORKER_HCL_BIN", str(fake)) + + with pytest.raises(ValueError): + hcl_parser.parse_string("terraform {}") + + def test_parse_files_multi_go_and_errors(self, tmp_path: Path, monkeypatch): + # Create two files: one valid, one invalid + good = tmp_path / "good.hcl" + bad = tmp_path / "bad.hcl" + good.write_text("terraform { required_providers {} }") + bad.write_text("terraform { izgreat! }") + + # Fake helper: returns ok for good and error for bad + payload = { + "ok": {str(good): {"terraform": [{"required_providers": []}]}}, + "errors": {str(bad): "parse error"}, + } + fake = tmp_path / "tfworker-hcl2json" + fake.write_text("#!/bin/sh\n" + f"echo '{json.dumps(payload)}'\n") + fake.chmod(fake.stat().st_mode | stat.S_IEXEC) + + monkeypatch.setenv("TFWORKER_HCL_ENGINE", "go") + monkeypatch.setenv("TFWORKER_HCL_BIN", str(fake)) + + ok, errors = hcl_parser.parse_files([str(good), str(bad)]) + assert str(good) in ok + assert str(bad) in errors diff --git a/tfworker/commands/config.py b/tfworker/commands/config.py index c532464..357495c 100644 --- a/tfworker/commands/config.py +++ b/tfworker/commands/config.py @@ -6,7 +6,6 @@ from typing import Any, Dict, List, Type, Union import click -import hcl2 import jinja2 import yaml from jinja2.runtime import StrictUndefined @@ -17,6 +16,7 @@ from tfworker.app_state import AppState from tfworker.custom_types.config_file import ConfigFile from tfworker.util.cli import handle_config_error +from tfworker.util.hcl_parser import parse_string as parse_hcl_string from .. import cli_options @@ -39,7 +39,7 @@ def load_config( log.safe_trace(f"rendered config: {rendered}") if cf.endswith(".hcl"): - loaded: Dict[str, Any] = hcl2.loads(rendered)["terraform"] + loaded: Dict[str, Any] = parse_hcl_string(rendered)["terraform"] else: loaded = yaml.safe_load(rendered)["terraform"] diff --git a/tfworker/definitions/prepare.py b/tfworker/definitions/prepare.py index 2f7ea55..3f98fde 100644 --- a/tfworker/definitions/prepare.py +++ b/tfworker/definitions/prepare.py @@ -1,6 +1,6 @@ import json from os import environ -from typing import TYPE_CHECKING, Dict, Union +from typing import TYPE_CHECKING, Dict, List, Union import jinja2 @@ -38,6 +38,7 @@ class DefinitionPrepare: def __init__(self, app_state: "AppState"): self._app_state: "AppState" = app_state + self._used_providers_cache: Dict[tuple[str, str], Union[List[str], None]] = {} def copy_files(self, name: str) -> None: """ @@ -102,8 +103,9 @@ def create_worker_tf(self, name: str) -> None: """Create remote data sources, and required providers""" log.trace(f"creating remote data sources for definition {name}") remotes = self._get_remotes(name) - provider_content = self._get_provider_content(name) - self._write_worker_tf(name, remotes, provider_content) + provider_names = self._get_used_providers(name) + provider_content = self._get_provider_content(name, provider_names) + self._write_worker_tf(name, remotes, provider_content, provider_names) def create_terraform_vars(self, name: str) -> None: """Create the variable definitions""" @@ -133,9 +135,7 @@ def create_terraform_lockfile(self, name: str) -> None: log.trace(f"creating terraform lockfile for definition {name}") result = generate_terraform_lockfile( providers=self._app_state.providers, - included_providers=definition.get_used_providers( - self._app_state.working_dir - ), + included_providers=self._get_used_providers(name), cache_dir=self._app_state.terraform_options.provider_cache, ) @@ -167,11 +167,10 @@ def download_modules(self, name: str, stream_output: bool = True) -> None: f"could not download modules for definition {name}: {strip_ansi(result.stderr.decode())}" ) - def _get_provider_content(self, name: str) -> str: + def _get_provider_content( + self, _name: str, provider_names: Union[List[str], None] + ) -> str: """Get the provider content""" - definition = self._app_state.definitions[name] - provider_names = definition.get_used_providers(self._app_state.working_dir) - if provider_names is not None: return "" return self._app_state.providers.required_hcl(provider_names) @@ -195,7 +194,13 @@ def _get_remotes(self, name: str) -> list: log.trace(f"using remotes {remotes} for definition {name}") return remotes - def _write_worker_tf(self, name: str, remotes: list, provider_content: str) -> None: + def _write_worker_tf( + self, + name: str, + remotes: list, + provider_content: str, + provider_names: Union[List[str], None], + ) -> None: """Write the worker.tf file""" definition = self._app_state.definitions[name] @@ -205,7 +210,7 @@ def _write_worker_tf(self, name: str, remotes: list, provider_content: str) -> N ) as tffile: # Write out the provider configurations for each provider tffile.write( - f"{self._app_state.providers.provider_hcl(includes=definition.get_used_providers(self._app_state.working_dir))}\n\n" + f"{self._app_state.providers.provider_hcl(includes=provider_names)}\n\n" ) tffile.write( TERRAFORM_TPL.format( @@ -217,6 +222,16 @@ def _write_worker_tf(self, name: str, remotes: list, provider_content: str) -> N ) tffile.write(self._app_state.backend.data_hcl(remotes)) + def _get_used_providers(self, name: str) -> Union[List[str], None]: + """Return cached list of providers used by a definition""" + cache_key = (name, self._app_state.working_dir) + if cache_key not in self._used_providers_cache: + definition = self._app_state.definitions[name] + self._used_providers_cache[cache_key] = definition.get_used_providers( + self._app_state.working_dir + ) + return self._used_providers_cache[cache_key] + def _get_template_vars(self, name: str) -> Dict[str, str]: """ Prepares the vars for rendering in a jinja template diff --git a/tfworker/util/hcl_parser.py b/tfworker/util/hcl_parser.py new file mode 100644 index 0000000..06b1871 --- /dev/null +++ b/tfworker/util/hcl_parser.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from typing import Any, Dict, List, Tuple + + +def _engine() -> str: + """ + Determine which HCL parsing engine to use. + + Selection logic: + - If TFWORKER_HCL_ENGINE is set to 'go' or 'python', force that engine. + - If 'go' is forced but the helper binary is missing, raise ValueError. + - If TFWORKER_HCL_ENGINE is unset or set to 'auto', auto-detect: + - Use 'go' if the helper binary is available (via TFWORKER_HCL_BIN or PATH), + otherwise use 'python'. + """ + prefer = os.getenv("TFWORKER_HCL_ENGINE", "").strip().lower() + + if prefer in ("go", "python"): + if prefer == "go": + if _go_binary_path() is None: + raise ValueError( + "TFWORKER_HCL_ENGINE=go but Go HCL helper binary not found; " + "ensure it's on PATH or set TFWORKER_HCL_BIN" + ) + return prefer + + # Auto-detect ('auto' or unset) + if _go_binary_path() is not None: + return "go" + return "python" + + +def _go_binary_path() -> str | None: + """ + Resolve the path to the Go-backed HCL helper binary. + Looks for TFWORKER_HCL_BIN or a binary named 'tfworker-hcl2json' on PATH. + """ + explicit = os.getenv("TFWORKER_HCL_BIN") + if explicit: + return explicit + return shutil.which("tfworker-hcl2json") + + +def parse_string(rendered: str) -> Dict[str, Any]: + """Parse HCL from a string into a Python dict. + + Attempts to use the Go-backed parser if configured; otherwise falls back to + python-hcl2 for broad compatibility. + """ + if _engine() == "go": + payload = _invoke_go(["--stdin"], stdin=rendered) + if not isinstance(payload, dict): + raise ValueError("unexpected response from Go HCL helper for string input") + return payload + + import hcl2 # type: ignore + + return hcl2.loads(rendered) + + +def parse_file(path: str) -> Dict[str, Any]: + """Parse HCL from a file into a Python dict.""" + if _engine() == "go": + payload = _invoke_go([path]) + if not isinstance(payload, dict): + raise ValueError("unexpected response from Go HCL helper for file input") + return payload + + import hcl2 # type: ignore + + with open(path, "r") as f: + return hcl2.load(f) + + +def parse_files(paths: List[str]) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, str]]: + """Parse multiple HCL files. + + Returns a tuple of (ok, errors): + - ok: mapping of path -> parsed dict + - errors: mapping of path -> error string + + Uses Go helper in multi-file mode when available; otherwise + falls back to parsing each file with python-hcl2. + """ + engine = _engine() + if engine == "go": + if len(paths) == 1: + payload = _invoke_go([paths[0]]) + if not isinstance(payload, dict): + raise ValueError( + "unexpected response from Go HCL helper for file input" + ) + return {paths[0]: payload}, {} + payload = _invoke_go(["--multi", *paths]) + if not isinstance(payload, dict): + raise ValueError("invalid multi-file response structure from Go HCL helper") + ok = payload.get("ok", {}) or {} + errors = payload.get("errors", {}) or {} + if not isinstance(ok, dict) or not isinstance(errors, dict): + raise ValueError("invalid multi-file response structure from Go HCL helper") + return ok, errors + + import hcl2 # type: ignore + + ok: Dict[str, Dict[str, Any]] = {} + errors: Dict[str, str] = {} + for p in paths: + try: + with open(p, "r") as f: + ok[p] = hcl2.load(f) + except Exception as e: # mirror behavior: capture errors and continue + errors[p] = str(e) + return ok, errors + + +def _invoke_go(args: List[str], stdin: str | None = None) -> Any: + """Invoke the Go helper with arbitrary args and optional stdin; return parsed JSON.""" + bin_path = _go_binary_path() + if not bin_path: + raise ValueError("Go HCL helper binary not found on PATH") + + cmd = [bin_path] + list(args) + try: + proc = subprocess.run( + cmd, + input=(stdin.encode() if stdin is not None else None), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + except OSError as e: + raise ValueError(f"failed to run Go HCL helper: {e}") from e + + if proc.returncode != 0: + raise ValueError( + f"Go HCL helper failed with code {proc.returncode}: {proc.stderr.decode().strip()}" + ) + + try: + payload = json.loads(proc.stdout.decode()) + except json.JSONDecodeError as e: + raise ValueError(f"invalid JSON from Go HCL helper: {e}") from e + + return payload diff --git a/tfworker/util/terraform_helpers.py b/tfworker/util/terraform_helpers.py index 5034ee1..0bf94e8 100644 --- a/tfworker/util/terraform_helpers.py +++ b/tfworker/util/terraform_helpers.py @@ -5,12 +5,11 @@ from tempfile import TemporaryDirectory from typing import TYPE_CHECKING, Dict, List, Union -import hcl2 -from lark.exceptions import UnexpectedToken from packaging.specifiers import InvalidSpecifier, SpecifierSet import tfworker.util.log as log from tfworker.exceptions import TFWorkerException +from tfworker.util.hcl_parser import parse_files as parse_hcl_files from tfworker.util.system import get_platform if TYPE_CHECKING: @@ -151,21 +150,46 @@ def _find_required_providers( Dict[str, Dict[str, ProviderRequirements]]: A dictionary of required providers. """ providers = {} + # Collect all .tf files first + tf_files: List[str] = [] for root, _, files in os.walk(search_dir, followlinks=True): for file in files: if file.endswith(".tf"): - with open(f"{root}/{file}", "r") as f: - try: - content = hcl2.load(f) - except UnexpectedToken as e: - log.info( - f"not processing {root}/{file} for required providers; see debug output for HCL parsing errors" - ) - log.debug(f"HCL processing errors in {root}/{file}: {e}") - continue - _update_parsed_providers( - providers, _parse_required_providers(content) - ) + tf_files.append(f"{root}/{file}") + + if not tf_files: + log.trace("No .tf files found for required providers search") + return providers + + # Parse in batch when possible + try: + ok_map, err_map = parse_hcl_files(tf_files) + except Exception as e: + # If the batch parsing fails catastrophically, fallback to per-file parse + log.debug(f"Batch HCL parsing failed; falling back to per-file: {e}") + log.trace("Using per-file HCL parser for required providers") + ok_map, err_map = {}, {} + from tfworker.util.hcl_parser import ( + parse_file as parse_hcl_file, # local import fallback + ) + + for fp in tf_files: + try: + ok_map[fp] = parse_hcl_file(fp) + except Exception as ee: + err_map[fp] = str(ee) + else: + log.trace("Using batch HCL parser for required providers") + + # Log errors like before and process successes + for fp, emsg in err_map.items(): + log.info( + f"not processing {fp} for required providers; see debug output for HCL parsing errors" + ) + log.debug(f"HCL processing errors in {fp}: {emsg}") + + for fp, content in ok_map.items(): + _update_parsed_providers(providers, _parse_required_providers(content)) log.trace( f"Found required providers: {[x for x in providers.keys()]} in {search_dir}" ) diff --git a/tools/hcl2json/go.mod b/tools/hcl2json/go.mod new file mode 100644 index 0000000..43aaaf6 --- /dev/null +++ b/tools/hcl2json/go.mod @@ -0,0 +1,14 @@ +module tfworker/tools/hcl2json + +go 1.20 + +require github.com/tmccombs/hcl2json v0.6.1 + +require ( + github.com/agext/levenshtein v1.2.3 // indirect + github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect + github.com/hashicorp/hcl/v2 v2.19.1 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/zclconf/go-cty v1.14.1 // indirect + golang.org/x/text v0.11.0 // indirect +) diff --git a/tools/hcl2json/go.sum b/tools/hcl2json/go.sum new file mode 100644 index 0000000..7b0a39f --- /dev/null +++ b/tools/hcl2json/go.sum @@ -0,0 +1,21 @@ +github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= +github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= +github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/hashicorp/hcl/v2 v2.19.1 h1://i05Jqznmb2EXqa39Nsvyan2o5XyMowW5fnCKW5RPI= +github.com/hashicorp/hcl/v2 v2.19.1/go.mod h1:ThLC89FV4p9MPW804KVbe/cEXoQ8NZEh+JtMeeGErHE= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/tmccombs/hcl2json v0.6.1 h1:KYk0h4vqLR2LzecgsgCEhuDbNfdvSu65CEZ7VGDgBl0= +github.com/tmccombs/hcl2json v0.6.1/go.mod h1:Bqe5itpqem41iD5O2vCfiP1MoDednwR4/vHTRDpjM4A= +github.com/zclconf/go-cty v1.14.1 h1:t9fyA35fwjjUMcmL5hLER+e/rEPqrbCK1/OSE4SI9KA= +github.com/zclconf/go-cty v1.14.1/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= diff --git a/tools/hcl2json/main.go b/tools/hcl2json/main.go new file mode 100644 index 0000000..fee2ce7 --- /dev/null +++ b/tools/hcl2json/main.go @@ -0,0 +1,111 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "os" + + convert "github.com/tmccombs/hcl2json/convert" +) + +// This small CLI bridges HashiCorp's native Go HCL parser into tfworker. +// It reads HCL from a file or stdin and emits a JSON object to stdout that +// closely matches python-hcl2's structure to minimize Python-side changes. +// +// Build: +// go build -o tfworker-hcl2json ./tools/hcl2json +// Usage: +// tfworker-hcl2json path/to/file.hcl +// cat file.hcl | tfworker-hcl2json --stdin +func main() { + useStdin := flag.Bool("stdin", false, "read HCL from stdin") + multi := flag.Bool("multi", false, "parse multiple files and emit {ok, errors}") + flag.Parse() + + var ( + b []byte + err error + ) + + if *multi { + if *useStdin { + fmt.Fprintln(os.Stderr, "--multi does not support --stdin; pass file paths") + os.Exit(2) + } + if flag.NArg() == 0 { + fmt.Fprintln(os.Stderr, "--multi requires at least one file path") + os.Exit(2) + } + ok := map[string]any{} + errs := map[string]string{} + for i := 0; i < flag.NArg(); i++ { + fp := flag.Arg(i) + b, err := os.ReadFile(fp) + if err != nil { + errs[fp] = err.Error() + continue + } + jb, err := convert.Bytes(b, fp, convert.Options{}) + if err != nil { + errs[fp] = err.Error() + continue + } + var obj map[string]any + if err := json.Unmarshal(jb, &obj); err != nil { + errs[fp] = err.Error() + continue + } + ok[fp] = obj + } + out := map[string]any{"ok": ok, "errors": errs} + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + if err := enc.Encode(out); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(0) + } + + if *useStdin { + b, err = io.ReadAll(os.Stdin) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + } else { + if flag.NArg() != 1 { + fmt.Fprintln(os.Stderr, "expected a single HCL file path or --stdin") + os.Exit(2) + } + fp := flag.Arg(0) + b, err = os.ReadFile(fp) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + } + + // Use convert.Bytes to parse and convert to JSON bytes + jsonBytes, err := convert.Bytes(b, "", convert.Options{}) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + // Ensure we print normalized JSON (convert.Bytes already returns canonical JSON) + var obj map[string]any + if err := json.Unmarshal(jsonBytes, &obj); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + if err := enc.Encode(obj); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/tools/hcl2json/main_test.go b/tools/hcl2json/main_test.go new file mode 100644 index 0000000..d3741ea --- /dev/null +++ b/tools/hcl2json/main_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "encoding/json" + "os" + "testing" + + convert "github.com/tmccombs/hcl2json/convert" +) + +func TestConvertBasicTerraform(t *testing.T) { + hclData := []byte(` +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.0.0" + } + } +} +`) + b, err := convert.Bytes(hclData, "inmem", convert.Options{}) + if err != nil { + t.Fatalf("convert.Bytes error: %v", err) + } + if len(b) == 0 { + t.Fatalf("expected non-empty JSON output") + } +} + +func TestConvertFixtureVersionsTF(t *testing.T) { + // Use a repo fixture file to ensure we handle real Terraform syntax + fp := "../../tests/fixtures/definitions/test_a/versions.tf" + b, err := os.ReadFile(fp) + if err != nil { + t.Fatalf("read fixture: %v", err) + } + jb, err := convert.Bytes(b, fp, convert.Options{}) + if err != nil { + t.Fatalf("convert fixture: %v", err) + } + var m map[string]any + if err := json.Unmarshal(jb, &m); err != nil { + t.Fatalf("unmarshal json: %v", err) + } + if _, ok := m["terraform"]; !ok { + t.Fatalf("expected 'terraform' key in converted output") + } +}