diff --git a/.gitignore b/.gitignore index 0d2c1c5c2..afb5191aa 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,10 @@ __pycache__/ # C extensions *.so +# developer utilities +scratch +typescript + # Distribution / packaging .Python build/ diff --git a/.rat-excludes b/.rat-excludes new file mode 100644 index 000000000..7a51824eb --- /dev/null +++ b/.rat-excludes @@ -0,0 +1,26 @@ +# Python marker file +.*py\.typed + +# JSON Lines data files +.*\.jsonl + +# Git and version control config +\.gitignore +\.gitmodules +\.rat-excludes + +# Jupyter notebooks (JSON format, cannot practically add headers) +.*\.ipynb + +# Data files (CSV - not source code) +.*\.csv + +# Text files (documentation/data files, not source code) +.*\.txt + +# Build and tool config files +.*\.bat +robots\.txt + +# Examples pattern (legacy - keeping for compatibility) +apache_burr-.*/burr/examples diff --git a/burr/tracking/server/s3/deployment/terraform/templates/ecs/burr_app.json.tpl b/burr/tracking/server/s3/deployment/terraform/templates/ecs/burr_app.json.tpl index ba610dba5..bc036bf3d 100644 --- a/burr/tracking/server/s3/deployment/terraform/templates/ecs/burr_app.json.tpl +++ b/burr/tracking/server/s3/deployment/terraform/templates/ecs/burr_app.json.tpl @@ -1,4 +1,21 @@ -[ +${/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ ""}[ { "name": "burr-app", "image": "${app_image}", diff --git a/burr_logo.svg b/burr_logo.svg index 4681ea991..972d9bd42 100644 --- a/burr_logo.svg +++ b/burr_logo.svg @@ -1,4 +1,22 @@ + + diff --git a/docs/_templates/page.html b/docs/_templates/page.html new file mode 100644 index 000000000..8ac039968 --- /dev/null +++ b/docs/_templates/page.html @@ -0,0 +1,23 @@ +{% extends "!page.html" %} + +{% block extrahead %} +{{ super() }} + + + +{% endblock %} diff --git a/examples/other-examples/cowsay/digraph b/examples/other-examples/cowsay/digraph index e4be488bd..80cd763ed 100644 --- a/examples/other-examples/cowsay/digraph +++ b/examples/other-examples/cowsay/digraph @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + digraph { graph [compound=false concentrate=false rankdir=TB ranksep=0.4] say_nothing [label=say_nothing shape=box style=rounded] diff --git a/scripts/README.md b/scripts/README.md index ed9c12b15..92393c9bf 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -17,232 +17,129 @@ under the License. --> -# Burr Release Scripts +# Policy on source versus distribution -This directory contains helper scripts to automate the Apache release workflow. +Apache Burr is an apache-incubating project. As such, we intend to follow all Apache guidelines to +both the spirit (and when applicable) the letter. -## Overview +That said, there is occasional ambiguity. Thus we aim to clarify with a reasonable and consistently maintained +approach. The question that we found most ambiguous when determining our release process is +1. What counts as source code, and should thus be included in the "sdist" (the source-only distribution) +2. What should be included in the build? -The release process has two phases: +Specifically, we set the following guidelines: -1. **Source-only release** (for Apache voting): Contains source code, build scripts, and UI source—but NO pre-built artifacts -2. **Wheel build** (for PyPI): Built from the source release, includes pre-built UI assets +| | source (to vote on) -- tar.gz | sdist -- source used to build | whl file | Reasoning | +|---|---|---|---|---| +| Build Scripts | ✓ | ✓ | ✗ | Included in tar.gz and sdist as they are needed to reproduce the build, but not in the whl. These are only meant to be consumed by developers/pod members. | +| Library Source code | ✓ | ✓ | ✓ | Core library source code is included in all three distributions: tar.gz, sdist, and whl. | +| Tests (integration and unit) | ✓ | ✓ | ✗ | We expect users/PMC to download the source distribution, build from source, run the tests, and validate. Thus we include in the tar.gz and sdist, but not in the whl. | +| READMEs | ✓ | ✓ | ✓ | Standard project metadata files (README.md, LICENSE, NOTICE, DISCLAIMER) are included in all three distributions: tar.gz, sdist, and whl. | +| Documentation | ✓ | ✗ | ✗ | Documentation source is included in the tar.gz for voters to review, but not in the sdist or whl as it is not needed for building or using the package. | +| Deployment templates | ✓ | ✓ | ✓ | Convenience deployment templates are included in tar.gz, sdist, and whl as they are referred to by specific utility commands for deploying that are included in source. | +| Built artifacts (UI, etc...) | ✗ | ✗ | ✓ | These are not source code and are only included in the whl. They are created through a build process from the UI source. Notable examples include the built npm packages. | +| Examples (by default required for demo server) | ✓ | ✓ | ✓ | We have four examples (see pyproject.toml) required by the demo server which can be run by a single command. These are included in tar.gz, sdist, and whl as they are needed for the demo functionality. | +| Other examples | ✓ | ✗ | ✗ | These are included in the tar.gz for voters to review but not included in the sdist or whl as they are not needed to build or run the package. They serve more as documentation. | -All packaging configuration lives in `pyproject.toml`: -- `[build-system]` uses `flit_core` as the build backend -- `[tool.flit.sdist]` controls what goes in the source tarball -- Wheel contents are controlled by what exists in `burr/` when `flit build --format wheel` runs -## 1. Create the Source Release Candidate -From the repo root: -```bash -python scripts/release_helper.py [--dry-run] [--build-wheel] -``` - -Example: - -```bash -# Dry run (no git tag or SVN upload) -python scripts/release_helper.py 0.41.0 0 myid --dry-run +# Release Process -# Real release -python scripts/release_helper.py 0.41.0 0 myid +**Note:** This is a work in progress and subject to change. -# With optional wheel -python scripts/release_helper.py 0.41.0 0 myid --build-wheel -``` - -**What it does:** -1. Reads version from `pyproject.toml` -2. Cleans `dist/` directory -3. **Removes `burr/tracking/server/build/`** to ensure no pre-built UI in source tarball -4. Runs `flit build --format sdist` - - Includes files specified in `[tool.flit.sdist] include` - - Excludes files specified in `[tool.flit.sdist] exclude` -5. Creates Apache-branded tarball with GPG signatures and SHA512 checksums -6. Tags git as `v{version}-incubating-RC{num}` (unless `--dry-run`) -7. Uploads to Apache SVN (unless `--dry-run`) - -**Output:** -- `dist/apache-burr--incubating.tar.gz` — ASF-branded source tarball -- `dist/apache-burr--incubating.tar.gz.asc` — GPG signature -- `dist/apache-burr--incubating.tar.gz.sha512` — SHA512 checksum +## Environment Setup -## 2. Test the Source Release (Voter Simulation) -This simulates what Apache voters and release managers will do when validating the release. - -**Automated testing:** +Prerequisites: +- Python 3.9+ +- `flit` for building (`pip install flit`) +- GPG key configured for signing +- Node.js + npm for UI builds +- Apache RAT jar for license checking (optional) ```bash -bash scripts/simulate_release.sh -``` - -This script: -1. Cleans `/tmp/burr-release-test/` -2. Extracts the Apache tarball -3. Creates a fresh virtual environment -4. Builds UI artifacts and wheel (next step) -5. Verifies both packages and prints their locations - -**Manual testing:** - -```bash -cd /tmp -tar -xzf /path/to/dist/apache-burr--incubating.tar.gz -cd apache-burr--incubating - -# Verify source contents -ls scripts/ # Build scripts should be present -ls telemetry/ui/ # UI source should be present -ls examples/ # Example directories should be present -ls burr/tracking/server/build/ # Should NOT exist (no pre-built UI) - -# Create clean environment -python -m venv venv && source venv/bin/activate -pip install -e ".[cli]" +# Install build dependencies pip install flit +pip install -e ".[cli]" # Installs burr-admin-build-ui command -# Build artifacts and wheel (see step 3) -python scripts/build_artifacts.py all --clean -ls dist/*.whl -deactivate -``` +# Verify GPG setup +gpg --list-secret-keys -Alternatively, instead of manually creating the `venv` and installing burr with `pip install`, you can use -`uv` and use simplified development workflow of uv you can run the command directly: - -```bash -uv run scripts/build_artifacts.py all --clean -ls dist/*.whl +# Build UI assets (one-time or when UI changes) +cd telemetry/ui && npm install && npm run build && cd ../.. ``` -This will automatically: - -* download the right python version if you do not have python installed -* create virtual environment in local `.venv` directory -* activates the venv -* installs `burr` in editable mode with `dev` dependency group (that contains `cli` extra, `developer` extra - and `flit` package. -* deactivates the venv - -Next time when you run `uv run` it will also automatically sync the environment with latest `pyproject.toml` +## Building Artifacts -## 3. Build Artifacts and Wheel +Creates the three required distributions: git archive (voting artifact), sdist (source distribution), and wheel (binary distribution). All artifacts are automatically signed with GPG and checksummed with SHA512. The `all` command is the typical workflow - it builds everything in sequence. -The `build_artifacts.py` script has three subcommands: - -### Build everything (recommended): +Main release script: `scripts/apache_release.py` ```bash -python scripts/build_artifacts.py all --clean -``` - -This runs both `artifacts` and `wheel` subcommands in sequence. +# Full release build (creates all artifacts, signs, checksums, and generates vote email) +# Note: version, rc_num, and apache_id are POSITIONAL arguments +python scripts/apache_release.py all 0.41.0 0 your_apache_id -### Build UI artifacts only: - -```bash -python scripts/build_artifacts.py artifacts [--skip-install] -``` +# Individual steps +python scripts/apache_release.py archive 0.41.0 0 # Git archive +python scripts/apache_release.py sdist 0.41.0 0 # Source dist +python scripts/apache_release.py wheel 0.41.0 0 # Wheel dist -**What it does:** -1. Checks for Node.js and npm -2. **Cleans `burr/tracking/server/build/`** to ensure fresh UI build -3. Installs burr from source: `pip install -e .` (unless `--skip-install`) -4. Runs `burr-admin-build-ui`: - - `npm install --prefix telemetry/ui` - - `npm run build --prefix telemetry/ui` - - **Creates `burr/tracking/server/build/`** and copies built UI into it -5. Verifies UI assets exist in `burr/tracking/server/build/` +# Upload to SVN +python scripts/apache_release.py upload 0.41.0 0 your_apache_id +python scripts/apache_release.py upload 0.41.0 0 your_apache_id --dry-run # Test first -### Build wheel only (assumes artifacts exist): +# Verify artifacts locally +python scripts/apache_release.py verify 0.41.0 0 -```bash -python scripts/build_artifacts.py wheel [--clean] +# Skip upload step in 'all' command +python scripts/apache_release.py all 0.41.0 0 your_apache_id --no-upload ``` -**What it does:** -1. Checks for `flit` -2. Verifies `burr/tracking/server/build/` contains UI assets -3. Optionally cleans `dist/` (with `--clean`) -4. Runs `flit build --format wheel` - - **Packages all files in `burr/` directory, including `burr/tracking/server/build/`** - - Does NOT include files outside `burr/` (e.g., `telemetry/ui/`, `scripts/`, `examples/`) -5. Verifies `.whl` file was created +Output: `dist/` directory with tar.gz (archive + sdist), whl, plus .asc and .sha512 files. Install from the whl file to test it out after runnig the `wheel` subcommand. + +## Verification -**Output:** `dist/apache_burr--py3-none-any.whl` (includes bundled UI) +Validate artifacts before uploading or voting. Checks GPG signatures, SHA512 checksums, archive integrity, and license compliance with Apache RAT. The `list-contents` command is useful for inspecting what's actually packaged in each artifact. -**Note:** Flit normalizes the package name `apache-burr` to `apache_burr` (underscore) in the filename. +Verification script: `scripts/verify_apache_artifacts.py` -## 4. Upload to PyPI +### Prerequisites -After building the wheel: +For license verification, you'll need Apache RAT. Download it from: ```bash -twine upload dist/apache_burr--py3-none-any.whl +# Download Apache RAT jar +curl -O https://repo1.maven.org/maven2/org/apache/rat/apache-rat/0.15/apache-rat-0.15.jar ``` -**Note:** For PyPI, you may want to publish as `burr` instead of `apache-burr`. See the dual distribution strategy documentation. - -## Package Contents Reference - -Understanding what goes in each package type: +Or download manually from: https://repo1.maven.org/maven2/org/apache/rat/apache-rat/0.15/ -### Source tarball (`apache-burr-{version}-incubating.tar.gz`) +### Running Verification -**Controlled by:** `[tool.flit.sdist]` in `pyproject.toml` + `release_helper.py` cleanup +```bash +# Verify signatures and checksums +python scripts/verify_apache_artifacts.py signatures -**Includes:** -- ✅ `burr/` — Full package source code -- ✅ `scripts/` — Build helper scripts (this directory!) -- ✅ `telemetry/ui/` — UI source code (package.json, src/, public/, etc.) -- ✅ `examples/email-assistant/`, `examples/multi-modal-chatbot/`, etc. — Selected example directories -- ✅ `LICENSE`, `NOTICE`, `DISCLAIMER` — Apache license files +# Verify licenses (requires Apache RAT) +python scripts/verify_apache_artifacts.py licenses --rat-jar apache-rat-0.15.jar -**Excludes:** -- ❌ `burr/tracking/server/build/` — Deleted by `release_helper.py` before build -- ❌ `telemetry/ui/node_modules/` — Excluded by `[tool.flit.sdist]` -- ❌ `telemetry/ui/build/`, `telemetry/ui/dist/` — Excluded by `[tool.flit.sdist]` -- ❌ `docs/`, `.git/`, `.github/` — Excluded by `[tool.flit.sdist]` +# Verify everything +python scripts/verify_apache_artifacts.py all --rat-jar apache-rat-0.15.jar -**How it's built:** -```bash -rm -rf burr/tracking/server/build # Ensure no pre-built UI -flit build --format sdist # Build from [tool.flit.sdist] config +# Inspect artifact contents +python scripts/verify_apache_artifacts.py list-contents dist/apache-burr-0.41.0.tar.gz +python scripts/verify_apache_artifacts.py list-contents dist/apache_burr-0.41.0-py3-none-any.whl ``` ---- - -### Wheel (`apache_burr-{version}-py3-none-any.whl`) +## Local Development -**Controlled by:** What exists in `burr/` directory when `flit build --format wheel` runs +Simpler workflow for building wheels during development without signing or creating full release artifacts. Useful for testing packaging changes or building wheels to install locally. -**Includes:** -- ✅ `burr/` — Complete package (all `.py` files, `py.typed`, etc.) -- ✅ `burr/tracking/server/build/` — **Pre-built UI assets** (created by `build_artifacts.py`) -- ✅ `burr/tracking/server/demo_data/` — Demo data files +For local wheel building/testing (simpler, no signing): -**Excludes:** -- ❌ `telemetry/ui/` — Not in `burr/` package -- ❌ `examples/` — Not in `burr/` package (sdist-only) -- ❌ `scripts/` — Not in `burr/` package (sdist-only) -- ❌ `LICENSE`, `NOTICE`, `DISCLAIMER` — Not needed in wheel (sdist-only) - -**How it's built:** ```bash -burr-admin-build-ui # Creates burr/tracking/server/build/ -flit build --format wheel # Packages everything in burr/ +python scripts/build_artifacts.py build-ui # Build UI only +python scripts/build_artifacts.py wheel # Build wheel with UI ``` - ---- - -### Key Insight - -The **same `burr/` source directory** produces different outputs based on **when you build** and **what format**: - -1. **sdist (source tarball):** Includes external files (`scripts/`, `telemetry/ui/`, `examples/`) via `[tool.flit.sdist]` config, but excludes `burr/tracking/server/build/` because we delete it first. - -2. **wheel (binary distribution):** Only packages `burr/` directory contents, but includes `burr/tracking/server/build/` because we create it before building the wheel. diff --git a/scripts/apache_release.py b/scripts/apache_release.py new file mode 100644 index 000000000..64965feb2 --- /dev/null +++ b/scripts/apache_release.py @@ -0,0 +1,945 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Apache Burr Release Script (SIMPLIFIED VERSION) + +This script automates the Apache release process: +1. Create git archive (voting artifact) +2. Build source distribution (sdist) +3. Build wheel +4. Upload to Apache SVN + +Usage: + python scripts/apache_release_simplified.py all 0.41.0 0 myid + python scripts/apache_release_simplified.py wheel 0.41.0 0 +""" + +import argparse +import glob +import hashlib +import os +import re +import shutil +import subprocess +import sys +from typing import NoReturn, Optional + +# --- Configuration --- +PROJECT_SHORT_NAME = "burr" +VERSION_FILE = "pyproject.toml" +VERSION_PATTERN = r'version\s*=\s*"(\d+\.\d+\.\d+)"' + +# Required examples for wheel (from pyproject.toml) +REQUIRED_EXAMPLES = [ + "__init__.py", + "email-assistant", + "multi-modal-chatbot", + "streaming-fastapi", + "deep-researcher", +] + + +# ============================================================================ +# Utility Functions +# ============================================================================ + + +def _fail(message: str) -> NoReturn: + """Print error message and exit.""" + print(f"\n❌ {message}") + sys.exit(1) + + +def _print_section(title: str) -> None: + """Print a formatted section header.""" + print("\n" + "=" * 80) + print(f" {title}") + print("=" * 80 + "\n") + + +def _print_step(step_num: int, total: int, description: str) -> None: + """Print a formatted step header.""" + print(f"\n[Step {step_num}/{total}] {description}") + print("-" * 80) + + +# ============================================================================ +# Environment Validation +# ============================================================================ + + +def _validate_environment_for_command(args) -> None: + """Validate required tools for the requested command.""" + print("\n" + "=" * 80) + print(" Environment Validation") + print("=" * 80 + "\n") + + # Define required tools for each command + command_requirements = { + "archive": ["git", "gpg"], + "sdist": ["git", "gpg", "flit"], + "wheel": ["git", "gpg", "flit", "node", "npm"], + "upload": ["git", "gpg", "svn"], + "all": ["git", "gpg", "flit", "node", "npm", "svn"], + "verify": ["git", "gpg"], + } + + required_tools = command_requirements.get(args.command, ["git", "gpg"]) + + # Check for RAT if needed + if hasattr(args, "check_licenses") or hasattr(args, "check_licenses_report"): + if getattr(args, "check_licenses", False) or getattr(args, "check_licenses_report", False): + required_tools.append("java") + if not getattr(args, "rat_jar", None): + _fail("--rat-jar is required when using --check-licenses") + + # Check each tool + missing_tools = [] + print("Checking required tools:") + + for tool in required_tools: + if shutil.which(tool) is None: + missing_tools.append(tool) + print(f" ✗ '{tool}' not found") + else: + print(f" ✓ '{tool}' found") + + if missing_tools: + print("\n❌ Missing required tools:") + for tool in missing_tools: + if tool == "flit": + print(f" • {tool}: Install with 'pip install flit'") + elif tool in ["node", "npm"]: + print(f" • {tool}: Install from https://nodejs.org/") + else: + print(f" • {tool}") + sys.exit(1) + + print("\n✓ All required tools are available\n") + + +# ============================================================================ +# Prerequisites +# ============================================================================ + + +def _verify_project_root() -> bool: + """Verify script is running from project root.""" + if not os.path.exists("pyproject.toml"): + _fail("pyproject.toml not found. Please run from project root.") + return True + + +def _get_version_from_file(file_path: str) -> str: + """Extract version from pyproject.toml.""" + with open(file_path, encoding="utf-8") as f: + content = f.read() + match = re.search(VERSION_PATTERN, content) + if match: + return match.group(1) + _fail(f"Could not find version in {file_path}") + + +def _validate_version(requested_version: str) -> bool: + """Validate that requested version matches pyproject.toml.""" + current_version = _get_version_from_file(VERSION_FILE) + if current_version != requested_version: + _fail( + f"Version mismatch!\n" + f" Requested: {requested_version}\n" + f" In {VERSION_FILE}: {current_version}\n" + f"Please update {VERSION_FILE} to {requested_version} first." + ) + print(f"✓ Version validated: {requested_version}\n") + return True + + +def _check_git_working_tree() -> None: + """Check git working tree status and warn if dirty.""" + try: + dirty = ( + subprocess.check_output(["git", "status", "--porcelain"], stderr=subprocess.DEVNULL) + .decode() + .strip() + ) + if dirty: + print("⚠️ Warning: Git working tree has uncommitted changes:") + for line in dirty.splitlines()[:10]: + print(f" {line}") + if len(dirty.splitlines()) > 10: + print(f" ... and {len(dirty.splitlines()) - 10} more files") + print() + except subprocess.CalledProcessError: + pass + + +# ============================================================================ +# Signing and Verification +# ============================================================================ + + +def _sign_artifact(artifact_path: str) -> tuple[str, str]: + """Sign artifact with GPG and create SHA512 checksum.""" + signature_path = f"{artifact_path}.asc" + checksum_path = f"{artifact_path}.sha512" + + # GPG signature + try: + subprocess.run( + ["gpg", "--armor", "--output", signature_path, "--detach-sig", artifact_path], + check=True, + ) + print(f" ✓ Created GPG signature: {signature_path}") + except subprocess.CalledProcessError as e: + _fail(f"Error signing artifact: {e}") + + # SHA512 checksum + sha512_hash = hashlib.sha512() + with open(artifact_path, "rb") as f: + while chunk := f.read(65536): + sha512_hash.update(chunk) + + with open(checksum_path, "w", encoding="utf-8") as f: + f.write(f"{sha512_hash.hexdigest()}\n") + print(f" ✓ Created SHA512 checksum: {checksum_path}") + + return (signature_path, checksum_path) + + +def _verify_artifact_signature(artifact_path: str, signature_path: str) -> bool: + """Verify GPG signature of artifact.""" + if not os.path.exists(signature_path): + print(f" ✗ Signature file not found: {signature_path}") + return False + + try: + result = subprocess.run( + ["gpg", "--verify", signature_path, artifact_path], + capture_output=True, + check=False, + ) + if result.returncode == 0: + print(" ✓ GPG signature is valid") + return True + else: + print(" ✗ GPG signature verification failed") + return False + except subprocess.CalledProcessError: + return False + + +def _verify_artifact_checksum(artifact_path: str, checksum_path: str) -> bool: + """Verify SHA512 checksum of artifact.""" + if not os.path.exists(checksum_path): + print(f" ✗ Checksum file not found: {checksum_path}") + return False + + # Read expected checksum + with open(checksum_path, "r", encoding="utf-8") as f: + expected_checksum = f.read().strip().split()[0] + + # Calculate actual checksum + sha512_hash = hashlib.sha512() + with open(artifact_path, "rb") as f: + while chunk := f.read(65536): + sha512_hash.update(chunk) + + actual_checksum = sha512_hash.hexdigest() + + if actual_checksum == expected_checksum: + print(" ✓ SHA512 checksum is valid") + return True + else: + print(" ✗ SHA512 checksum mismatch!") + return False + + +def _verify_artifact_complete(artifact_path: str) -> bool: + """Verify artifact and its signature/checksum files.""" + print(f"\nVerifying artifact: {os.path.basename(artifact_path)}") + + if not os.path.exists(artifact_path): + print(f" ✗ Artifact not found: {artifact_path}") + return False + + # Verify signature and checksum + signature_path = f"{artifact_path}.asc" + checksum_path = f"{artifact_path}.sha512" + + sig_valid = _verify_artifact_signature(artifact_path, signature_path) + checksum_valid = _verify_artifact_checksum(artifact_path, checksum_path) + + if sig_valid and checksum_valid: + print(f" ✓ All checks passed for {os.path.basename(artifact_path)}\n") + return True + return False + + +# ============================================================================ +# Step 1: Git Archive +# ============================================================================ + + +def _create_git_archive(version: str, rc_num: str, output_dir: str = "dist") -> str: + """Create git archive tar.gz for voting.""" + print(f"Creating git archive for version {version}-incubating...") + + os.makedirs(output_dir, exist_ok=True) + + archive_name = f"apache-burr-{version}-incubating-src.tar.gz" + archive_path = os.path.join(output_dir, archive_name) + prefix = f"apache-burr-{version}-incubating-src/" + + try: + subprocess.run( + [ + "git", + "archive", + "HEAD", + f"--prefix={prefix}", + "--format=tar.gz", + "--output", + archive_path, + ], + check=True, + ) + print(f" ✓ Created git archive: {archive_path}") + except subprocess.CalledProcessError as e: + _fail(f"Error creating git archive: {e}") + + file_size = os.path.getsize(archive_path) + print(f" ✓ Archive size: {file_size:,} bytes") + + # Sign the archive + print("Signing archive...") + _sign_artifact(archive_path) + + # Verify + if not _verify_artifact_complete(archive_path): + _fail("Archive verification failed!") + + return archive_path + + +# ============================================================================ +# Step 2: Build Source Distribution (sdist) +# ============================================================================ + + +def _remove_ui_build_artifacts() -> None: + """Remove pre-built UI artifacts to ensure clean build.""" + ui_build_dir = os.path.join("burr", "tracking", "server", "build") + if os.path.exists(ui_build_dir): + print(f" Removing UI build artifacts: {ui_build_dir}") + shutil.rmtree(ui_build_dir) + print(" ✓ UI build artifacts removed") + + +def _build_sdist_from_git(version: str, output_dir: str = "dist") -> str: + """Build source distribution from git using flit.""" + _print_step(1, 2, "Building sdist with flit") + + os.makedirs(output_dir, exist_ok=True) + _remove_ui_build_artifacts() + _check_git_working_tree() + + print(" Running flit build --format sdist...") + try: + env = os.environ.copy() + env["FLIT_USE_VCS"] = "0" + subprocess.run( + ["flit", "build", "--format", "sdist"], + env=env, + capture_output=True, + text=True, + check=True, + ) + print(" ✓ flit sdist created successfully") + except subprocess.CalledProcessError as e: + _fail(f"Failed to build sdist: {e.stderr}") + + # Find and rename sdist + expected_pattern = f"dist/apache_burr-{version.lower()}.tar.gz" + sdist_files = glob.glob(expected_pattern) + + if not sdist_files: + _fail(f"Could not find sdist: {expected_pattern}") + + original_sdist = sdist_files[0] + apache_sdist = os.path.join( + output_dir, f"apache-burr-{version.lower()}-incubating-src-sdist.tar.gz" + ) + + if os.path.exists(apache_sdist): + os.remove(apache_sdist) + + shutil.move(original_sdist, apache_sdist) + print(f" ✓ Renamed to: {os.path.basename(apache_sdist)}") + + return apache_sdist + + +# ============================================================================ +# Step 3: Build Wheel (SIMPLIFIED!) +# ============================================================================ + + +def _build_ui_artifacts() -> None: + """Build UI artifacts using burr-admin-build-ui.""" + print("Building UI artifacts...") + + ui_build_dir = "burr/tracking/server/build" + + # Clean existing UI build + if os.path.exists(ui_build_dir): + shutil.rmtree(ui_build_dir) + + # Check for burr-admin-build-ui + if shutil.which("burr-admin-build-ui") is None: + _fail("burr-admin-build-ui not found. Install with: pip install -e .[cli]") + + # Build UI + env = os.environ.copy() + env["BURR_PROJECT_ROOT"] = os.getcwd() + + try: + subprocess.run(["burr-admin-build-ui"], check=True, env=env, capture_output=True) + print(" ✓ UI artifacts built successfully") + except subprocess.CalledProcessError as e: + _fail(f"Error building UI: {e}") + + # Verify + if not os.path.exists(ui_build_dir) or not os.listdir(ui_build_dir): + _fail(f"UI build directory is empty: {ui_build_dir}") + + +def _prepare_wheel_contents() -> tuple[bool, bool, Optional[str]]: + """Handle burr/examples symlink: replace with real files for wheel.""" + burr_examples_dir = "burr/examples" + source_examples_dir = "examples" + + if not os.path.exists(source_examples_dir): + print(f" ⚠️ {source_examples_dir} not found") + return (False, False, None) + + # Check if burr/examples is a symlink (should be in dev repo) + was_symlink = False + symlink_target = None + + if os.path.exists(burr_examples_dir): + if os.path.islink(burr_examples_dir): + was_symlink = True + symlink_target = os.readlink(burr_examples_dir) + print(f" Removing symlink: burr/examples -> {symlink_target}") + os.remove(burr_examples_dir) + else: + shutil.rmtree(burr_examples_dir) + + # Copy the 4 required examples + print(" Copying examples to burr/examples/...") + os.makedirs(burr_examples_dir, exist_ok=True) + + # Copy __init__.py + init_src = os.path.join(source_examples_dir, "__init__.py") + if os.path.exists(init_src): + shutil.copy2(init_src, os.path.join(burr_examples_dir, "__init__.py")) + + # Copy example directories + for example_dir in REQUIRED_EXAMPLES[1:]: # Skip __init__.py + src_path = os.path.join(source_examples_dir, example_dir) + dest_path = os.path.join(burr_examples_dir, example_dir) + + if os.path.exists(src_path) and os.path.isdir(src_path): + shutil.copytree(src_path, dest_path, dirs_exist_ok=True) + print(f" ✓ Copied {example_dir}") + + return (True, was_symlink, symlink_target) + + +def _cleanup_wheel_contents(was_symlink: bool, symlink_target: Optional[str]) -> None: + """Restore burr/examples symlink after wheel build.""" + burr_examples_dir = "burr/examples" + + if os.path.exists(burr_examples_dir): + shutil.rmtree(burr_examples_dir) + + if was_symlink and symlink_target: + print(f" Restoring symlink: burr/examples -> {symlink_target}") + os.symlink(symlink_target, burr_examples_dir) + print(" ✓ Symlink restored") + + +def _build_wheel_from_current_dir(version: str, output_dir: str = "dist") -> str: + """Build wheel from current directory (matches what voters do). + + This is MUCH simpler than the old approach: + - No temp directory extraction + - No copying UI between directories + - Just build in place and clean up + """ + _print_step(1, 3, "Building UI artifacts") + _build_ui_artifacts() + + _print_step(2, 3, "Preparing wheel contents") + copied, was_symlink, symlink_target = _prepare_wheel_contents() + + _print_step(3, 3, "Building wheel with flit") + + try: + env = os.environ.copy() + env["FLIT_USE_VCS"] = "0" + + subprocess.run( + ["flit", "build", "--format", "wheel"], + env=env, + check=True, + capture_output=True, + ) + print(" ✓ Wheel built successfully") + + # Find the wheel + wheel_pattern = f"dist/apache_burr-{version}*.whl" + wheel_files = glob.glob(wheel_pattern) + + if not wheel_files: + _fail(f"No wheel found matching: {wheel_pattern}") + + wheel_path = wheel_files[0] + print(f" ✓ Wheel created: {os.path.basename(wheel_path)}") + + return wheel_path + + except subprocess.CalledProcessError as e: + _fail(f"Wheel build failed: {e}") + finally: + # Always restore symlinks + if copied: + _cleanup_wheel_contents(was_symlink, symlink_target) + + +def _verify_wheel(wheel_path: str) -> bool: + """Verify wheel contents are correct.""" + import zipfile + + print(f" Verifying wheel contents: {os.path.basename(wheel_path)}") + + try: + with zipfile.ZipFile(wheel_path, "r") as whl: + file_list = whl.namelist() + + # Check for UI build artifacts + ui_files = [f for f in file_list if "burr/tracking/server/build/" in f] + if not ui_files: + print(" ✗ No UI build artifacts found") + return False + print(f" ✓ Found {len(ui_files)} UI build files") + + # Check for required examples + for example in REQUIRED_EXAMPLES: + prefix = f"burr/examples/{example}" + example_files = [f for f in file_list if f.startswith(prefix)] + if not example_files: + print(f" ✗ Required example not found: {example}") + return False + + print(" ✓ All 4 required examples found") + print(f" ✓ Wheel contains {len(file_list)} total files") + return True + + except Exception as e: + print(f" ✗ Error verifying wheel: {e}") + return False + + +# ============================================================================ +# Upload to Apache SVN +# ============================================================================ + + +def _collect_all_artifacts(version: str, output_dir: str = "dist") -> list[str]: + """Collect all artifacts for upload.""" + if not os.path.exists(output_dir): + return [] + + artifacts = [] + for filename in os.listdir(output_dir): + if f"{version}-incubating" in filename: + if any(filename.endswith(ext) for ext in [".tar.gz", ".whl", ".asc", ".sha512"]): + artifacts.append(os.path.join(output_dir, filename)) + + return sorted(artifacts) + + +def _upload_to_svn( + version: str, + rc_num: str, + apache_id: str, + artifacts: list[str], + dry_run: bool = False, +) -> Optional[str]: + """Upload artifacts to Apache SVN distribution repository.""" + svn_url = f"https://dist.apache.org/repos/dist/dev/incubator/{PROJECT_SHORT_NAME}/{version}-incubating-RC{rc_num}" + + if dry_run: + print(f"\n[DRY RUN] Would upload to: {svn_url}") + return svn_url + + print(f"Uploading to: {svn_url}") + + try: + # Create directory + subprocess.run( + [ + "svn", + "mkdir", + "--parents", + "-m", + f"Creating directory for {version}-incubating-RC{rc_num}", + svn_url, + ], + check=True, + ) + + # Upload each file + for file_path in artifacts: + filename = os.path.basename(file_path) + print(f" Uploading {filename}...") + subprocess.run( + [ + "svn", + "import", + file_path, + f"{svn_url}/{filename}", + "-m", + f"Adding {filename}", + "--username", + apache_id, + ], + check=True, + ) + + print(f"\n✅ Artifacts uploaded to: {svn_url}") + return svn_url + + except subprocess.CalledProcessError as e: + print(f"Error during SVN upload: {e}") + return None + + +def _generate_vote_email(version: str, rc_num: str, svn_url: str) -> str: + """Generate [VOTE] email template.""" + version_with_incubating = f"{version}-incubating" + tag = f"v{version}-incubating-RC{rc_num}" + + return f"""[VOTE] Release Apache {PROJECT_SHORT_NAME} {version_with_incubating} (RC{rc_num}) + +Hi all, + +This is a call for a vote on releasing Apache {PROJECT_SHORT_NAME} {version_with_incubating}, +release candidate {rc_num}. + +The artifacts for this release candidate can be found at: +{svn_url} + +The Git tag to be voted upon is: +{tag} + +Release artifacts are signed with your GPG key. The KEYS file is available at: +https://downloads.apache.org/incubator/{PROJECT_SHORT_NAME}/KEYS + +Please download, verify, and test the release candidate. + +Some ideas to verify the release: +1. Build from source - see README in scripts/ directory for instructions +2. Install the wheel using pip to test functionality +3. Run license verification using the verify_apache_artifacts.py script or manually check + - Verify checksums and signatures match + - Check LICENSE/NOTICE files are present + - Ensure all source files have Apache headers + +The vote will run for a minimum of 72 hours. +Please vote: + +[ ] +1 Release this package as Apache {PROJECT_SHORT_NAME} {version_with_incubating} +[ ] +0 No opinion +[ ] -1 Do not release this package because... (reason required) + +Checklist for reference: +[ ] Download links are valid +[ ] Checksums and signatures are valid +[ ] LICENSE/NOTICE files exist +[ ] No unexpected binary files in source +[ ] All source files have ASF headers +[ ] Can compile from source + +On behalf of the Apache {PROJECT_SHORT_NAME} PPMC, +[Your Name] +""" + + +# ============================================================================ +# Command Handlers +# ============================================================================ + + +def cmd_archive(args) -> bool: + """Handle 'archive' subcommand.""" + _print_section(f"Creating Git Archive - v{args.version}-RC{args.rc_num}") + + _verify_project_root() + _validate_version(args.version) + _check_git_working_tree() + + archive_path = _create_git_archive(args.version, args.rc_num, args.output_dir) + print(f"\n✅ Archive created: {archive_path}") + return True + + +def cmd_sdist(args) -> bool: + """Handle 'sdist' subcommand.""" + _print_section(f"Building Source Distribution - v{args.version}-RC{args.rc_num}") + + _verify_project_root() + _validate_version(args.version) + + sdist_path = _build_sdist_from_git(args.version, args.output_dir) + + _print_step(2, 2, "Signing sdist") + _sign_artifact(sdist_path) + + if not _verify_artifact_complete(sdist_path): + _fail("sdist verification failed!") + + print(f"\n✅ Source distribution created: {sdist_path}") + return True + + +def cmd_wheel(args) -> bool: + """Handle 'wheel' subcommand.""" + _print_section(f"Building Wheel - v{args.version}-RC{args.rc_num}") + + _verify_project_root() + _validate_version(args.version) + + wheel_path = _build_wheel_from_current_dir(args.version, args.output_dir) + + print("\nSigning wheel...") + _sign_artifact(wheel_path) + + print("\nVerifying wheel...") + if not _verify_wheel(wheel_path): + _fail("Wheel verification failed!") + + if not _verify_artifact_complete(wheel_path): + _fail("Wheel signature/checksum verification failed!") + + print(f"\n✅ Wheel created and verified: {os.path.basename(wheel_path)}") + return True + + +def cmd_upload(args) -> bool: + """Handle 'upload' subcommand.""" + _print_section(f"Uploading Artifacts - v{args.version}-RC{args.rc_num}") + + artifacts = _collect_all_artifacts(args.version, args.artifacts_dir) + if not artifacts: + _fail(f"No artifacts found in {args.artifacts_dir}") + + print(f"Found {len(artifacts)} artifact(s):") + for artifact in artifacts: + print(f" - {os.path.basename(artifact)}") + + svn_url = _upload_to_svn( + args.version, args.rc_num, args.apache_id, artifacts, dry_run=args.dry_run + ) + + if not svn_url: + return False + + return True + + +def cmd_verify(args) -> bool: + """Handle 'verify' subcommand.""" + _print_section(f"Verifying Artifacts - v{args.version}-RC{args.rc_num}") + + artifacts = _collect_all_artifacts(args.version, args.artifacts_dir) + + if not artifacts: + print(f"⚠️ No artifacts found in {args.artifacts_dir}") + return False + + all_valid = True + for artifact in artifacts: + if artifact.endswith((".asc", ".sha512")): + continue # Skip signature/checksum files + if not _verify_artifact_complete(artifact): + all_valid = False + + if all_valid: + print("✅ All artifacts verified successfully!") + else: + print("❌ Some artifacts failed verification") + + return all_valid + + +def cmd_all(args) -> bool: + """Handle 'all' subcommand - run complete workflow.""" + _print_section(f"Apache Burr Release Process - v{args.version}-RC{args.rc_num}") + + if args.dry_run: + print("*** DRY RUN MODE ***\n") + + _verify_project_root() + _validate_version(args.version) + _check_git_working_tree() + + # Step 1: Git Archive + _print_step(1, 4, "Creating git archive") + _create_git_archive(args.version, args.rc_num, args.output_dir) + + # Step 2: Build sdist + _print_step(2, 4, "Building sdist") + sdist_path = _build_sdist_from_git(args.version, args.output_dir) + _sign_artifact(sdist_path) + if not _verify_artifact_complete(sdist_path): + _fail("sdist verification failed!") + + # Step 3: Build wheel + _print_step(3, 4, "Building wheel") + wheel_path = _build_wheel_from_current_dir(args.version, args.output_dir) + _sign_artifact(wheel_path) + if not _verify_wheel(wheel_path) or not _verify_artifact_complete(wheel_path): + _fail("Wheel verification failed!") + + # Step 4: Upload (if not disabled) + if not args.no_upload: + _print_step(4, 4, "Uploading to Apache SVN") + all_artifacts = _collect_all_artifacts(args.version, args.output_dir) + svn_url = _upload_to_svn( + args.version, args.rc_num, args.apache_id, all_artifacts, dry_run=args.dry_run + ) + if not svn_url: + _fail("SVN upload failed!") + else: + svn_url = f"https://dist.apache.org/repos/dist/dev/incubator/{PROJECT_SHORT_NAME}/{args.version}-incubating-RC{args.rc_num}" + if args.dry_run: + print(f"\n[DRY RUN] Would upload to: {svn_url}") + + # Generate email template + _print_section("Release Complete!") + email_content = _generate_vote_email(args.version, args.rc_num, svn_url) + print(email_content) + + return True + + +# ============================================================================ +# CLI Entry Point +# ============================================================================ + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser(description="Apache Burr Release Script (Simplified)") + subparsers = parser.add_subparsers(dest="command", required=True) + + # archive subcommand + archive_parser = subparsers.add_parser("archive", help="Create git archive") + archive_parser.add_argument("version", help="Version (e.g., '0.41.0')") + archive_parser.add_argument("rc_num", help="RC number (e.g., '0')") + archive_parser.add_argument("--output-dir", default="dist", help="Output directory") + + # sdist subcommand + sdist_parser = subparsers.add_parser("sdist", help="Build source distribution") + sdist_parser.add_argument("version", help="Version") + sdist_parser.add_argument("rc_num", help="RC number") + sdist_parser.add_argument("--output-dir", default="dist") + + # wheel subcommand + wheel_parser = subparsers.add_parser("wheel", help="Build wheel") + wheel_parser.add_argument("version", help="Version") + wheel_parser.add_argument("rc_num", help="RC number") + wheel_parser.add_argument("--output-dir", default="dist") + + # upload subcommand + upload_parser = subparsers.add_parser("upload", help="Upload to SVN") + upload_parser.add_argument("version", help="Version") + upload_parser.add_argument("rc_num", help="RC number") + upload_parser.add_argument("apache_id", help="Apache ID") + upload_parser.add_argument("--artifacts-dir", default="dist") + upload_parser.add_argument("--dry-run", action="store_true") + + # verify subcommand + verify_parser = subparsers.add_parser("verify", help="Verify artifacts") + verify_parser.add_argument("version", help="Version") + verify_parser.add_argument("rc_num", help="RC number") + verify_parser.add_argument("--artifacts-dir", default="dist") + + # all subcommand + all_parser = subparsers.add_parser("all", help="Run complete workflow") + all_parser.add_argument("version", help="Version") + all_parser.add_argument("rc_num", help="RC number") + all_parser.add_argument("apache_id", help="Apache ID") + all_parser.add_argument("--output-dir", default="dist") + all_parser.add_argument("--dry-run", action="store_true") + all_parser.add_argument("--no-upload", action="store_true") + + args = parser.parse_args() + + # Validate environment + _validate_environment_for_command(args) + + # Dispatch to command handler + try: + if args.command == "archive": + success = cmd_archive(args) + elif args.command == "sdist": + success = cmd_sdist(args) + elif args.command == "wheel": + success = cmd_wheel(args) + elif args.command == "upload": + success = cmd_upload(args) + elif args.command == "verify": + success = cmd_verify(args) + elif args.command == "all": + success = cmd_all(args) + else: + _fail(f"Unknown command: {args.command}") + except KeyboardInterrupt: + print("\n\n⚠️ Interrupted by user") + sys.exit(130) + except Exception as e: + print(f"\n❌ Unexpected error: {e}") + import traceback + + traceback.print_exc() + sys.exit(1) + + if success: + print("\n✅ Command completed successfully!") + sys.exit(0) + else: + print("\n❌ Command failed.") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/build_artifacts.py b/scripts/build_artifacts.py deleted file mode 100644 index 617873312..000000000 --- a/scripts/build_artifacts.py +++ /dev/null @@ -1,489 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -Build artifacts/wheels helper with subcommands: - - python scripts/build_artifacts.py artifacts [--skip-install] - python scripts/build_artifacts.py wheel [--clean] - python scripts/build_artifacts.py all [--skip-install] [--clean] - -Subcommands: - artifacts -> Build UI artifacts only - wheel -> Build wheel (requires artifacts to exist) - all -> Run both steps (artifacts then wheel) -""" - -import argparse -import os -import shutil -import subprocess -import sys - - -def _ensure_project_root() -> bool: - if not os.path.exists("pyproject.toml"): - print("Error: pyproject.toml not found.") - print("Please run this script from the root of the Burr source directory.") - return False - return True - - -def _check_node_prereqs() -> bool: - print("Checking for required tools...") - required_tools = ["node", "npm"] - missing_tools = [] - - for tool in required_tools: - if shutil.which(tool) is None: - missing_tools.append(tool) - print(f" ✗ '{tool}' not found") - else: - print(f" ✓ '{tool}' found") - - if missing_tools: - print(f"\nError: Missing required tools: {', '.join(missing_tools)}") - print("Please install Node.js and npm to build the UI.") - return False - - print("All required tools found.\n") - return True - - -def _require_flit() -> bool: - if shutil.which("flit") is None: - print("✗ flit CLI not found. Please install it with: pip install flit") - return False - print("✓ flit CLI found.\n") - return True - - -def _install_burr(skip_install: bool) -> bool: - if skip_install: - print("Skipping burr installation as requested.\n") - return True - - print("Installing burr from source...") - try: - subprocess.run( - [sys.executable, "-m", "pip", "install", "-e", "."], - check=True, - cwd=os.getcwd(), - ) - print("✓ Burr installed successfully.\n") - return True - except subprocess.CalledProcessError as exc: - print(f"✗ Error installing burr: {exc}") - return False - - -def _build_ui() -> bool: - print("Building UI assets...") - try: - env = os.environ.copy() - env["BURR_PROJECT_ROOT"] = os.getcwd() - subprocess.run(["burr-admin-build-ui"], check=True, env=env) - print("✓ UI build completed successfully.\n") - return True - except subprocess.CalledProcessError as exc: - print(f"✗ Error building UI: {exc}") - return False - - -def _verify_artifacts() -> bool: - build_dir = "burr/tracking/server/build" - print(f"Verifying build output in {build_dir}...") - - if not os.path.exists(build_dir): - print(f"Build directory missing, creating placeholder at {build_dir}...") - os.makedirs(build_dir, exist_ok=True) - - if not os.listdir(build_dir): - print(f"✗ Build directory is empty: {build_dir}") - return False - - print("✓ Build output verified.\n") - return True - - -def _clean_dist(): - if os.path.exists("dist"): - print("Cleaning dist/ directory...") - shutil.rmtree("dist") - print("✓ dist/ directory cleaned.\n") - - -def _clean_ui_build(): - """Remove any existing UI build directory to ensure clean state.""" - ui_build_dir = "burr/tracking/server/build" - if os.path.exists(ui_build_dir): - print(f"Cleaning existing UI build directory: {ui_build_dir}") - shutil.rmtree(ui_build_dir) - print("✓ UI build directory cleaned.\n") - - -def _replace_symlinks_with_copies(): - """ - Replace symlinked example files/directories with actual copies before building. - Returns a dict mapping paths to their symlink targets (if they were symlinks), - so they can be restored later. - """ - # Files and directories from pyproject.toml lines 266-270 that might be symlinks - example_paths = [ - "examples/__init__.py", - "examples/email-assistant", - "examples/multi-modal-chatbot", - "examples/streaming-fastapi", - "examples/deep-researcher", - ] - - symlink_info = {} # Maps path -> (was_symlink: bool, symlink_target: str or None, is_dir: bool) - - for path in example_paths: - if not os.path.exists(path): - continue - - if os.path.islink(path): - # It's a symlink - we need to replace it with a copy - # Store the original symlink target as read (may be relative) - original_symlink_target = os.readlink(path) - - # Resolve relative symlink targets to absolute paths for copying - if os.path.isabs(original_symlink_target): - resolved_target = original_symlink_target - else: - # Resolve relative to the directory containing the symlink - symlink_dir = os.path.dirname(os.path.abspath(path)) - resolved_target = os.path.join(symlink_dir, original_symlink_target) - resolved_target = os.path.normpath(resolved_target) - - print(f"Found symlink: {path} -> {original_symlink_target}") - print(" Replacing with copy...") - - # Verify the symlink target exists - if not os.path.exists(resolved_target): - print(f" ✗ Warning: Symlink target does not exist: {resolved_target}") - symlink_info[path] = (False, None, False) - continue - - is_directory = os.path.isdir(resolved_target) - - # Remove the symlink - os.remove(path) - - if is_directory: - # For directories, use copytree - shutil.copytree(resolved_target, path, dirs_exist_ok=True) - else: - # For files, use copy2 to preserve metadata - shutil.copy2(resolved_target, path) - - # Store the original symlink target (as it was originally read) - symlink_info[path] = (True, original_symlink_target, is_directory) - print(" ✓ Replaced symlink with copy.\n") - else: - # Not a symlink, nothing to do - symlink_info[path] = (False, None, False) - - return symlink_info - - -def _restore_symlinks(symlink_info): - """ - Restore symlinks that were replaced with copies. - symlink_info: dict from _replace_symlinks_with_copies() - """ - for path, (was_symlink, symlink_target, is_directory) in symlink_info.items(): - if was_symlink and symlink_target: - if os.path.exists(path) and not os.path.islink(path): - # Remove the copy and restore the symlink - print(f"Restoring symlink: {path} -> {symlink_target}") - try: - if is_directory: - shutil.rmtree(path) - else: - os.remove(path) - os.symlink(symlink_target, path) - print(" ✓ Symlink restored.\n") - except Exception as exc: - print(f" ✗ Error restoring symlink: {exc}\n") - - -def _copy_examples_to_burr(): - """ - Copy example directories into burr/examples/ so they're included in the wheel. - Flit wheels only package what's in the burr/ module directory. - If burr/examples exists (as symlink or directory), remove it first to ensure - we copy actual files, not symlinks. - Returns tuple: (copied: bool, was_symlink: bool, symlink_target: str or None) - """ - burr_examples_dir = "burr/examples" - source_examples_dir = "examples" - - if not os.path.exists(source_examples_dir): - print(f"Warning: {source_examples_dir} does not exist. Skipping copy.\n") - return (False, False, None) - - # Check if burr/examples exists and if it's a symlink - we'll need to restore it later - was_symlink = False - symlink_target = None - if os.path.exists(burr_examples_dir): - if os.path.islink(burr_examples_dir): - was_symlink = True - symlink_target = os.readlink(burr_examples_dir) - print(f"Removing existing {burr_examples_dir} symlink (-> {symlink_target})...") - os.remove(burr_examples_dir) - else: - print(f"Removing existing {burr_examples_dir} directory...") - shutil.rmtree(burr_examples_dir) - print(f" ✓ Removed existing {burr_examples_dir}\n") - - print( - f"Copying examples from {source_examples_dir} to {burr_examples_dir} for wheel packaging..." - ) - - # Create burr/examples directory - os.makedirs(burr_examples_dir, exist_ok=True) - - # Copy __init__.py if it exists - init_file = os.path.join(source_examples_dir, "__init__.py") - if os.path.exists(init_file): - shutil.copy2(init_file, os.path.join(burr_examples_dir, "__init__.py")) - - # Copy the specific example directories from pyproject.toml - example_dirs = [ - "email-assistant", - "multi-modal-chatbot", - "streaming-fastapi", - "deep-researcher", - ] - - for example_dir in example_dirs: - source_path = os.path.join(source_examples_dir, example_dir) - dest_path = os.path.join(burr_examples_dir, example_dir) - - if os.path.exists(source_path): - if os.path.isdir(source_path): - shutil.copytree(source_path, dest_path, dirs_exist_ok=True) - else: - shutil.copy2(source_path, dest_path) - print(f" ✓ Copied {example_dir}") - - print(f"✓ Examples copied to {burr_examples_dir}.\n") - return (True, was_symlink, symlink_target) - - -def _remove_examples_from_burr(was_symlink=False, symlink_target=None): - """ - Remove the examples directory from burr/ after building the wheel. - If it was originally a symlink, restore it. - """ - burr_examples_dir = "burr/examples" - if os.path.exists(burr_examples_dir): - print(f"Removing {burr_examples_dir} after wheel build...") - shutil.rmtree(burr_examples_dir) - print(f" ✓ Removed {burr_examples_dir}.\n") - - # Restore the original symlink if it existed - if was_symlink and symlink_target: - print(f"Restoring symlink: {burr_examples_dir} -> {symlink_target}") - try: - os.symlink(symlink_target, burr_examples_dir) - print(" ✓ Symlink restored.\n") - except Exception as exc: - print(f" ✗ Error restoring symlink: {exc}\n") - - -def _build_wheel() -> bool: - print("Building wheel distribution with 'flit build --format wheel'...") - - # Replace symlinked directories with copies before building - symlink_info = _replace_symlinks_with_copies() - - # Copy examples into burr/ so they're included in the wheel - examples_copied, examples_was_symlink, examples_symlink_target = _copy_examples_to_burr() - - try: - env = os.environ.copy() - env["FLIT_USE_VCS"] = "0" - subprocess.run(["flit", "build", "--format", "wheel"], check=True, env=env) - print("✓ Wheel build completed successfully.\n") - - # Remove examples from burr/ after successful build (and restore symlink if needed) - if examples_copied: - _remove_examples_from_burr(examples_was_symlink, examples_symlink_target) - - # Restore symlinks after successful build - _restore_symlinks(symlink_info) - return True - except subprocess.CalledProcessError as exc: - print(f"✗ Error building wheel: {exc}") - # Remove examples from burr/ even on error (and restore symlink if needed) - if examples_copied: - _remove_examples_from_burr(examples_was_symlink, examples_symlink_target) - # Restore symlinks even on error - _restore_symlinks(symlink_info) - return False - except Exception as exc: - # Remove examples from burr/ on any other error (and restore symlink if needed) - if examples_copied: - _remove_examples_from_burr(examples_was_symlink, examples_symlink_target) - # Restore symlinks on any other error - print(f"✗ Unexpected error building wheel: {exc}") - _restore_symlinks(symlink_info) - return False - - -def _verify_wheel() -> bool: - print("Verifying wheel output...") - - if not os.path.exists("dist"): - print("✗ dist/ directory not found") - return False - - wheel_files = [f for f in os.listdir("dist") if f.endswith(".whl")] - if not wheel_files: - print("✗ No wheel files found in dist/") - if os.listdir("dist"): - print("Contents of dist/ directory:") - for item in os.listdir("dist"): - print(f" - {item}") - return False - - print(f"✓ Found {len(wheel_files)} wheel file(s):") - for wheel_file in wheel_files: - wheel_path = os.path.join("dist", wheel_file) - size = os.path.getsize(wheel_path) - print(f" - {wheel_file} ({size:,} bytes)") - - print() - return True - - -def create_artifacts(skip_install: bool) -> bool: - if not _ensure_project_root(): - print("Failed to confirm project root.") - return False - if not _check_node_prereqs(): - print("Node/npm prerequisite check failed.") - return False - # Clean any existing UI build to ensure fresh state - _clean_ui_build() - if not _install_burr(skip_install): - print("Installing burr from source failed.") - return False - if not _build_ui(): - print("UI build failed.") - return False - if not _verify_artifacts(): - print("UI artifact verification failed.") - return False - return True - - -def create_wheel(clean: bool) -> bool: - if not _ensure_project_root(): - print("Failed to confirm project root.") - return False - if not _require_flit(): - print("Missing flit CLI.") - return False - if not _verify_artifacts(): - print("Please run the 'artifacts' subcommand first.") - return False - if clean: - _clean_dist() - if not _build_wheel(): - return False - if not _verify_wheel(): - return False - return True - - -def build_all(skip_install: bool, clean: bool) -> bool: - if not create_artifacts(skip_install=skip_install): - return False - if not create_wheel(clean=clean): - return False - return True - - -def main(): - parser = argparse.ArgumentParser( - description="Build artifacts/wheels for Burr using subcommands." - ) - subparsers = parser.add_subparsers(dest="command", required=True) - - artifacts_parser = subparsers.add_parser("artifacts", help="Build UI artifacts only.") - artifacts_parser.add_argument( - "--skip-install", - action="store_true", - help="Skip reinstalling burr when building artifacts", - ) - - wheel_parser = subparsers.add_parser( - "wheel", help="Build wheel distribution (requires artifacts)." - ) - wheel_parser.add_argument( - "--clean", - action="store_true", - help="Clean dist/ directory before building wheel", - ) - - all_parser = subparsers.add_parser("all", help="Build artifacts and wheel in sequence.") - all_parser.add_argument( - "--skip-install", - action="store_true", - help="Skip reinstalling burr when building artifacts", - ) - all_parser.add_argument( - "--clean", - action="store_true", - help="Clean dist/ directory before building wheel", - ) - - args = parser.parse_args() - - print("=" * 80) - print(f"Burr Build Helper - command: {args.command}") - print("=" * 80) - print() - - success = False - if args.command == "artifacts": - success = create_artifacts(skip_install=args.skip_install) - elif args.command == "wheel": - success = create_wheel(clean=args.clean) - elif args.command == "all": - success = build_all(skip_install=args.skip_install, clean=args.clean) - - if success: - print("=" * 80) - print("✅ Build Complete!") - print("=" * 80) - if args.command in {"wheel", "all"}: - print("\nWheel files are in the dist/ directory.") - print("You can now upload to PyPI with:") - print(" twine upload dist/*.whl") - print() - else: - print("\n❌ Build failed.") - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/scripts/release_helper.py b/scripts/release_helper.py deleted file mode 100644 index b08b99ee7..000000000 --- a/scripts/release_helper.py +++ /dev/null @@ -1,451 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import argparse -import glob -import hashlib -import os -import re -import shutil -import subprocess -import sys - -# --- Configuration --- -# You need to fill these in for your project. -# The name of your project's short name (e.g., 'myproject'). -PROJECT_SHORT_NAME = "burr" -# The file where you want to update the version number. -VERSION_FILE = "pyproject.toml" -# A regular expression pattern to find the version string in the VERSION_FILE. -VERSION_PATTERN = r'version\s*=\s*"(\d+\.\d+\.\d+)"' - - -def _fail(message: str) -> None: - print(f"\n❌ {message}") - sys.exit(1) - - -def get_version_from_file(file_path: str) -> str: - """Get the version from a file.""" - with open(file_path, encoding="utf-8") as f: - content = f.read() - match = re.search(VERSION_PATTERN, content) - if match: - version = match.group(1) - return version - raise ValueError(f"Could not find version in {file_path}") - - -def check_prerequisites(): - """Checks for necessary command-line tools and Python modules.""" - print("Checking for required tools...") - required_tools = ["git", "gpg", "svn", "flit"] - for tool in required_tools: - if shutil.which(tool) is None: - _fail( - f"Required tool '{tool}' not found. Please install it and ensure it's in your PATH." - ) - - print("All required tools found.") - - -def update_version(version, _rc_num): - """Updates the version number in the specified file.""" - print(f"Updating version in {VERSION_FILE} to {version}...") - try: - with open(VERSION_FILE, "r", encoding="utf-8") as f: - content = f.read() - # For pyproject.toml, we just update the version string directly - new_version_string = f'version = "{version}"' - new_content = re.sub(VERSION_PATTERN, new_version_string, content) - if new_content == content: - print("Error: Could not find or replace version string. Check your VERSION_PATTERN.") - return False - - with open(VERSION_FILE, "w", encoding="utf-8") as f: - f.write(new_content) - - print("Version updated successfully.") - return True - - except FileNotFoundError: - _fail(f"{VERSION_FILE} not found.") - except (OSError, re.error) as e: - _fail(f"An error occurred while updating the version: {e}") - - -def sign_artifacts(archive_name: str) -> list[str]: - """Creates signed files for the designated artifact.""" - files = [] - # Sign the tarball with GPG. The user must have a key configured. - try: - subprocess.run( - ["gpg", "--armor", "--output", f"{archive_name}.asc", "--detach-sig", archive_name], - check=True, - ) - files.append(f"{archive_name}.asc") - print(f"Created GPG signature: {archive_name}.asc") - except subprocess.CalledProcessError as e: - _fail(f"Error signing tarball {archive_name}: {e}") - - # Generate SHA512 checksum. - sha512_hash = hashlib.sha512() - with open(archive_name, "rb") as f: - while True: - data = f.read(65536) - if not data: - break - sha512_hash.update(data) - - with open(f"{archive_name}.sha512", "w", encoding="utf-8") as f: - f.write(f"{sha512_hash.hexdigest()}\n") - print(f"Created SHA512 checksum: {archive_name}.sha512") - files.append(f"{archive_name}.sha512") - return files - - -def create_release_artifacts(version, build_wheel=False) -> list[str]: - """Creates the source tarball, GPG signatures, and checksums using flit. - - Args: - version: The version string for the release - build_wheel: If True, also build and sign a wheel distribution - """ - print("\n[Step 1/1] Creating source release artifacts with 'flit build'...") - - # Clean the dist directory before building. - if os.path.exists("dist"): - shutil.rmtree("dist") - # Ensure no pre-built UI assets slip into the source package. - ui_build_dir = os.path.join("burr", "tracking", "server", "build") - if os.path.exists(ui_build_dir): - print("Removing previously built UI artifacts...") - shutil.rmtree(ui_build_dir) - - # Warn if git working tree is dirty/untracked - try: - dirty = ( - subprocess.check_output(["git", "status", "--porcelain"], stderr=subprocess.DEVNULL) - .decode() - .strip() - ) - if dirty: - print( - "⚠️ Detected untracked or modified files. flit may refuse to build; " - "consider committing/stashing or verify FLIT_USE_VCS=0." - ) - print(" Git status summary:") - for line in dirty.splitlines(): - print(f" {line}") - except subprocess.CalledProcessError: - pass - - # Use flit to create the source distribution. - try: - env = os.environ.copy() - env["FLIT_USE_VCS"] = "0" - subprocess.run(["flit", "build", "--format", "sdist"], check=True, env=env) - print("✓ flit sdist created successfully.") - except subprocess.CalledProcessError as e: - _fail(f"Error creating source distribution: {e}") - - # Find the created tarball in the dist directory. - # Note: flit normalizes hyphens to underscores in filenames - expected_tar_ball = f"dist/apache_burr-{version.lower()}.tar.gz" - tarball_path = glob.glob(expected_tar_ball) - - if not tarball_path: - details = [] - if os.path.exists("dist"): - details.append("Contents of 'dist':") - for item in os.listdir("dist"): - details.append(f"- {item}") - else: - details.append("'dist' directory not found.") - _fail( - "Could not find the generated source tarball in the 'dist' directory.\n" - + "\n".join(details) - ) - - # Rename the tarball to apache-burr-{version.lower()}-incubating.tar.gz - apache_tar_ball = f"dist/apache-burr-{version.lower()}-incubating.tar.gz" - shutil.move(tarball_path[0], apache_tar_ball) - print(f"✓ Created source tarball: {apache_tar_ball}") - - # Sign the Apache tarball - signed_files = sign_artifacts(apache_tar_ball) - all_files = [apache_tar_ball] + signed_files - - # Optionally build the wheel (without built UI artifacts) - if build_wheel: - print("\n[Step 2/2] Creating wheel distribution with 'flit build --format wheel'...") - try: - env = os.environ.copy() - env["FLIT_USE_VCS"] = "0" - subprocess.run(["flit", "build", "--format", "wheel"], check=True, env=env) - print("✓ flit wheel created successfully.") - except subprocess.CalledProcessError as e: - _fail(f"Error creating wheel distribution: {e}") - - # Find the created wheel in the dist directory. - # Note: flit normalizes hyphens to underscores in filenames - expected_wheel = f"dist/apache_burr-{version.lower()}-*.whl" - wheel_path = glob.glob(expected_wheel) - - if not wheel_path: - details = [] - if os.path.exists("dist"): - details.append("Contents of 'dist':") - for item in os.listdir("dist"): - details.append(f"- {item}") - else: - details.append("'dist' directory not found.") - _fail( - "Could not find the generated wheel in the 'dist' directory.\n" + "\n".join(details) - ) - - # Rename the wheel to apache-burr-{version.lower()}-incubating-{rest}.whl - # Extract the wheel tags (e.g., py3-none-any.whl) - original_wheel = os.path.basename(wheel_path[0]) - # Pattern: apache_burr-{version}-{tags}.whl -> apache-burr-{version}-incubating-{tags}.whl - wheel_tags = original_wheel.replace(f"apache_burr-{version.lower()}-", "") - apache_wheel = f"dist/apache-burr-{version.lower()}-incubating-{wheel_tags}" - shutil.move(wheel_path[0], apache_wheel) - print(f"✓ Created wheel: {apache_wheel}") - - # Sign the Apache wheel - wheel_signed_files = sign_artifacts(apache_wheel) - all_files.extend([apache_wheel] + wheel_signed_files) - - return all_files - - -def svn_upload(version, rc_num, archive_files, apache_id): - """Uploads the artifacts to the ASF dev distribution repository.""" - print("Uploading artifacts to ASF SVN...") - svn_path = f"https://dist.apache.org/repos/dist/dev/incubator/{PROJECT_SHORT_NAME}/{version}-incubating-RC{rc_num}" - - try: - # Create a new directory for the release candidate. - subprocess.run( - [ - "svn", - "mkdir", - "--parents", - "-m", - f"Creating directory for {version}-incubating-RC{rc_num}", - svn_path, - ], - check=True, - ) - - # Get the files to import (tarball, asc, sha512). - files_to_import = archive_files - - # Use svn import for the new directory. - for file_path in files_to_import: - subprocess.run( - [ - "svn", - "import", - file_path, - f"{svn_path}/{os.path.basename(file_path)}", - "-m", - f"Adding {os.path.basename(file_path)}", - "--username", - apache_id, - ], - check=True, - ) - - print(f"Artifacts successfully uploaded to: {svn_path}") - return svn_path - - except subprocess.CalledProcessError as e: - print(f"Error during SVN upload: {e}") - print("Make sure you have svn access configured for your Apache ID.") - return None - - -def generate_email_template(version, rc_num, svn_url): - """Generates the content for the [VOTE] email.""" - print("Generating email template...") - version_with_incubating = f"{version}-incubating" - tag = f"v{version}" - - email_content = f"""[VOTE] Release Apache {PROJECT_SHORT_NAME} {version_with_incubating} (release candidate {rc_num}) - -Hi all, - -This is a call for a vote on releasing Apache {PROJECT_SHORT_NAME} {version_with_incubating}, -release candidate {rc_num}. - -This release includes the following changes (see CHANGELOG for details): -- [List key changes here] - -The artifacts for this release candidate can be found at: -{svn_url} - -The Git tag to be voted upon is: -{tag} - -The release hash is: -[Insert git commit hash here] - - -Release artifacts are signed with the following key: -[Insert your GPG key ID here] -The KEYS file is available at: -https://downloads.apache.org/incubator/{PROJECT_SHORT_NAME}/KEYS - -Please download, verify, and test the release candidate. - -For testing use your best judgement. Any of the following will suffice - -1. Build/run the UI following the instructions in scripts/README.md -2. Run the tests in tests/ -3. Import into a jupyter notebook and play around - -The vote will run for a minimum of 72 hours. -Please vote: - -[ ] +1 Release this package as Apache {PROJECT_SHORT_NAME} {version_with_incubating} -[ ] +0 No opinion -[ ] -1 Do not release this package because... (Please provide a reason) - -Checklist for reference: -[ ] Download links are valid. -[ ] Checksums and signatures. -[ ] LICENSE/NOTICE files exist -[ ] LICENSE/NOTICE files exist in convenience packages -[ ] No unexpected binary files in source -[ ] No unexpected binary files in convenience packages -[ ] All source files have ASF headers -[ ] Can compile from source -[ ] Build script recreates convenience packages (contents need to match) - -On behalf of the Apache {PROJECT_SHORT_NAME} PPMC, -[Your Name] -""" - print("\n" + "=" * 80) - print("EMAIL TEMPLATE (COPY AND PASTE TO YOUR MAILING LIST)") - print("=" * 80) - print(email_content) - print("=" * 80) - - -def main(): - """ - ### How to Use the Updated Script - - 1. **Install flit**: - ```bash - pip install flit - ``` - 2. **Configure the Script**: Open `apache_release_helper.py` in a text editor and update the three variables at the top of the file with your project's details: - * `PROJECT_SHORT_NAME` - * `VERSION_FILE` and `VERSION_PATTERN` - 3. **Prerequisites**: - * You must have `git`, `gpg`, `svn`, and `flit` installed. - * Your GPG key and SVN access must be configured for your Apache ID. - 4. **Run the Script**: - Open your terminal, navigate to the root of your project directory, and run the script with the desired version, release candidate number, and Apache ID. - - - python apache_release_helper.py 1.2.3 0 your_apache_id - """ - parser = argparse.ArgumentParser(description="Automates parts of the Apache release process.") - parser.add_argument("version", help="The new release version (e.g., '1.0.0').") - parser.add_argument("rc_num", help="The release candidate number (e.g., '0' for RC0).") - parser.add_argument("apache_id", help="Your apache user ID.") - parser.add_argument( - "--dry-run", - action="store_true", - help="Run in dry-run mode (skip git tag creation and SVN upload)", - ) - parser.add_argument( - "--build-wheel", - action="store_true", - help="Also build and sign a wheel distribution (in addition to the source tarball)", - ) - args = parser.parse_args() - - version = args.version - rc_num = args.rc_num - apache_id = args.apache_id - dry_run = args.dry_run - build_wheel = args.build_wheel - - if dry_run: - print("\n*** DRY RUN MODE - No git tags or SVN uploads will be performed ***\n") - - check_prerequisites() - - current_version = get_version_from_file(VERSION_FILE) - print(current_version) - if current_version != version: - _fail( - "Version mismatch. Update pyproject.toml to the requested version before running the script." - ) - - tag_name = f"v{version}-incubating-RC{rc_num}" - if dry_run: - print(f"\n[DRY RUN] Would create git tag '{tag_name}'") - else: - print(f"\nChecking for git tag '{tag_name}'...") - try: - # Check if the tag already exists - existing_tag = subprocess.check_output(["git", "tag", "-l", tag_name]).decode().strip() - if existing_tag == tag_name: - print(f"Git tag '{tag_name}' already exists.") - response = ( - input("Do you want to continue without creating a new tag? (y/n): ") - .lower() - .strip() - ) - if response != "y": - print("Aborting.") - sys.exit(1) - else: - # Tag does not exist, create it - print(f"Creating git tag '{tag_name}'...") - subprocess.run(["git", "tag", tag_name], check=True) - print(f"Git tag {tag_name} created.") - except subprocess.CalledProcessError as e: - _fail(f"Error checking or creating Git tag: {e}") - - # Create artifacts - archive_files = create_release_artifacts(version, build_wheel=build_wheel) - - # Upload artifacts - # NOTE: You MUST have your SVN client configured to use your Apache ID and have permissions. - if dry_run: - svn_url = f"https://dist.apache.org/repos/dist/dev/incubator/{PROJECT_SHORT_NAME}/{version}-incubating-RC{rc_num}" - print(f"\n[DRY RUN] Would upload artifacts to: {svn_url}") - else: - svn_url = svn_upload(version, rc_num, archive_files, apache_id) - if not svn_url: - _fail("SVN upload failed.") - - # Generate email - generate_email_template(version, rc_num, svn_url) - - print("\nProcess complete. Please copy the email template to your mailing list.") - - -if __name__ == "__main__": - main() diff --git a/scripts/verify_apache_artifacts.py b/scripts/verify_apache_artifacts.py new file mode 100755 index 000000000..f4861d323 --- /dev/null +++ b/scripts/verify_apache_artifacts.py @@ -0,0 +1,721 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Apache Artifacts Verification Script + +Comprehensive verification tool for Apache release artifacts. +Checks signatures, checksums, licenses, and archive integrity. + +Usage: + # List contents of an artifact + python scripts/verify_apache_artifacts.py list-contents dist/apache-burr-0.41.0.tar.gz + + # Verify signatures and checksums + python scripts/verify_apache_artifacts.py signatures + + # Verify licenses with Apache RAT + python scripts/verify_apache_artifacts.py licenses --rat-jar path/to/apache-rat.jar + + # Verify everything + python scripts/verify_apache_artifacts.py all --rat-jar path/to/apache-rat.jar + + # Specify custom artifacts directory + python scripts/verify_apache_artifacts.py signatures --artifacts-dir /path/to/dist +""" + +import argparse +import hashlib +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import xml.etree.ElementTree as ET +import zipfile + +# Configuration +PROJECT_SHORT_NAME = "burr" + + +def _fail(message: str) -> None: + """Print error message and exit.""" + print(f"\n❌ {message}") + sys.exit(1) + + +def _print_section(title: str) -> None: + """Print formatted section header.""" + print("\n" + "=" * 80) + print(f" {title}") + print("=" * 80 + "\n") + + +# ============================================================================ +# Signature and Checksum Verification +# ============================================================================ + + +def _verify_artifact_signature(artifact_path: str, signature_path: str) -> bool: + """Verify GPG signature of artifact.""" + print(f" Verifying GPG signature: {os.path.basename(signature_path)}") + + if not os.path.exists(signature_path): + print(" ✗ Signature file not found") + return False + + try: + result = subprocess.run( + ["gpg", "--verify", signature_path, artifact_path], + capture_output=True, + check=False, + ) + if result.returncode == 0: + print(" ✓ GPG signature is valid") + return True + else: + print(" ✗ GPG signature verification failed") + if result.stderr: + print(f" Error: {result.stderr.decode()}") + return False + except subprocess.CalledProcessError: + print(" ✗ Error running GPG") + return False + + +def _verify_artifact_checksum(artifact_path: str, checksum_path: str) -> bool: + """Verify SHA512 checksum of artifact.""" + print(f" Verifying SHA512 checksum: {os.path.basename(checksum_path)}") + + if not os.path.exists(checksum_path): + print(" ✗ Checksum file not found") + return False + + # Read expected checksum + with open(checksum_path, "r", encoding="utf-8") as f: + expected_checksum = f.read().strip().split()[0] + + # Calculate actual checksum + sha512_hash = hashlib.sha512() + with open(artifact_path, "rb") as f: + while chunk := f.read(65536): + sha512_hash.update(chunk) + + actual_checksum = sha512_hash.hexdigest() + + if actual_checksum == expected_checksum: + print(" ✓ SHA512 checksum is valid") + return True + else: + print(" ✗ SHA512 checksum mismatch!") + print(f" Expected: {expected_checksum}") + print(f" Actual: {actual_checksum}") + return False + + +def _verify_tar_gz_readable(artifact_path: str) -> bool: + """Verify tar.gz archive can be read and contains files.""" + print(f" Checking archive readability: {os.path.basename(artifact_path)}") + + try: + with tarfile.open(artifact_path, "r:gz") as tar: + members = tar.getmembers() + + if len(members) == 0: + print(" ✗ Archive is empty (no files)") + return False + + print(f" ✓ Archive is readable and contains {len(members)} files") + return True + except tarfile.TarError as e: + print(f" ✗ Archive is corrupted or unreadable: {e}") + return False + except Exception as e: + print(f" ✗ Error reading archive: {e}") + return False + + +def _verify_wheel_readable(wheel_path: str) -> bool: + """Verify wheel can be read and contains expected structure.""" + print(f" Checking wheel readability: {os.path.basename(wheel_path)}") + + try: + with zipfile.ZipFile(wheel_path, "r") as whl: + file_list = whl.namelist() + + if len(file_list) == 0: + print(" ✗ Wheel is empty (no files)") + return False + + # Check for metadata + metadata_files = [f for f in file_list if "METADATA" in f or "WHEEL" in f] + if not metadata_files: + print(" ✗ Wheel missing required metadata files") + return False + + print(f" ✓ Wheel is readable and contains {len(file_list)} files") + return True + except zipfile.BadZipFile: + print(" ✗ Wheel is corrupted or not a valid ZIP file") + return False + except Exception as e: + print(f" ✗ Error reading wheel: {e}") + return False + + +def _verify_artifact_exists(artifact_path: str, min_size: int = 1000) -> bool: + """Verify artifact exists and has reasonable size.""" + if not os.path.exists(artifact_path): + print(f" ✗ Artifact not found: {os.path.basename(artifact_path)}") + return False + + file_size = os.path.getsize(artifact_path) + if file_size < min_size: + print( + f" ✗ Artifact is suspiciously small ({file_size} bytes): {os.path.basename(artifact_path)}" + ) + return False + + print(f" ✓ Artifact exists: {os.path.basename(artifact_path)} ({file_size:,} bytes)") + return True + + +def verify_signatures(artifacts_dir: str) -> bool: + """Verify all signatures and checksums in artifacts directory.""" + _print_section("Verifying Signatures and Checksums") + + if not os.path.exists(artifacts_dir): + _fail(f"Artifacts directory not found: {artifacts_dir}") + + # Find all artifacts (exclude .asc and .sha512 files) + all_files = [ + f for f in os.listdir(artifacts_dir) if os.path.isfile(os.path.join(artifacts_dir, f)) + ] + artifacts = [f for f in all_files if not f.endswith((".asc", ".sha512"))] + + if not artifacts: + print(f"⚠️ No artifacts found in {artifacts_dir}") + return False + + print(f"Found {len(artifacts)} artifact(s) to verify:\n") + + all_valid = True + for artifact_name in artifacts: + artifact_path = os.path.join(artifacts_dir, artifact_name) + + print(f"Verifying: {artifact_name}") + print("-" * 80) + + # Check existence and size + if not _verify_artifact_exists(artifact_path): + all_valid = False + continue + + # Verify signature + signature_path = f"{artifact_path}.asc" + if not _verify_artifact_signature(artifact_path, signature_path): + all_valid = False + + # Verify checksum + checksum_path = f"{artifact_path}.sha512" + if not _verify_artifact_checksum(artifact_path, checksum_path): + all_valid = False + + # Verify archive/wheel structure + if artifact_name.endswith(".tar.gz"): + if not _verify_tar_gz_readable(artifact_path): + all_valid = False + elif artifact_name.endswith(".whl"): + if not _verify_wheel_readable(artifact_path): + all_valid = False + + print() + + return all_valid + + +# ============================================================================ +# License Verification (Apache RAT) +# ============================================================================ + + +def _check_licenses_with_rat( + artifact_path: str, + rat_jar_path: str, + report_name: str, + report_only: bool = False, +) -> bool: + """Run Apache RAT license checker on artifact.""" + print(f"\nRunning Apache RAT on: {os.path.basename(artifact_path)}") + print("-" * 80) + + # Create reports directory + report_dir = "dist" + os.makedirs(report_dir, exist_ok=True) + + rat_report_xml = os.path.join(report_dir, f"rat-report-{report_name}.xml") + rat_report_txt = os.path.join(report_dir, f"rat-report-{report_name}.txt") + + # Extract archive to temp directory + with tempfile.TemporaryDirectory() as temp_dir: + extract_dir = os.path.join(temp_dir, "extracted") + os.makedirs(extract_dir) + + print(" Extracting archive...") + try: + with tarfile.open(artifact_path, "r:gz") as tar: + # Use data filter for Python 3.12+ to avoid deprecation warning + tar.extractall(extract_dir, filter="data") + print(" ✓ Extracted to temp directory") + except Exception as e: + print(f" ✗ Error extracting archive: {e}") + return False + + # Locate .rat-excludes file + rat_excludes = ".rat-excludes" + if not os.path.exists(rat_excludes): + print(f" ⚠️ Warning: {rat_excludes} not found, running without excludes") + rat_excludes = None + + # Run RAT with XML output + print(" Running Apache RAT (XML format for parsing)...") + rat_cmd_xml = [ + "java", + "-jar", + rat_jar_path, + "-x", # XML output + "-d", + extract_dir, + ] + if rat_excludes: + rat_cmd_xml.extend(["-E", rat_excludes]) + + try: + with open(rat_report_xml, "w", encoding="utf-8") as report_file: + result = subprocess.run( + rat_cmd_xml, + stdout=report_file, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + + if result.returncode != 0: + print(f" ⚠️ RAT exited with code {result.returncode}") + + print(f" ✓ RAT XML report: {rat_report_xml}") + except Exception as e: + print(f" ✗ Error running RAT (XML): {e}") + return False + + # Run RAT with plain text output + print(" Running Apache RAT (text format for review)...") + rat_cmd_txt = [ + "java", + "-jar", + rat_jar_path, + "-d", + extract_dir, + ] + if rat_excludes: + rat_cmd_txt.extend(["-E", rat_excludes]) + + try: + with open(rat_report_txt, "w", encoding="utf-8") as report_file: + subprocess.run( + rat_cmd_txt, + stdout=report_file, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + print(f" ✓ RAT text report: {rat_report_txt}") + except Exception as e: + print(f" ⚠️ Warning: Could not generate text report: {e}") + + # Parse XML report + print(" Parsing RAT report...") + try: + tree = ET.parse(rat_report_xml) + root = tree.getroot() + + # Find license issues + unapproved_licenses = [] + unknown_licenses = [] + + for resource in root.findall(".//resource"): + name = resource.get("name", "unknown") + + # Get license approval and family from child elements + license_approval_elem = resource.find("license-approval") + license_family_elem = resource.find("license-family") + + license_approval = ( + license_approval_elem.get("name", "true") + if license_approval_elem is not None + else "true" + ) + license_family = ( + license_family_elem.get("name", "") if license_family_elem is not None else "" + ) + + if license_approval == "false" or license_family == "Unknown license": + if license_family == "Unknown license" or not license_family: + unknown_licenses.append(name) + else: + unapproved_licenses.append(name) + + # Report findings + total_files = len(root.findall(".//resource")) + issues_count = len(unapproved_licenses) + len(unknown_licenses) + + print(f" ✓ Scanned {total_files} files") + print(f" ✓ Found {issues_count} files with license issues") + + if issues_count > 0: + print("\n ⚠️ License Issues Found:") + + if unknown_licenses: + print(f"\n Unknown/Missing Licenses ({len(unknown_licenses)} files):") + for file in unknown_licenses[:10]: + print(f" - {file}") + if len(unknown_licenses) > 10: + print(f" ... and {len(unknown_licenses) - 10} more") + + if unapproved_licenses: + print(f"\n Unapproved Licenses ({len(unapproved_licenses)} files):") + for file in unapproved_licenses[:10]: + print(f" - {file}") + if len(unapproved_licenses) > 10: + print(f" ... and {len(unapproved_licenses) - 10} more") + + print("\n 📄 Reports saved:") + print(f" - {rat_report_xml} (structured)") + print(f" - {rat_report_txt} (human-readable)") + + if report_only: + print("\n ℹ️ Report-only mode: continuing despite license issues") + return True + else: + print("\n ❌ License check failed!") + return False + else: + print(" ✅ All files have approved licenses") + print("\n 📄 Reports saved:") + print(f" - {rat_report_xml} (structured)") + print(f" - {rat_report_txt} (human-readable)") + return True + + except Exception as e: + print(f" ✗ Error parsing RAT report: {e}") + if report_only: + print(" ℹ️ Report-only mode: continuing despite parse error") + return True + return False + + +def verify_licenses(artifacts_dir: str, rat_jar_path: str, report_only: bool = False) -> bool: + """Verify licenses in all tar.gz artifacts using Apache RAT.""" + _print_section("Verifying Licenses with Apache RAT") + + if not os.path.exists(artifacts_dir): + _fail(f"Artifacts directory not found: {artifacts_dir}") + + if not rat_jar_path or not os.path.exists(rat_jar_path): + _fail( + f"Apache RAT JAR not found: {rat_jar_path}\nDownload from: https://creadur.apache.org/rat/download_rat.cgi" + ) + + # Check for java + if shutil.which("java") is None: + _fail("Java not found. Required for Apache RAT.") + + # Find all tar.gz artifacts (not wheels) + all_files = [ + f for f in os.listdir(artifacts_dir) if os.path.isfile(os.path.join(artifacts_dir, f)) + ] + tar_artifacts = [f for f in all_files if f.endswith(".tar.gz")] + + if not tar_artifacts: + print(f"⚠️ No tar.gz artifacts found in {artifacts_dir}") + return False + + print(f"Found {len(tar_artifacts)} tar.gz artifact(s) to check:\n") + + all_valid = True + for artifact_name in tar_artifacts: + artifact_path = os.path.join(artifacts_dir, artifact_name) + + # Generate report name from artifact name + report_name = artifact_name.replace(".tar.gz", "").replace(".", "-") + + if not _check_licenses_with_rat(artifact_path, rat_jar_path, report_name, report_only): + all_valid = False + + return all_valid + + +# ============================================================================ +# List Contents +# ============================================================================ + + +def _list_tar_gz_contents(artifact_path: str) -> None: + """List contents of a tar.gz archive.""" + print(f"\nContents of: {os.path.basename(artifact_path)}") + print("=" * 80) + + try: + with tarfile.open(artifact_path, "r:gz") as tar: + members = tar.getmembers() + + print(f"Total files: {len(members)}\n") + + # Group by type + files = [m for m in members if m.isfile()] + dirs = [m for m in members if m.isdir()] + symlinks = [m for m in members if m.issym() or m.islnk()] + + print(f"Files: {len(files)}, Directories: {len(dirs)}, Symlinks: {len(symlinks)}\n") + + # Show all files + print("Files:\n") + + for member in members: + size = f"{member.size:>12,}" if member.isfile() else " " + prefix = " " + if member.issym() or member.islnk(): + prefix = "→ " + if member.linkname: + print(f"{prefix}{member.name} -> {member.linkname}") + continue + print(f"{prefix}{member.name:<70} {size}") + + except Exception as e: + print(f"Error reading archive: {e}") + + +def _list_wheel_contents(wheel_path: str) -> None: + """List contents of a wheel file.""" + print(f"\nContents of: {os.path.basename(wheel_path)}") + print("=" * 80) + + try: + with zipfile.ZipFile(wheel_path, "r") as whl: + file_list = whl.namelist() + + print(f"Total files: {len(file_list)}\n") + + # Group by directory + top_level_dirs = {} + for file in file_list: + top_dir = file.split("/")[0] + top_level_dirs[top_dir] = top_level_dirs.get(top_dir, 0) + 1 + + print("Top-level structure:") + for dir_name, count in sorted(top_level_dirs.items()): + print(f" {dir_name:<50} ({count} files)") + + # Show all files + print("\nFiles:\n") + + for filename in sorted(file_list): + info = whl.getinfo(filename) + size = f"{info.file_size:>12,}" if not filename.endswith("/") else " " + print(f" {filename:<70} {size}") + + except Exception as e: + print(f"Error reading wheel: {e}") + + +def list_contents(artifact_path: str) -> None: + """List contents of a specific artifact.""" + _print_section("Listing Artifact Contents") + + if not os.path.exists(artifact_path): + _fail(f"Artifact not found: {artifact_path}") + + if artifact_path.endswith(".tar.gz"): + _list_tar_gz_contents(artifact_path) + elif artifact_path.endswith(".whl"): + _list_wheel_contents(artifact_path) + else: + _fail(f"Unsupported file type: {artifact_path}\nSupported: .tar.gz, .whl") + + +# ============================================================================ +# Command Handlers +# ============================================================================ + + +def cmd_signatures(args) -> bool: + """Verify signatures and checksums.""" + return verify_signatures(args.artifacts_dir) + + +def cmd_licenses(args) -> bool: + """Verify licenses with Apache RAT.""" + if not args.rat_jar: + _fail("--rat-jar is required for license verification") + + return verify_licenses(args.artifacts_dir, args.rat_jar, args.report_only) + + +def cmd_all(args) -> bool: + """Verify everything: signatures, checksums, and licenses.""" + _print_section("Complete Apache Artifacts Verification") + + # Step 1: Verify signatures + print("\n[1/2] Verifying signatures and checksums...") + signatures_ok = verify_signatures(args.artifacts_dir) + + # Step 2: Verify licenses + if args.rat_jar: + print("\n[2/2] Verifying licenses with Apache RAT...") + licenses_ok = verify_licenses(args.artifacts_dir, args.rat_jar, args.report_only) + else: + print("\n[2/2] Skipping license verification (no --rat-jar provided)") + licenses_ok = True + + # Summary + _print_section("Verification Summary") + + print("Results:") + print(f" Signatures & Checksums: {'✅ PASS' if signatures_ok else '❌ FAIL'}") + print( + f" License Compliance: {'✅ PASS' if licenses_ok else '❌ FAIL' if args.rat_jar else '⊘ SKIPPED'}" + ) + + return signatures_ok and licenses_ok + + +def cmd_list_contents(args) -> None: + """List contents of a specific artifact.""" + list_contents(args.artifact) + + +# ============================================================================ +# CLI Entry Point +# ============================================================================ + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser( + description="Apache Artifacts Verification Tool", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # List contents of a specific artifact + python scripts/verify_apache_artifacts.py list-contents dist/apache-burr-0.41.0.tar.gz + python scripts/verify_apache_artifacts.py list-contents dist/apache_burr-0.41.0-py3-none-any.whl + + # Verify signatures and checksums only + python scripts/verify_apache_artifacts.py signatures + + # Verify licenses with Apache RAT + python scripts/verify_apache_artifacts.py licenses --rat-jar /path/to/apache-rat.jar + + # Verify everything + python scripts/verify_apache_artifacts.py all --rat-jar /path/to/apache-rat.jar + + # Report-only mode (don't fail on license issues) + python scripts/verify_apache_artifacts.py licenses --rat-jar /path/to/apache-rat.jar --report-only + + # Custom artifacts directory + python scripts/verify_apache_artifacts.py all --artifacts-dir /path/to/artifacts --rat-jar /path/to/rat.jar + """, + ) + + subparsers = parser.add_subparsers(dest="command", required=True) + + # list-contents subcommand + list_parser = subparsers.add_parser( + "list-contents", help="List contents of a specific artifact" + ) + list_parser.add_argument("artifact", help="Path to artifact file (.tar.gz or .whl)") + + # signatures subcommand + sig_parser = subparsers.add_parser( + "signatures", help="Verify GPG signatures and SHA512 checksums" + ) + sig_parser.add_argument( + "--artifacts-dir", default="dist", help="Directory containing artifacts (default: dist)" + ) + + # licenses subcommand + lic_parser = subparsers.add_parser("licenses", help="Verify licenses with Apache RAT") + lic_parser.add_argument( + "--artifacts-dir", default="dist", help="Directory containing artifacts (default: dist)" + ) + lic_parser.add_argument("--rat-jar", required=True, help="Path to Apache RAT JAR file") + lic_parser.add_argument( + "--report-only", action="store_true", help="Generate report but don't fail on issues" + ) + + # all subcommand + all_parser = subparsers.add_parser("all", help="Verify everything (signatures + licenses)") + all_parser.add_argument( + "--artifacts-dir", default="dist", help="Directory containing artifacts (default: dist)" + ) + all_parser.add_argument( + "--rat-jar", help="Path to Apache RAT JAR file (optional for signatures-only)" + ) + all_parser.add_argument( + "--report-only", + action="store_true", + help="Generate report but don't fail on license issues", + ) + + args = parser.parse_args() + + # Dispatch to command handler + success = False + try: + if args.command == "list-contents": + cmd_list_contents(args) + sys.exit(0) + elif args.command == "signatures": + success = cmd_signatures(args) + elif args.command == "licenses": + success = cmd_licenses(args) + elif args.command == "all": + success = cmd_all(args) + else: + _fail(f"Unknown command: {args.command}") + except KeyboardInterrupt: + print("\n\n⚠️ Interrupted by user") + sys.exit(130) + except Exception as e: + print(f"\n❌ Unexpected error: {e}") + import traceback + + traceback.print_exc() + sys.exit(1) + + if success: + print("\n✅ Verification completed successfully!") + sys.exit(0) + else: + print("\n❌ Verification failed.") + sys.exit(1) + + +if __name__ == "__main__": + main()