From 16e873f610ae21bd522ac44eda0b451d1678f7f9 Mon Sep 17 00:00:00 2001 From: Zoran Simic Date: Mon, 10 Mar 2025 16:10:19 -0700 Subject: [PATCH 1/7] More resilient uv auto-upgrade --- .github/workflows/tests.yml | 3 ++- src/pickley/__init__.py | 3 +-- src/pickley/bstrap.py | 49 +++++++++++++++++++++++++++---------- src/pickley/cli.py | 6 +++-- tests/test_bootstrap.py | 4 +-- 5 files changed, 45 insertions(+), 20 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 807b085..bdb21c8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,7 +21,8 @@ jobs: with: python-version: ${{ matrix.python-version }} - - run: pip install -U pip tox + - uses: astral-sh/setup-uv@v5 + - run: uv pip install -U tox-uv - run: tox -e py - uses: codecov/codecov-action@v4 with: diff --git a/src/pickley/__init__.py b/src/pickley/__init__.py index cb40da0..3e56201 100644 --- a/src/pickley/__init__.py +++ b/src/pickley/__init__.py @@ -30,7 +30,6 @@ "version", "version_check_delay", } -KNOWN_ENTRYPOINTS = {bstrap.PICKLEY: (bstrap.PICKLEY,), "tox": ("tox",), "uv": ("uv", "uvx")} PLATFORM = platform.system().lower() @@ -750,7 +749,7 @@ def configured_entrypoints(self, canonical_name) -> Optional[list]: if value: return value - return KNOWN_ENTRYPOINTS.get(canonical_name) + return bstrap.KNOWN_ENTRYPOINTS.get(canonical_name) def require_bootstrap(self): """ diff --git a/src/pickley/bstrap.py b/src/pickley/bstrap.py index 3093b88..033469e 100644 --- a/src/pickley/bstrap.py +++ b/src/pickley/bstrap.py @@ -8,6 +8,7 @@ import argparse import json import os +import re import shutil import subprocess import sys @@ -27,6 +28,7 @@ CURRENT_PYTHON_MM = sys.version_info[:2] UV_CUTOFF = (3, 8) USE_UV = CURRENT_PYTHON_MM >= UV_CUTOFF # Default to `uv` for python versions >= this +KNOWN_ENTRYPOINTS = {PICKLEY: (PICKLEY,), "tox": ("tox",), "uv": ("uv", "uvx")} class _Reporter: @@ -84,9 +86,8 @@ def seed_pickley_config(self, desired_cfg): if not hdry(f"Would seed {msg}"): Reporter.inform(f"Seeding {msg}") ensure_folder(pickley_config.parent) - with open(pickley_config, "wt") as fh: - json.dump(desired_cfg, fh, sort_keys=True, indent=2) - fh.write("\n") + payload = json.dumps(desired_cfg, sort_keys=True, indent=2) + pickley_config.write_text(f"{payload}\n") def bootstrap_pickley(self): """Run `pickley bootstrap` in a temporary venv""" @@ -167,12 +168,36 @@ def __init__(self, pickley_base): def auto_bootstrap_uv(self): self.freshly_bootstrapped = self.bootstrap_reason() if self.freshly_bootstrapped: - Reporter.trace(f"Auto-bootstrapping uv, reason: {self.freshly_bootstrapped}") + Reporter.inform(f"Auto-bootstrapping uv, reason: {self.freshly_bootstrapped}") uv_tmp = self.download_uv() shutil.move(uv_tmp / "uv", self.pickley_base / "uv") shutil.move(uv_tmp / "uvx", self.pickley_base / "uvx") shutil.rmtree(uv_tmp, ignore_errors=True) + # Touch cooldown file to let pickley know no need to check for uv upgrade for a while + cooldown_relative_path = f"{DOT_META}/.cache/uv.cooldown" + cooldown_path = self.pickley_base / cooldown_relative_path + ensure_folder(cooldown_path.parent, dryrun=False) + cooldown_path.write_text("") + Reporter.debug(f"[bootstrap] Touched {cooldown_relative_path}") + + # Create a manifest file to let pickley know uv is installed .cooldown + uv_version = run_program(self.uv_path, "--version", fatal=False, dryrun=False) + if uv_version: + m = re.search(r"(\d+\.\d+\.\d+)", uv_version) + if m: + uv_version = m.group(1) + manifest_relative_path = f"{DOT_META}/.manifest/uv.manifest.json" + manifest_path = self.pickley_base / manifest_relative_path + manifest = { + "entrypoints": KNOWN_ENTRYPOINTS["uv"], + "tracked_settings": {"auto_upgrade_spec": "uv"}, + "version": uv_version, + } + ensure_folder(manifest_path.parent, dryrun=False) + manifest_path.write_text(json.dumps(manifest)) + Reporter.debug(f"[bootstrap] Saved {manifest_relative_path}") + def bootstrap_reason(self): if not self.uv_path.exists(): return "uv not present" @@ -210,8 +235,7 @@ def download_uv(self, version=None, dryrun=False): def built_in_download(target, url): request = Request(url) response = urlopen(request, timeout=10) - with open(target, "wb") as fh: - fh.write(response.read()) + target.write_bytes(response.read()) def clean_env_vars(keys=("__PYVENV_LAUNCHER__", "CLICOLOR_FORCE", "PYTHONPATH")): @@ -320,9 +344,9 @@ def run_program(program, *args, **kwargs): description = " ".join(short(x) for x in args) description = f"{short(program)} {description}" if not hdry(f"Would run: {description}", dryrun=kwargs.pop("dryrun", None)): + Reporter.inform(f"Running: {description}") if fatal: stdout = stderr = None - Reporter.debug(f"Running: {description}") else: stdout = stderr = subprocess.PIPE @@ -350,13 +374,12 @@ def seed_mirror(mirror, path, section): msg = f"{short(config_path)} with {mirror}" if not hdry(f"Would seed {msg}"): Reporter.inform(f"Seeding {msg}") - with open(config_path, "wt") as fh: - if section == "pip" and not mirror.startswith('"'): - # This assumes user passed a reasonable URL as --mirror, no further validation is done - # We only ensure the URL is quoted, as uv.toml requires it - mirror = f'"{mirror}"' + if section == "pip" and not mirror.startswith('"'): + # This assumes user passed a reasonable URL as --mirror, no further validation is done + # We only ensure the URL is quoted, as uv.toml requires it + mirror = f'"{mirror}"' - fh.write(f"[{section}]\nindex-url = {mirror}\n") + config_path.write_text(f"[{section}]\nindex-url = {mirror}\n") except Exception as e: Reporter.inform(f"Seeding {path} failed: {e}") diff --git a/src/pickley/cli.py b/src/pickley/cli.py index 04ef84e..6438164 100644 --- a/src/pickley/cli.py +++ b/src/pickley/cli.py @@ -309,7 +309,9 @@ def auto_upgrade_uv(cooldown_hours=12): settings = TrackedSettings() settings.auto_upgrade_spec = "uv" pspec = PackageSpec("uv", settings=settings) - perform_upgrade(pspec) + + # Automatic background upgrade of `uv` is not treated as fatal, for more resilience + perform_upgrade(pspec, fatal=False) @main.command() @@ -401,7 +403,7 @@ def bootstrap(base_folder, pickley_spec): runez.Anchored.add(CFG.base) setup_audit_log() if bstrap.USE_UV: - auto_upgrade_uv(cooldown_hours=0) + auto_upgrade_uv() bootstrap_marker = CFG.manifests / ".bootstrap.json" if not bootstrap_marker.exists(): diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index b8e1b35..c753bfb 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -23,7 +23,7 @@ def test_bootstrap_command(cli): assert CFG._uv_bootstrap.freshly_bootstrapped == "uv not present" assert "Deleted .pk/uv-0.0.1" in cli.logged assert "Auto-bootstrapping uv, reason: uv not present" in cli.logged - assert "Saved .pk/.manifest/uv.manifest.json" in cli.logged + assert "[bootstrap] Saved .pk/.manifest/uv.manifest.json" in cli.logged assert CFG.program_version(".local/bin/uv") else: @@ -54,7 +54,7 @@ def test_bootstrap_script(cli, monkeypatch): # Verify that uv is seeded even in dryrun mode uv_path = CFG.resolved_path(".local/bin/uv") - assert not runez.is_executable(uv_path) # Not seed by conftest.py (it seeds ./uv) + assert not runez.is_executable(uv_path) # Not seeded by conftest.py (it seeds ./uv) # Simulate bogus mirror, verify that we fail bootstrap in that case cli.run("-nvv", cli.project_folder, "-mhttp://localhost:12345") From d60866f23c0199255ea3940b9442bd8a934bb578 Mon Sep 17 00:00:00 2001 From: Zoran Simic Date: Mon, 10 Mar 2025 16:13:32 -0700 Subject: [PATCH 2/7] Create uv venv in GH actions --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bdb21c8..32873f2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,6 +22,7 @@ jobs: python-version: ${{ matrix.python-version }} - uses: astral-sh/setup-uv@v5 + - run: uv venv - run: uv pip install -U tox-uv - run: tox -e py - uses: codecov/codecov-action@v4 From be6e5e8ecb237f08d5fe13cbf3b87bf2f6b57f43 Mon Sep 17 00:00:00 2001 From: Zoran Simic Date: Mon, 10 Mar 2025 16:20:08 -0700 Subject: [PATCH 3/7] Corrected uv usage --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 32873f2..f0c600a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,7 +24,7 @@ jobs: - uses: astral-sh/setup-uv@v5 - run: uv venv - run: uv pip install -U tox-uv - - run: tox -e py + - run: .venv/bin/tox -e py - uses: codecov/codecov-action@v4 with: files: .tox/test-reports/coverage.xml From 8005f118a7e667050f2edbf2ca82c9d6d188b6ea Mon Sep 17 00:00:00 2001 From: Zoran Simic Date: Mon, 10 Mar 2025 16:42:09 -0700 Subject: [PATCH 4/7] Corrected comment --- src/pickley/bstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pickley/bstrap.py b/src/pickley/bstrap.py index 033469e..759d72b 100644 --- a/src/pickley/bstrap.py +++ b/src/pickley/bstrap.py @@ -181,7 +181,7 @@ def auto_bootstrap_uv(self): cooldown_path.write_text("") Reporter.debug(f"[bootstrap] Touched {cooldown_relative_path}") - # Create a manifest file to let pickley know uv is installed .cooldown + # Let pickley know which version of uv is installed uv_version = run_program(self.uv_path, "--version", fatal=False, dryrun=False) if uv_version: m = re.search(r"(\d+\.\d+\.\d+)", uv_version) From 6650845e802d7172c9ca5996f4e78794eaec06c6 Mon Sep 17 00:00:00 2001 From: Zoran Simic Date: Fri, 21 Mar 2025 14:29:12 -0700 Subject: [PATCH 5/7] Bootstrap uv only once on first run --- src/pickley/cli.py | 19 ++++++++++--------- tests/test_bootstrap.py | 18 +++++++++++------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/pickley/cli.py b/src/pickley/cli.py index 6438164..39ea2e6 100644 --- a/src/pickley/cli.py +++ b/src/pickley/cli.py @@ -303,15 +303,16 @@ def auto_upgrade_uv(cooldown_hours=12): cooldown_hours : int Cooldown period in hours, auto-upgrade won't be attempted any more frequently than that. """ - cooldown_path = CFG.cache / "uv.cooldown" - if not cooldown_hours or not runez.file.is_younger(cooldown_path, cooldown_hours * runez.date.SECONDS_IN_ONE_HOUR): - runez.touch(cooldown_path) - settings = TrackedSettings() - settings.auto_upgrade_spec = "uv" - pspec = PackageSpec("uv", settings=settings) - - # Automatic background upgrade of `uv` is not treated as fatal, for more resilience - perform_upgrade(pspec, fatal=False) + if not CFG.uv_bootstrap.freshly_bootstrapped: + cooldown_path = CFG.cache / "uv.cooldown" + if not cooldown_hours or not runez.file.is_younger(cooldown_path, cooldown_hours * runez.date.SECONDS_IN_ONE_HOUR): + runez.touch(cooldown_path) + settings = TrackedSettings() + settings.auto_upgrade_spec = "uv" + pspec = PackageSpec("uv", settings=settings) + + # Automatic background upgrade of `uv` is not treated as fatal, for more resilience + perform_upgrade(pspec, fatal=False) @main.command() diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index c753bfb..11f8258 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -9,31 +9,35 @@ from pickley.cli import CFG -def test_bootstrap_command(cli): +def test_bootstrap_command(cli, monkeypatch): cli.run("-n", "bootstrap", ".local/bin", cli.project_folder) assert cli.failed assert "Folder .local/bin does not exist" in cli.logged - # Simulate an old uv semi-venv present - runez.touch(".local/bin/.pk/uv-0.0.1/bin/uv", logger=None) + runez.ensure_folder(".local/bin", logger=None) cli.run("--no-color", "-vv", "bootstrap", ".local/bin", cli.project_folder) assert cli.succeeded assert "Saved .pk/.manifest/.bootstrap.json" in cli.logged + assert "Installed pickley v" in cli.logged + assert CFG.program_version(".local/bin/pickley") if bstrap.USE_UV: assert CFG._uv_bootstrap.freshly_bootstrapped == "uv not present" - assert "Deleted .pk/uv-0.0.1" in cli.logged assert "Auto-bootstrapping uv, reason: uv not present" in cli.logged assert "[bootstrap] Saved .pk/.manifest/uv.manifest.json" in cli.logged assert CFG.program_version(".local/bin/uv") + # Simulate an old uv semi-venv present + runez.touch(".local/bin/.pk/uv-0.0.1/bin/uv", logger=None) + monkeypatch.setenv("PICKLEY_ROOT", ".local/bin") + cli.run("-vv", "install", "-f", "uv") + assert cli.succeeded + assert "Deleted .pk/uv-0.0.1" in cli.logged + else: # Verify that no uv bootstrap took place assert "/uv" not in cli.logged assert CFG._uv_bootstrap is None - assert "Installed pickley v" in cli.logged - assert CFG.program_version(".local/bin/pickley") - def test_bootstrap_script(cli, monkeypatch): # Ensure changes to bstrap.py globals are restored From 622480f28ad97ddeed3d672ba554917276f2a687 Mon Sep 17 00:00:00 2001 From: Zoran Simic Date: Fri, 21 Mar 2025 14:45:53 -0700 Subject: [PATCH 6/7] Test with py3.13 as well --- .github/workflows/tests.yml | 4 ++-- setup.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f0c600a..a0b520d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: ['3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 @@ -36,7 +36,7 @@ jobs: strategy: matrix: - python-version: ['3.6', '3.7', '3.8'] + python-version: ['3.6', '3.7', '3.8', '3.9'] steps: - uses: actions/checkout@v4 diff --git a/setup.py b/setup.py index 2006003..d114ebc 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Software Development :: Build Tools", "Topic :: System :: Installation/Setup", From 8b47310ddfaad2d3f4fa13d39f05e7fc6892d3ea Mon Sep 17 00:00:00 2001 From: Zoran Simic Date: Fri, 21 Mar 2025 16:14:19 -0700 Subject: [PATCH 7/7] Ensure exit code != 0 when unknown bundle is referenced --- src/pickley/cli.py | 1 + tests/test_config.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/src/pickley/cli.py b/src/pickley/cli.py index 39ea2e6..33c31bc 100644 --- a/src/pickley/cli.py +++ b/src/pickley/cli.py @@ -529,6 +529,7 @@ def install(force, packages): setup_audit_log() specs = CFG.package_specs(packages, authoritative=True) + runez.abort_if(not specs, f"Can't install '{runez.joined(packages)}', not configured") for pspec in specs: perform_install(pspec) diff --git a/tests/test_config.py b/tests/test_config.py index 263c7cf..d9bc559 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -85,6 +85,10 @@ def test_good_config(cli): assert cli.succeeded assert "Would wrap mgit -> .pk/mgit-1.2.1/bin/mgit" in cli.logged + cli.run("-n install bundle:foo") + assert cli.failed + assert "Can't install 'bundle:foo', not configured" in cli.logged + def test_despecced(): assert CFG.despecced("mgit") == ("mgit", None)