From deb24f3236b41883090c3b9c627de257f681001b Mon Sep 17 00:00:00 2001 From: Johannes Koester Date: Mon, 6 Oct 2025 16:55:30 +0200 Subject: [PATCH 1/6] fixes --- snakedeploy/client.py | 11 +-- snakedeploy/snakemake_wrappers.py | 143 ++++++++++++++++++++++-------- 2 files changed, 111 insertions(+), 43 deletions(-) diff --git a/snakedeploy/client.py b/snakedeploy/client.py index ff6a461..c90eb3f 100644 --- a/snakedeploy/client.py +++ b/snakedeploy/client.py @@ -217,17 +217,12 @@ def get_parser(): update_snakemake_wrappers = subparsers.add_parser( "update-snakemake-wrappers", - help="Update all snakemake wrappers in given Snakefiles.", - description="Update all snakemake wrappers in given Snakefiles.", + help="Update all snakemake wrappers in given Snakefiles to their latest versions.", + description="Update all snakemake wrappers in given Snakefiles to their latest versions.", ) update_snakemake_wrappers.add_argument( "snakefiles", nargs="+", help="Paths to Snakefiles which should be updated." ) - update_snakemake_wrappers.add_argument( - "--git-ref", - help="Git reference to use for updating the wrappers (e.g. a snakemake-wrapper release). " - "If nothing specified, the latest release will be used.", - ) scaffold_snakemake_plugin = subparsers.add_parser( "scaffold-snakemake-plugin", @@ -317,7 +312,7 @@ def help(return_code=0): warn_on_error=args.warn_on_error, ) elif args.subcommand == "update-snakemake-wrappers": - update_snakemake_wrappers(args.snakefiles, git_ref=args.git_ref) + update_snakemake_wrappers(args.snakefiles) elif args.subcommand == "scaffold-snakemake-plugin": scaffold_plugin(args.plugin_type) except UserError as e: diff --git a/snakedeploy/snakemake_wrappers.py b/snakedeploy/snakemake_wrappers.py index 1eb9a72..2ae3fdc 100644 --- a/snakedeploy/snakemake_wrappers.py +++ b/snakedeploy/snakemake_wrappers.py @@ -1,44 +1,117 @@ +from pathlib import Path import re -from typing import List +import tempfile +from typing import Iterable, List from urllib.parse import urlparse +import subprocess as sp from snakedeploy.logger import logger from github import Github -def update_snakemake_wrappers(snakefiles: List[str], git_ref: str): - """Set all snakemake wrappers to the given git ref (e.g. tag or branch).""" +def get_latest_git_tag(path: Path, repo: Path) -> str | None: + """Get the latest git tag of any file in the given directory or below. + Thereby ignore later git tags outside of the given directory. + """ + + # get the latest git commit that changed the given dir: + commit = ( + sp.run( + ["git", "rev-list", "-1", "HEAD", "--", str(path)], + stdout=sp.PIPE, + cwd=repo, + check=True, + ) + .stdout.decode() + .strip() + ) + # get the first git tag that includes this commit: + tags = ( + sp.run( + ["git", "tag", "--sort", "creatordate", "--contains", commit], + check=True, + cwd=repo, + stdout=sp.PIPE, + ) + .stdout.decode() + .strip() + .splitlines() + ) + if not tags: + return None + else: + return tags[0] - if git_ref is None: - logger.info("Obtaining latest release of snakemake-wrappers...") - github = Github() - repo = github.get_repo("snakemake/snakemake-wrappers") - releases = repo.get_releases() - git_ref = releases[0].tag_name - - for snakefile in snakefiles: - with open(snakefile, "r") as infile: - snakefile_content = infile.read() - - def update_spec(matchobj): - spec = matchobj.group("spec") - url = urlparse(spec) - if not url.scheme: - old_git_ref, rest = spec.split("/", 1) - return ( - matchobj.group("def") - + matchobj.group("quote") - + f"{git_ref}/{rest}" - + matchobj.group("quote") - ) - else: - return matchobj.group() - - logger.info(f"Updating snakemake-wrappers in {snakefile} to {git_ref}...") - snakefile_content = re.sub( - "(?Pwrapper:\\n?\\s*)(?P['\"])(?P.+)(?P=quote)", - update_spec, - snakefile_content, + +def get_sparse_checkout_patterns() -> Iterable[str]: + for wrapper_pattern in ("*", "*/*"): + for filetype in ("wrapper.*", "environment.yaml"): + yield f"/*/{wrapper_pattern}/{filetype}" + yield "/meta/*/*/test/Snakefile" + + +class WrapperRepo: + def __init__(self): + self.tmpdir = tempfile.TemporaryDirectory() + logger.info("Cloning snakemake-wrappers repository...") + sp.run( + ["git", "clone", "--filter=blob:none", "--no-checkout", "https://github.com/snakemake/snakemake-wrappers.git", "."], + cwd=self.tmpdir.name, + check=True, ) - with open(snakefile, "w") as outfile: - outfile.write(snakefile_content) + sp.run(["git", "config", "core.sparseCheckoutCone", "false"], cwd=self.tmpdir.name, check=True) + sp.run(["git", "sparse-checkout", "disable"], cwd=self.tmpdir.name, check=True) + sp.run(["git", "sparse-checkout", "set", "--no-cone"] + list(get_sparse_checkout_patterns()), cwd=self.tmpdir.name, check=True) + sp.run(["git", "read-tree", "-mu", "HEAD"], cwd=self.tmpdir.name, check=True) + self.repo_dir = Path(self.tmpdir.name) + + def get_wrapper_version(self, spec: str) -> str | None: + if not (self.repo_dir / spec).exists(): + return None + return get_latest_git_tag(Path(spec), self.repo_dir) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.tmpdir.cleanup() + + +def update_snakemake_wrappers(snakefiles: List[str]): + """Set all snakemake wrappers to the given git ref (e.g. tag or branch).""" + + with WrapperRepo() as wrapper_repo: + for snakefile in snakefiles: + with open(snakefile, "r") as infile: + snakefile_content = infile.read() + + def update_spec(matchobj): + spec = matchobj.group("spec") + url = urlparse(spec) + if not url.scheme: + old_git_ref, rest = spec.split("/", 1) + git_ref = wrapper_repo.get_wrapper_version(rest) + if git_ref is None: + logger.warning(f"Could not determine latest version of wrapper '{rest}'. Leaving unchanged.") + git_ref = old_git_ref + elif git_ref != old_git_ref: + logger.info(f"Updated wrapper '{rest}' from {old_git_ref} to {git_ref}.") + else: + logger.info(f"Wrapper '{rest}' is already at latest version {git_ref}.") + return ( + matchobj.group("def") + + matchobj.group("quote") + + f"{git_ref}/{rest}" + + matchobj.group("quote") + ) + else: + return matchobj.group() + + logger.info(f"Updating snakemake-wrappers and meta-wrappers in {snakefile}...") + snakefile_content = re.sub( + "(?P(meta_)?wrapper:\\n?\\s*)(?P['\"])(?P.+)(?P=quote)", + update_spec, + snakefile_content, + ) + with open(snakefile, "w") as outfile: + outfile.write(snakefile_content) From e129cdda6303cda47d55a7555475ae8106e4c35b Mon Sep 17 00:00:00 2001 From: Johannes Koester Date: Mon, 6 Oct 2025 16:57:18 +0200 Subject: [PATCH 2/6] fmt --- snakedeploy/snakemake_wrappers.py | 38 +++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/snakedeploy/snakemake_wrappers.py b/snakedeploy/snakemake_wrappers.py index 2ae3fdc..357073c 100644 --- a/snakedeploy/snakemake_wrappers.py +++ b/snakedeploy/snakemake_wrappers.py @@ -55,13 +55,29 @@ def __init__(self): self.tmpdir = tempfile.TemporaryDirectory() logger.info("Cloning snakemake-wrappers repository...") sp.run( - ["git", "clone", "--filter=blob:none", "--no-checkout", "https://github.com/snakemake/snakemake-wrappers.git", "."], + [ + "git", + "clone", + "--filter=blob:none", + "--no-checkout", + "https://github.com/snakemake/snakemake-wrappers.git", + ".", + ], + cwd=self.tmpdir.name, + check=True, + ) + sp.run( + ["git", "config", "core.sparseCheckoutCone", "false"], cwd=self.tmpdir.name, check=True, ) - sp.run(["git", "config", "core.sparseCheckoutCone", "false"], cwd=self.tmpdir.name, check=True) sp.run(["git", "sparse-checkout", "disable"], cwd=self.tmpdir.name, check=True) - sp.run(["git", "sparse-checkout", "set", "--no-cone"] + list(get_sparse_checkout_patterns()), cwd=self.tmpdir.name, check=True) + sp.run( + ["git", "sparse-checkout", "set", "--no-cone"] + + list(get_sparse_checkout_patterns()), + cwd=self.tmpdir.name, + check=True, + ) sp.run(["git", "read-tree", "-mu", "HEAD"], cwd=self.tmpdir.name, check=True) self.repo_dir = Path(self.tmpdir.name) @@ -92,12 +108,18 @@ def update_spec(matchobj): old_git_ref, rest = spec.split("/", 1) git_ref = wrapper_repo.get_wrapper_version(rest) if git_ref is None: - logger.warning(f"Could not determine latest version of wrapper '{rest}'. Leaving unchanged.") + logger.warning( + f"Could not determine latest version of wrapper '{rest}'. Leaving unchanged." + ) git_ref = old_git_ref elif git_ref != old_git_ref: - logger.info(f"Updated wrapper '{rest}' from {old_git_ref} to {git_ref}.") + logger.info( + f"Updated wrapper '{rest}' from {old_git_ref} to {git_ref}." + ) else: - logger.info(f"Wrapper '{rest}' is already at latest version {git_ref}.") + logger.info( + f"Wrapper '{rest}' is already at latest version {git_ref}." + ) return ( matchobj.group("def") + matchobj.group("quote") @@ -107,7 +129,9 @@ def update_spec(matchobj): else: return matchobj.group() - logger.info(f"Updating snakemake-wrappers and meta-wrappers in {snakefile}...") + logger.info( + f"Updating snakemake-wrappers and meta-wrappers in {snakefile}..." + ) snakefile_content = re.sub( "(?P(meta_)?wrapper:\\n?\\s*)(?P['\"])(?P.+)(?P=quote)", update_spec, From 407b019074daff234337c7cc624c14d71b26cffd Mon Sep 17 00:00:00 2001 From: Johannes Koester Date: Tue, 7 Oct 2025 08:18:26 +0200 Subject: [PATCH 3/6] fix lint --- snakedeploy/snakemake_wrappers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/snakedeploy/snakemake_wrappers.py b/snakedeploy/snakemake_wrappers.py index 357073c..af04ffe 100644 --- a/snakedeploy/snakemake_wrappers.py +++ b/snakedeploy/snakemake_wrappers.py @@ -6,8 +6,6 @@ import subprocess as sp from snakedeploy.logger import logger -from github import Github - def get_latest_git_tag(path: Path, repo: Path) -> str | None: """Get the latest git tag of any file in the given directory or below. From 720d9b2406811faba06e6badc6427de2efd07a30 Mon Sep 17 00:00:00 2001 From: Johannes Koester Date: Tue, 7 Oct 2025 08:21:50 +0200 Subject: [PATCH 4/6] fix --- tests/test_client.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/test_client.sh b/tests/test_client.sh index 394a4b0..313ce9e 100755 --- a/tests/test_client.sh +++ b/tests/test_client.sh @@ -73,15 +73,10 @@ echo "#### Testing snakedeploy pin-conda-envs" runTest 0 $output snakedeploy pin-conda-envs --conda-frontend conda $tmpdir/test-env.yaml echo -echo "#### Testing snakedeploy update-snakemake-wrappers with given git ref" +echo "#### Testing snakedeploy update-snakemake-wrappers" cp tests/test-snakefile.smk $tmpdir -runTest 0 $output snakedeploy update-snakemake-wrappers --git-ref v1.4.0 $tmpdir/test-snakefile.smk - -echo -echo "#### Testing snakedeploy update-snakemake-wrappers without git ref" runTest 0 $output snakedeploy update-snakemake-wrappers $tmpdir/test-snakefile.smk - echo echo "#### Testing snakedeploy scaffold-snakemake-plugin" workdir=$(pwd) From 6b2e41d384088860d24d60c5934e56283a258320 Mon Sep 17 00:00:00 2001 From: Johannes Koester Date: Tue, 7 Oct 2025 08:31:36 +0200 Subject: [PATCH 5/6] error handling --- snakedeploy/snakemake_wrappers.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/snakedeploy/snakemake_wrappers.py b/snakedeploy/snakemake_wrappers.py index af04ffe..eeb1a4e 100644 --- a/snakedeploy/snakemake_wrappers.py +++ b/snakedeploy/snakemake_wrappers.py @@ -92,7 +92,7 @@ def __exit__(self, exc_type, exc_value, traceback): def update_snakemake_wrappers(snakefiles: List[str]): - """Set all snakemake wrappers to the given git ref (e.g. tag or branch).""" + """Update all snakemake wrappers to their specific latest versions.""" with WrapperRepo() as wrapper_repo: for snakefile in snakefiles: @@ -103,13 +103,22 @@ def update_spec(matchobj): spec = matchobj.group("spec") url = urlparse(spec) if not url.scheme: - old_git_ref, rest = spec.split("/", 1) + parts = spec.split("/", 1) + if len(parts) != 2: + logger.warning( + f"Could not parse wrapper specification '{spec}' " + "(expected version/cat/name or version/cat/name/subcommand). " + "Leaving unchanged." + ) + return matchobj.group() + old_git_ref, rest = parts git_ref = wrapper_repo.get_wrapper_version(rest) if git_ref is None: logger.warning( - f"Could not determine latest version of wrapper '{rest}'. Leaving unchanged." + f"Could not determine latest version of wrapper '{rest}'. " + "Leaving unchanged." ) - git_ref = old_git_ref + return matchobj.group() elif git_ref != old_git_ref: logger.info( f"Updated wrapper '{rest}' from {old_git_ref} to {git_ref}." From d8949e1826801219d7884997614b2d20fe37fe30 Mon Sep 17 00:00:00 2001 From: Johannes Koester Date: Tue, 7 Oct 2025 08:44:36 +0200 Subject: [PATCH 6/6] fix --- snakedeploy/snakemake_wrappers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/snakedeploy/snakemake_wrappers.py b/snakedeploy/snakemake_wrappers.py index eeb1a4e..cc9e25d 100644 --- a/snakedeploy/snakemake_wrappers.py +++ b/snakedeploy/snakemake_wrappers.py @@ -24,6 +24,8 @@ def get_latest_git_tag(path: Path, repo: Path) -> str | None: .strip() ) # get the first git tag that includes this commit: + # Note: We want the EARLIEST tag containing the commit, which represents + # the first version where this wrapper reached its current state tags = ( sp.run( ["git", "tag", "--sort", "creatordate", "--contains", commit],