diff --git a/.github/workflows/deprecated.yml b/.github/workflows/deprecated.yml new file mode 100644 index 00000000..7ce64471 --- /dev/null +++ b/.github/workflows/deprecated.yml @@ -0,0 +1,30 @@ +name: Find deprecated softwares + +on: + schedule: + - cron: '0 20 * * 1' + +jobs: + black: + name: Find deprecated softwares + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: 3.11 + - name: Install toml python lib + run: | + pip3 install toml tomlkit gitpython + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + title: "Flag deprecated apps in the catalog" + commit-message: ":coffin: Flag deprecated apps in the catalog" + body: | + This was done with find_deprecated.py + base: ${{ github.head_ref }} # Creates pull request onto pull request or commit branch + branch: actions/deprecated diff --git a/autoupdate_app_sources/rest_api.py b/autoupdate_app_sources/rest_api.py index 9c33c489..a7f56d41 100644 --- a/autoupdate_app_sources/rest_api.py +++ b/autoupdate_app_sources/rest_api.py @@ -42,6 +42,10 @@ def releases(self) -> list[dict[str, Any]]: """Get a list of releases for project.""" return self.internal_api(f"repos/{self.upstream_repo}/releases?per_page=100") + def archived(self) -> bool: + """Return the archival status for the repository""" + return self.internal_api(f"repos/{self.upstream_repo}")["archived"] + def url_for_ref(self, ref: str, ref_type: RefType) -> str: """Get a URL for a ref.""" if ref_type == RefType.tags or ref_type == RefType.releases: @@ -62,6 +66,7 @@ def changelog_for_ref(self, new_ref: str, old_ref: str, ref_type: RefType) -> st class GitlabAPI: def __init__(self, upstream: str): # Find gitlab api root... + upstream = upstream.rstrip("/") self.forge_root = self.get_forge_root(upstream).rstrip("/") self.project_path = upstream.replace(self.forge_root, "").lstrip("/") self.project_id = self.find_project_id(self.project_path) @@ -145,6 +150,10 @@ def releases(self) -> list[dict[str, Any]]: return retval + def archived(self) -> bool: + """Return the archival status for the repository""" + return self.internal_api(f"projects/{self.project_id}").get("archived", False) + def url_for_ref(self, ref: str, _: RefType) -> str: name = self.project_path.split("/")[-1] clean_ref = ref.replace("/", "-") @@ -196,6 +205,10 @@ def releases(self) -> list[dict[str, Any]]: """Get a list of releases for project.""" return self.internal_api(f"repos/{self.project_path}/releases") + def archived(self) -> bool: + """Return the archival status for the repository""" + return self.internal_api(f"repos/{self.project_path}")["archived"] + def url_for_ref(self, ref: str, _: RefType) -> str: """Get a URL for a ref.""" return f"{self.forge_root}/{self.project_path}/archive/{ref}.tar.gz" diff --git a/find_deprecated.py b/find_deprecated.py new file mode 100755 index 00000000..cb302f64 --- /dev/null +++ b/find_deprecated.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 + +import traceback +import argparse +import tomlkit +import multiprocessing +import datetime +import json +import sys +from functools import cache +import logging +from pathlib import Path +from typing import Optional + +import toml +import tqdm +import github + +# add apps/tools to sys.path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from appslib.utils import REPO_APPS_ROOT, get_catalog # noqa: E402 pylint: disable=import-error,wrong-import-position +from app_caches import app_cache_folder # noqa: E402 pylint: disable=import-error,wrong-import-position +from autoupdate_app_sources.rest_api import GithubAPI, GitlabAPI, GiteaForgejoAPI, RefType # noqa: E402,E501 pylint: disable=import-error,wrong-import-position + + +@cache +def get_github() -> tuple[Optional[tuple[str, str]], Optional[github.Github], Optional[github.InputGitAuthor]]: + try: + github_login = (REPO_APPS_ROOT / ".github_login").open("r", encoding="utf-8").read().strip() + github_token = (REPO_APPS_ROOT / ".github_token").open("r", encoding="utf-8").read().strip() + github_email = (REPO_APPS_ROOT / ".github_email").open("r", encoding="utf-8").read().strip() + + auth = (github_login, github_token) + github_api = github.Github(github_token) + author = github.InputGitAuthor(github_login, github_email) + return auth, github_api, author + except Exception as e: + logging.warning(f"Could not get github: {e}") + return None, None, None + + +def upstream_last_update_ago(app: str) -> tuple[str, int | None]: + manifest_toml = app_cache_folder(app) / "manifest.toml" + manifest_json = app_cache_folder(app) / "manifest.json" + + if manifest_toml.exists(): + manifest = toml.load(manifest_toml.open("r", encoding="utf-8")) + upstream = manifest.get("upstream", {}).get("code") + + elif manifest_json.exists(): + manifest = json.load(manifest_json.open("r", encoding="utf-8")) + upstream = manifest.get("upstream", {}).get("code") + else: + raise RuntimeError(f"App {app} doesn't have a manifest!") + + if upstream is None: + raise RuntimeError(f"App {app} doesn't have an upstream code link!") + + api = None + try: + if upstream.startswith("https://github.com/"): + try: + api = GithubAPI(upstream, auth=get_github()[0]) + except AssertionError as e: + logging.error(f"Exception while handling {app}: {e}") + return app, None + + if upstream.startswith("https://gitlab.") or upstream.startswith("https://framagit.org"): + api = GitlabAPI(upstream) + + if upstream.startswith("https://codeberg.org"): + api = GiteaForgejoAPI(upstream) + + if not api: + autoupdate = manifest.get("resources", {}).get("sources", {}).get("main", {}).get("autoupdate") + if autoupdate: + strat = autoupdate["strategy"] + if "gitea" in strat or "forgejo" in strat: + api = GiteaForgejoAPI(upstream) + + if api: + if api.archived(): + # A stupid value that we know to be higher than the trigger value + return app, 1000 + + last_commit = api.commits()[0] + date = last_commit["commit"]["author"]["date"] + date = datetime.datetime.fromisoformat(date) + ago: datetime.timedelta = datetime.datetime.now() - date.replace(tzinfo=None) + return app, ago.days + except Exception: + logging.error(f"Exception while handling {app}", traceback.format_exc()) + raise + + raise RuntimeError(f"App {app} not handled (not github, gitlab or gitea with autoupdate). Upstream is {upstream}") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("apps", nargs="*", type=str, + help="If not passed, the script will run on the catalog. Github keys required.") + parser.add_argument("-j", "--processes", type=int, default=multiprocessing.cpu_count()) + args = parser.parse_args() + + apps_dict = get_catalog() + if args.apps: + apps_dict = {app: info for app, info in apps_dict.items() if app in args.apps} + + deprecated: list[str] = [] + not_deprecated: list[str] = [] + # for app, info in apps_dict.items(): + with multiprocessing.Pool(processes=args.processes) as pool: + tasks = pool.imap_unordered(upstream_last_update_ago, apps_dict.keys()) + + for _ in tqdm.tqdm(range(len(apps_dict)), ascii=" ยท#"): + try: + app, result = next(tasks) + except Exception as e: + print(f"Exception found: {e}") + continue + + if result is None: + continue + + if result > 365: + deprecated.append(app) + else: + not_deprecated.append(app) + + catalog = tomlkit.load(open("apps.toml")) + for app, info in catalog.items(): + antifeatures = info.get("antifeatures", []) + if app in deprecated: + if "deprecated-software" not in antifeatures: + antifeatures.append("deprecated-software") + elif app in not_deprecated: + if "deprecated-software" in antifeatures: + antifeatures.remove("deprecated-software") + else: + continue + # unique the keys + if antifeatures: + info["antifeatures"] = antifeatures + else: + if "antifeatures" in info.keys(): + info.pop("antifeatures") + tomlkit.dump(catalog, open("apps.toml", "w")) + + +if __name__ == "__main__": + main()