From f29867b826224c317f7447a75e8e4fed117f2ea3 Mon Sep 17 00:00:00 2001 From: ichigirl Date: Wed, 4 Mar 2026 20:49:17 +0100 Subject: [PATCH 1/5] Splitted code for maintenance purpose --- index.html | 2 +- static/api.js | 79 ++++++ static/app.js | 647 +---------------------------------------------- static/lib.js | 54 ++++ static/modal.js | 224 ++++++++++++++++ static/render.js | 152 +++++++++++ static/state.js | 41 +++ static/style.css | 20 +- static/ui.js | 118 +++++++++ 9 files changed, 685 insertions(+), 652 deletions(-) create mode 100644 static/api.js create mode 100644 static/lib.js create mode 100644 static/modal.js create mode 100644 static/render.js create mode 100644 static/state.js create mode 100644 static/ui.js diff --git a/index.html b/index.html index e9afd4e..2d9a6d9 100755 --- a/index.html +++ b/index.html @@ -97,6 +97,6 @@

Top GitHub Repositories

- + diff --git a/static/api.js b/static/api.js new file mode 100644 index 0000000..5fb046a --- /dev/null +++ b/static/api.js @@ -0,0 +1,79 @@ +import { $, cacheKey, fmtInt } from "./lib.js"; +import { hasItems, setItems } from "./render.js"; +import { getState } from "./state.js"; + +const cache = {}; + +export async function fetchJSON(url) { + const r = await fetch(url); + if (!r.ok) throw new Error(`HTTP ${r.status}`); + return r.json(); +} + +export async function loadLeaderboard(useCache = true) { + const state = getState(); + const key = cacheKey(state); + + $("err").style.display = "none"; + syncUI(state); + + if (useCache && cache[key]) { + applyData(cache[key], state); + return; + } + + const content = $("content"); + if (!hasItems()) { + content.innerHTML = '
Loading...
'; + } else { + content.style.opacity = "0.5"; + } + + const params = new URLSearchParams({ + page: state.page, + metric: state.metric, + }); + if (state.q) params.set("q", state.q); + if (state.language) params.set("language", state.language); + if (state.topic) params.set("topic", state.topic); + + try { + const data = await fetchJSON(`/api/leaderboard?${params}`); + cache[key] = data; + applyData(data, state); + } catch (e) { + $("err").textContent = e.message; + $("err").style.display = "block"; + if (!hasItems()) { + content.innerHTML = ""; + } + } finally { + content.style.opacity = "1"; + } +} + +export function applyData(data, state) { + const items = data.items || []; + const total = data.total || 0; + const totalPages = data.totalPages || 1; + const page = data.page || state.page; + + setItems(items, totalPages, page); + + $("pageInfo").textContent = `${fmtInt(total)} repos found`; + $("pageInput").value = page; + $("pageInput").max = totalPages; + $("pageMax").textContent = `/ ${totalPages}`; + $("prev").disabled = page <= 1; + $("next").disabled = page >= totalPages; +} + +// syncUI is still defined elsewhere - perhaps in app.js +export function syncUI(state) { + $("q").value = state.q; + $("metric").value = state.metric; + $("language").value = state.language; + $("topic").value = state.topic; + $("viewTable").classList.toggle("active", state.view === "table"); + $("viewCards").classList.toggle("active", state.view === "cards"); +} diff --git a/static/app.js b/static/app.js index 28377ef..09d36fa 100755 --- a/static/app.js +++ b/static/app.js @@ -1,647 +1,8 @@ -const $ = (id) => document.getElementById(id); -const STATIC_DATA = { languages: [], topics: [] }; - -const cache = {}; -const repoCache = {}; -let didAutoScroll = false; - -function getUIParams() { - const url = new URL(location.href); - return { - highlight: url.searchParams.get("highlight") || "", - open: url.searchParams.get("open") || "", - }; -} - -function consumeUrlParam(name) { - const url = new URL(location.href); - url.searchParams.delete(name); - history.replaceState({}, "", url.toString()); -} - -function cssEscape(s) { - return CSS.escape(String(s)); -} - -function cacheKey(state) { - return JSON.stringify([ - state.page, - state.metric, - state.q, - state.language, - state.topic, - ]); -} - -function fmtInt(n) { - if (n === null || n === undefined) return "–"; - return Number(n).toLocaleString(); -} - -function fmtDiskSize(kb) { - if (kb === null || kb === undefined) return "–"; - const num = Number(kb); - if (num >= 1000) { - return (num / 1000).toFixed(1) + " MB"; - } - return num.toLocaleString() + " KB"; -} - -function fmtDate(iso) { - return iso ? String(iso).slice(0, 10) : ""; -} - -function fmtDelta(n) { - if (n === null || n === undefined) return "–"; - const v = Number(n); - const s = v.toLocaleString(); - return v < 0 ? `-${s}` : v > 0 ? `+${s}` : s; -} - -function escapeHtml(str) { - if (!str) return ""; - return String(str) - .replace(/&/g, "&") - .replace(//g, ">") - .replace(/"/g, """); -} - -function archivedBadge(it) { - return it && it.i ? `Archived` : ""; -} - -function applyHighlightAndOpen() { - const { highlight, open } = getUIParams(); - - if (highlight && !didAutoScroll) { - const el = document.querySelector(`[data-repo="${cssEscape(highlight)}"]`); - if (el) { - el.scrollIntoView({ block: "center" }); - didAutoScroll = true; - } - } - - if (open && currentItems.some((it) => it.n === open)) { - openModal(open); - consumeUrlParam("open"); - } -} - -function getState() { - const url = new URL(location.href); - return { - page: Number(url.searchParams.get("page") || 1), - metric: url.searchParams.get("metric") || "stars", - q: url.searchParams.get("q") || "", - language: url.searchParams.get("language") || "", - topic: url.searchParams.get("topic") || "", - view: url.searchParams.get("view") || "table", - }; -} - -function setState(state) { - const url = new URL(location.href); - url.searchParams.set("page", state.page); - url.searchParams.set("metric", state.metric); - url.searchParams.set("view", state.view); - state.q ? url.searchParams.set("q", state.q) : url.searchParams.delete("q"); - state.language ? - url.searchParams.set("language", state.language) : - url.searchParams.delete("language"); - state.topic ? - url.searchParams.set("topic", state.topic) : - url.searchParams.delete("topic"); - history.replaceState({}, "", url.toString()); -} - -async function fetchJSON(url) { - const r = await fetch(url); - if (!r.ok) throw new Error(`HTTP ${r.status}`); - return r.json(); -} - -let currentItems = []; -let currentSortKey = null; -let currentSortAsc = false; -let currentTotalPages = 1; - -function sortItems(key) { - if (currentSortKey === key) { - currentSortAsc = !currentSortAsc; - } else { - currentSortKey = key; - currentSortAsc = false; - } - - currentItems.sort((a, b) => { - let va = a[key], - vb = b[key]; - - if (key === "n") { - va = va ? va.split("/").pop().toLowerCase() : ""; - vb = vb ? vb.split("/").pop().toLowerCase() : ""; - } else { - if (va === null || va === undefined) va = -Infinity; - if (vb === null || vb === undefined) vb = -Infinity; - if (typeof va === "string") va = va.toLowerCase(); - if (typeof vb === "string") vb = vb.toLowerCase(); - } - - if (va < vb) return currentSortAsc ? -1 : 1; - if (va > vb) return currentSortAsc ? 1 : -1; - return 0; - }); - - renderContent(); -} - -function renderTable() { - const state = getState(); - const isTrending = String(state.metric || "").startsWith("trending"); - const startRank = (state.page - 1) * 100; - const { highlight } = getUIParams(); - - let html = `
- - - - - ${isTrending ? `` : ``} - - - `; - - currentItems.forEach((it, i) => { - const isHi = highlight && highlight === it.n; - html += ` - - - - ${isTrending ? `` : ``} - - - `; - }); - - html += "
#RepositoryStarsNew starsForksLanguage
${startRank + i + 1} - ${escapeHtml(it.n)} - ${archivedBadge(it)} - ${fmtInt(it.s)}${fmtDelta(it.ns)}${fmtInt(it.f)}${escapeHtml(it.l || "")}
"; - return html; -} - -function renderCards() { - const state = getState(); - const isTrending = String(state.metric || "").startsWith("trending"); - let html = '
'; - const { highlight } = getUIParams(); - - currentItems.forEach((it) => { - const topics = (it.t || []).slice(0, 5); - const isHi = highlight && highlight === it.n; - - html += `
-
-
- ${escapeHtml(it.n)} - ${archivedBadge(it)} -
-
- ${fmtInt(it.s)}${isTrending ? ` (${fmtDelta(it.ns)})` : ``} - ${fmtInt(it.f)} -
-
- ${it.a ? `
${escapeHtml(it.a)}
` : ""} - ${topics.length > 0 ? `
${topics.map((t) => `${escapeHtml(t)}`).join("")}
` : ""} - ${it.h ? `
${escapeHtml(it.h)}
` : ""} -
-
${escapeHtml(it.l || "")}
-
- ${fmtInt(it.w)} watchers - ${it.d !== null ? `${fmtDiskSize(it.d)}` : ""} -
-
-
`; - }); - - html += "
"; - return html; -} - -function renderContent() { - const state = getState(); - const content = $("content"); - content.innerHTML = state.view === "cards" ? renderCards() : renderTable(); - - content.querySelectorAll(".repo-link, .card").forEach((el) => { - el.addEventListener("click", (e) => { - if (e.target.tagName === "A") return; - openModal(el.dataset.repo); - }); - }); - - content.querySelectorAll("th[data-sort]").forEach((th) => { - th.addEventListener("click", () => sortItems(th.dataset.sort)); - }); - applyHighlightAndOpen(); -} - -async function loadLeaderboard(useCache = true) { - const state = getState(); - const key = cacheKey(state); - - $("err").style.display = "none"; - syncUI(state); - - if (useCache && cache[key]) { - applyData(cache[key], state); - return; - } - - const content = $("content"); - if (currentItems.length === 0) { - content.innerHTML = '
Loading...
'; - } else { - content.style.opacity = "0.5"; - } - - const params = new URLSearchParams({ - page: state.page, - metric: state.metric, - }); - if (state.q) params.set("q", state.q); - if (state.language) params.set("language", state.language); - if (state.topic) params.set("topic", state.topic); - - try { - const data = await fetchJSON(`/api/leaderboard?${params}`); - cache[key] = data; - applyData(data, state); - } catch (e) { - $("err").textContent = e.message; - $("err").style.display = "block"; - if (currentItems.length === 0) { - content.innerHTML = ""; - } - } finally { - content.style.opacity = "1"; - } -} - -function applyData(data, state) { - currentItems = data.items || []; - currentSortKey = null; - currentSortAsc = false; - renderContent(); - - const total = data.total || 0; - const totalPages = data.totalPages || 1; - const page = data.page || state.page; - - currentTotalPages = totalPages; - - $("pageInfo").textContent = `${fmtInt(total)} repos found`; - $("pageInput").value = page; - $("pageInput").max = totalPages; - $("pageMax").textContent = `/ ${totalPages}`; - $("prev").disabled = page <= 1; - $("next").disabled = page >= totalPages; -} - -function syncUI(state) { - $("q").value = state.q; - $("metric").value = state.metric; - $("language").value = state.language; - $("topic").value = state.topic; - $("viewTable").classList.toggle("active", state.view === "table"); - $("viewCards").classList.toggle("active", state.view === "cards"); -} - -function loadLanguages() { - const dl = $("languagesList"); - dl.innerHTML = ""; - (STATIC_DATA.languages || []).forEach((name) => { - const opt = document.createElement("option"); - opt.value = name; - dl.appendChild(opt); - }); -} - -function loadTopics() { - const dl = $("topicsList"); - dl.innerHTML = ""; - (STATIC_DATA.topics || []).forEach((t) => { - const opt = document.createElement("option"); - opt.value = t.name; - opt.textContent = `${t.name} (${t.count})`; - dl.appendChild(opt); - }); -} - -let modalCharts = []; - -function openModal(repoName) { - $("modalOverlay").classList.remove("hidden"); - $("modalTitle").innerHTML = - `${escapeHtml(repoName)}`; - $("modalBody").innerHTML = '
Loading...
'; - document.body.style.overflow = "hidden"; - loadRepoDetails(repoName); -} - -function closeModal() { - $("modalOverlay").classList.add("hidden"); - document.body.style.overflow = ""; - modalCharts.forEach((c) => c.destroy()); - modalCharts = []; -} - -async function loadRepoDetails(name) { - if (repoCache[name]) { - renderModal(repoCache[name].repo, repoCache[name].segments); - return; - } - try { - const [repo, hist] = await Promise.all([ - fetchJSON(`/api/repo?name=${encodeURIComponent(name)}`), - fetchJSON(`/api/repo/history?name=${encodeURIComponent(name)}`), - ]); - repoCache[name] = { repo, segments: hist.segments || [] }; - renderModal(repo, hist.segments || []); - } catch (e) { - $("modalBody").innerHTML = - `
${escapeHtml(e.message)}
`; - } -} - -const chartOpts = (data) => { - const minX = data.length > 0 ? data[0].x : undefined; - const maxX = data.length > 0 ? data[data.length - 1].x : undefined; - - return { - type: "line", - data: { - datasets: [ - { - data: data, - borderColor: "#58a6ff", - borderWidth: 2, - pointRadius: 0, - tension: 0.2, - stepped: false, - fill: false, - }, - ], - }, - options: { - responsive: true, - maintainAspectRatio: false, - animation: false, - plugins: { - legend: { display: false }, - tooltip: { - callbacks: { - title: (ctx) => { - return new Date(ctx[0].parsed.x).toISOString().slice(0, 10); - }, - }, - }, - }, - scales: { - x: { - type: "linear", - min: minX, - max: maxX, - grid: { color: "#30363d" }, - ticks: { - color: "#8b949e", - maxTicksLimit: 6, - callback: (val) => new Date(val).toISOString().slice(0, 10), - }, - }, - y: { - grid: { color: "#30363d" }, - ticks: { - color: "#8b949e", - callback: function (val) { - return val % 1 === 0 ? val : null; - }, - }, - }, - }, - }, - }; -}; - -function renderModal(repo, segments) { - let metaHtml = '
'; - - if (repo.g !== null && repo.g !== undefined) { - const domain = window.location.origin; - const badgeUrl = `https://img.shields.io/endpoint?url=${encodeURIComponent(domain + "/api/rank?name=" + repo.n)}`; - const markdown = `[![Global Rank](${badgeUrl})](${domain}/${repo.n})`; - - metaHtml += ` -
Global rank
-
- #${fmtInt(repo.g)} - -
`; - } - if (repo.i) { - metaHtml += `
Status
${archivedBadge(repo)}
`; - } - - if (repo.a) { - metaHtml += `
Description
${escapeHtml(repo.a)}
`; - } - if (repo.h) { - metaHtml += `
Homepage
${escapeHtml(repo.h)}
`; - } - if (repo.l) { - metaHtml += `
Language
${escapeHtml(repo.l)}
`; - } - if (repo.t && repo.t.length > 0) { - const topicsHtml = repo.t - .map( - (t) => - `${escapeHtml(t)}`, - ) - .join(" "); - metaHtml += `
Topics
${topicsHtml}
`; - } - if (repo.c) { - metaHtml += `
Created
${fmtDate(repo.c)}
`; - } - if (repo.p) { - metaHtml += `
Last push
${fmtDate(repo.p)}
`; - } - metaHtml += "
"; - - const statsHtml = ` -
-
${fmtInt(repo.s)}
Stars
-
${fmtInt(repo.f)}
Forks
-
${fmtInt(repo.w)}
Watchers
- ${repo.d !== null ? `
${fmtDiskSize(repo.d)}
Disk
` : ""} -
- `; - - const chartsHtml = ` -
-

History

-
-

Stars

-
-
-

Forks

-
-

Watchers

-

Disk Usage

-
-
-
- `; - - $("modalBody").innerHTML = metaHtml + statsHtml + chartsHtml; - - modalCharts.forEach((c) => c.destroy()); - modalCharts = []; - - const makeData = (key) => { - const data = []; - segments.forEach((s) => { - if (s.startFetchedAt) { - data.push({ - x: new Date(s.startFetchedAt).getTime(), - y: s[key] - }); - - } - if (s.endFetchedAt && s.endFetchedAt !== s.startFetchedAt) { - data.push({ - x: new Date(s.endFetchedAt).getTime(), - y: s[key] - }); - - } - }); - return data; - }; - - const stars = makeData("s"); - const forks = makeData("f"); - const watchers = makeData("w"); - const disk = makeData("d"); - - modalCharts.push(new Chart($("chartStars"), chartOpts(stars))); - modalCharts.push(new Chart($("chartForks"), chartOpts(forks))); - modalCharts.push(new Chart($("chartWatchers"), chartOpts(watchers))); - modalCharts.push(new Chart($("chartDisk"), chartOpts(disk))); - - $("modalBody") - .querySelectorAll(".modal-filter-link") - .forEach((link) => { - link.addEventListener("click", (e) => { - e.preventDefault(); - closeModal(); - window.location.href = link.href; - }); - }); -} - -function goToPage() { - const val = parseInt($("pageInput").value, 10); - if (val >= 1 && val <= currentTotalPages) { - const state = getState(); - state.page = val; - setState(state); - loadLeaderboard(); - } else { - const state = getState(); - $("pageInput").value = state.page; - } -} - -function wire() { - $("apply").addEventListener("click", () => { - const state = getState(); - state.page = 1; - state.metric = $("metric").value; - state.q = $("q").value.trim(); - state.language = $("language").value.trim(); - state.topic = $("topic").value.trim(); - setState(state); - loadLeaderboard(false); - }); - - $("appTitle").addEventListener("click", (e) => { - e.preventDefault(); - history.pushState({}, "", "/"); - loadLeaderboard(); - }); - - $("q").addEventListener("keydown", (e) => { - if (e.key === "Enter") $("apply").click(); - }); - - $("prev").addEventListener("click", () => { - const state = getState(); - if (state.page > 1) { - state.page--; - setState(state); - loadLeaderboard(); - } - }); - - $("next").addEventListener("click", () => { - const state = getState(); - if (state.page < currentTotalPages) { - state.page++; - setState(state); - loadLeaderboard(); - } - }); - - $("goPage").addEventListener("click", goToPage); - - $("pageInput").addEventListener("keydown", (e) => { - if (e.key === "Enter") goToPage(); - }); - - $("viewTable").addEventListener("click", () => { - const state = getState(); - state.view = "table"; - setState(state); - $("viewTable").classList.add("active"); - $("viewCards").classList.remove("active"); - renderContent(); - }); - - $("viewCards").addEventListener("click", () => { - const state = getState(); - state.view = "cards"; - setState(state); - $("viewCards").classList.add("active"); - $("viewTable").classList.remove("active"); - renderContent(); - }); - - $("modalOverlay").addEventListener("click", (e) => { - if (e.target === $("modalOverlay")) closeModal(); - }); - $("modalClose").addEventListener("click", closeModal); - document.addEventListener("keydown", (e) => { - if (e.key === "Escape" && !$("modalOverlay").classList.contains("hidden")) { - closeModal(); - } - }); -} +// main entrypoint remnant; actual logic is split into modules +import { loadLeaderboard } from './api.js'; +import { loadLanguages, loadTopics, wire } from './ui.js'; +// start the application (async function main() { wire(); loadLanguages(); diff --git a/static/lib.js b/static/lib.js new file mode 100644 index 0000000..ef8671a --- /dev/null +++ b/static/lib.js @@ -0,0 +1,54 @@ +// utility helper functions +export const $ = (id) => document.getElementById(id); + +export function cssEscape(s) { + return CSS.escape(String(s)); +} + +export function fmtInt(n) { + if (n === null || n === undefined) return "–"; + return Number(n).toLocaleString(); +} + +export function fmtDiskSize(kb) { + if (kb === null || kb === undefined) return "–"; + const num = Number(kb); + if (num >= 1000) { + return (num / 1000).toFixed(1) + " MB"; + } + return num.toLocaleString() + " KB"; +} + +export function fmtDate(iso) { + return iso ? String(iso).slice(0, 10) : ""; +} + +export function fmtDelta(n) { + if (n === null || n === undefined) return "–"; + const v = Number(n); + const s = v.toLocaleString(); + return v < 0 ? `-${s}` : v > 0 ? `+${s}` : s; +} + +export function escapeHtml(str) { + if (!str) return ""; + return String(str) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/\"/g, """); +} + +export function archivedBadge(it) { + return it && it.i ? `Archived` : ""; +} + +export function cacheKey(state) { + return JSON.stringify([ + state.page, + state.metric, + state.q, + state.language, + state.topic, + ]); +} diff --git a/static/modal.js b/static/modal.js new file mode 100644 index 0000000..62c199b --- /dev/null +++ b/static/modal.js @@ -0,0 +1,224 @@ +import { fetchJSON } from "./api.js"; +import { $, archivedBadge, escapeHtml, fmtDate, fmtDiskSize, fmtInt } from "./lib.js"; + +// shared data caches +const repoCache = {}; + +let modalCharts = []; + +export function openModal(repoName) { + $("modalOverlay").classList.remove("hidden"); + $("modalTitle").innerHTML = + `${escapeHtml(repoName)}`; + $("modalBody").innerHTML = '
Loading...
'; + document.body.style.overflow = "hidden"; + loadRepoDetails(repoName); +} + +export function closeModal() { + $("modalOverlay").classList.add("hidden"); + document.body.style.overflow = ""; + modalCharts.forEach((c) => c.destroy()); + modalCharts = []; +} + +async function loadRepoDetails(name) { + if (repoCache[name]) { + renderModal(repoCache[name].repo, repoCache[name].segments); + return; + } + try { + const [repo, hist] = await Promise.all([ + fetchJSON(`/api/repo?name=${encodeURIComponent(name)}`), + fetchJSON(`/api/repo/history?name=${encodeURIComponent(name)}`), + ]); + repoCache[name] = { repo, segments: hist.segments || [] }; + renderModal(repo, hist.segments || []); + } catch (e) { + $("modalBody").innerHTML = + `
${escapeHtml(e.message)}
`; + } +} + +const chartOpts = (data) => { + const minX = data.length > 0 ? data[0].x : undefined; + const maxX = data.length > 0 ? data[data.length - 1].x : undefined; + + return { + type: "line", + data: { + datasets: [ + { + data: data, + borderColor: "#58a6ff", + borderWidth: 2, + pointRadius: 0, + tension: 0.2, + stepped: false, + fill: false, + }, + ], + }, + options: { + responsive: true, + maintainAspectRatio: false, + animation: false, + plugins: { + legend: { display: false }, + tooltip: { + callbacks: { + title: (ctx) => { + return new Date(ctx[0].parsed.x).toISOString().slice(0, 10); + }, + }, + }, + }, + scales: { + x: { + type: "linear", + min: minX, + max: maxX, + grid: { color: "#30363d" }, + ticks: { + color: "#8b949e", + maxTicksLimit: 6, + callback: (val) => new Date(val).toISOString().slice(0, 10), + }, + }, + y: { + grid: { color: "#30363d" }, + ticks: { + color: "#8b949e", + callback: function (val) { + return val % 1 === 0 ? val : null; + }, + }, + }, + }, + }, + }; +}; + +function renderModal(repo, segments) { + let metaHtml = '
'; + + if (repo.g !== null && repo.g !== undefined) { + const domain = window.location.origin; + const badgeUrl = `https://img.shields.io/endpoint?url=${encodeURIComponent(domain + "/api/rank?name=" + repo.n)}`; + const markdown = `[![Global Rank](${badgeUrl})](${domain}/${repo.n})`; + + metaHtml += ` +
Global rank
+
+ #${fmtInt(repo.g)} + +
`; + } + if (repo.i) { + metaHtml += `
Status
${archivedBadge(repo)}
`; + } + + if (repo.a) { + metaHtml += `
Description
${escapeHtml(repo.a)}
`; + } + if (repo.h) { + metaHtml += `
Homepage
${escapeHtml(repo.h)}
`; + } + if (repo.l) { + metaHtml += `
Language
${escapeHtml(repo.l)}
`; + } + if (repo.t && repo.t.length > 0) { + const topicsHtml = repo.t + .map( + (t) => + `${escapeHtml(t)}`, + ) + .join(" "); + metaHtml += `
Topics
${topicsHtml}
`; + } + if (repo.c) { + metaHtml += `
Created
${fmtDate(repo.c)}
`; + } + if (repo.p) { + metaHtml += `
Last push
${fmtDate(repo.p)}
`; + } + metaHtml += "
"; + + const statsHtml = ` +
+
${fmtInt(repo.s)}
Stars
+
${fmtInt(repo.f)}
Forks
+
${fmtInt(repo.w)}
Watchers
+ ${repo.d !== null ? `
${fmtDiskSize(repo.d)}
Disk
` : ""} +
+ `; + + const chartsHtml = ` +
+

History

+
+

Stars

+
+
+

Forks

+
+

Watchers

+

Disk Usage

+
+
+
+ `; + + $("modalBody").innerHTML = metaHtml + statsHtml + chartsHtml; + + modalCharts.forEach((c) => c.destroy()); + modalCharts = []; + + const makeData = (key) => { + const data = []; + segments.forEach((s) => { + if (s.startFetchedAt) { + data.push({ + x: new Date(s.startFetchedAt).getTime(), + y: s[key] + }); + + } + if (s.endFetchedAt && s.endFetchedAt !== s.startFetchedAt) { + data.push({ + x: new Date(s.endFetchedAt).getTime(), + y: s[key] + }); + + } + }); + return data; + }; + + const stars = makeData("s"); + const forks = makeData("f"); + const watchers = makeData("w"); + const disk = makeData("d"); + + modalCharts.push(new Chart($("chartStars"), chartOpts(stars))); + modalCharts.push(new Chart($("chartForks"), chartOpts(forks))); + modalCharts.push(new Chart($("chartWatchers"), chartOpts(watchers))); + modalCharts.push(new Chart($("chartDisk"), chartOpts(disk))); + + $("modalBody") + .querySelectorAll(".modal-filter-link") + .forEach((link) => { + link.addEventListener("click", (e) => { + e.preventDefault(); + closeModal(); + window.location.href = link.href; + }); + }); +} + +// functions used by render.js +export { applyHighlightAndOpen } from "./render.js"; +export { openModal }; diff --git a/static/render.js b/static/render.js new file mode 100644 index 0000000..ec8cd40 --- /dev/null +++ b/static/render.js @@ -0,0 +1,152 @@ +import { archivedBadge, escapeHtml, fmtDelta, fmtInt } from "./lib.js"; +import { getState } from "./state.js"; + +let currentItems = []; +let currentSortKey = null; +let currentSortAsc = false; +let currentTotalPages = 1; + +export function sortItems(key) { + if (currentSortKey === key) { + currentSortAsc = !currentSortAsc; + } else { + currentSortKey = key; + currentSortAsc = false; + } + + currentItems.sort((a, b) => { + let va = a[key], + vb = b[key]; + + if (key === "n") { + va = va ? va.split("/").pop().toLowerCase() : ""; + vb = vb ? vb.split("/").pop().toLowerCase() : ""; + } else { + if (va === null || va === undefined) va = -Infinity; + if (vb === null || vb === undefined) vb = -Infinity; + if (typeof va === "string") va = va.toLowerCase(); + if (typeof vb === "string") vb = vb.toLowerCase(); + } + + if (va < vb) return currentSortAsc ? -1 : 1; + if (va > vb) return currentSortAsc ? 1 : -1; + return 0; + }); + + renderContent(); +} + +export function renderTable() { + const state = getState(); + const isTrending = String(state.metric || "").startsWith("trending"); + const startRank = (state.page - 1) * 100; + const { highlight } = getUIParams(); + + let html = `
+ + + + + ${isTrending ? `` : ``} + + + `; + + currentItems.forEach((it, i) => { + const isHi = highlight && highlight === it.n; + html += ` + + + + ${isTrending ? `` : ``} + + + `; + }); + + html += "
#RepositoryStarsNew starsForksLanguage
${startRank + i + 1} + ${escapeHtml(it.n)} + ${archivedBadge(it)} + ${fmtInt(it.s)}${fmtDelta(it.ns)}${fmtInt(it.f)}${escapeHtml(it.l || "")}
"; + return html; +} + +export function renderCards() { + const state = getState(); + const isTrending = String(state.metric || "").startsWith("trending"); + let html = '
'; + const { highlight } = getUIParams(); + + currentItems.forEach((it) => { + const topics = (it.t || []).slice(0, 5); + const isHi = highlight && highlight === it.n; + + html += `
+
+
+ ${escapeHtml(it.n)} + ${archivedBadge(it)} +
+
+ ${fmtInt(it.s)}${isTrending ? ` (${fmtDelta(it.ns)})` : ``} + ${fmtInt(it.f)} +
+
+ ${it.a ? `
${escapeHtml(it.a)}
` : ""} + ${topics.length > 0 ? `
${topics.map((t) => `${escapeHtml(t)}`).join("")}
` : ""} + ${it.h ? `
${escapeHtml(it.h)}
` : ""} +
+
${escapeHtml(it.l || "")}
+
+ ${fmtInt(it.w)} watchers + ${it.d !== null ? `${fmtDiskSize(it.d)}` : ""} +
+
+
`; + }); + + html += "
"; + return html; +} + +export function renderContent() { + const state = getState(); + const content = $("content"); + content.innerHTML = state.view === "cards" ? renderCards() : renderTable(); + + content.querySelectorAll(".repo-link, .card").forEach((el) => { + el.addEventListener("click", (e) => { + if (e.target.tagName === "A") return; + openModal(el.dataset.repo); + }); + }); + + content.querySelectorAll("th[data-sort]").forEach((th) => { + th.addEventListener("click", () => sortItems(th.dataset.sort)); + }); + applyHighlightAndOpen(); +} + +// export for other modules to set state +export function setItems(items, totalPages, page) { + currentItems = items; + currentSortKey = null; + currentSortAsc = false; + renderContent(); + currentTotalPages = totalPages; + return currentTotalPages; +} + +export function hasItems() { + return currentItems.length > 0; +} + +export function getTotalPages() { + return currentTotalPages; +} + +// import functions used in renderContent but defined elsewhere +import { $ } from "./lib.js"; +import { applyHighlightAndOpen, openModal } from "./modal.js"; +import { getUIParams } from "./state.js"; + diff --git a/static/state.js b/static/state.js new file mode 100644 index 0000000..4b43c82 --- /dev/null +++ b/static/state.js @@ -0,0 +1,41 @@ +// functions related to URL state and parameters +export function getUIParams() { + const url = new URL(location.href); + return { + highlight: url.searchParams.get("highlight") || "", + open: url.searchParams.get("open") || "", + }; +} + +export function consumeUrlParam(name) { + const url = new URL(location.href); + url.searchParams.delete(name); + history.replaceState({}, "", url.toString()); +} + +export function getState() { + const url = new URL(location.href); + return { + page: Number(url.searchParams.get("page") || 1), + metric: url.searchParams.get("metric") || "stars", + q: url.searchParams.get("q") || "", + language: url.searchParams.get("language") || "", + topic: url.searchParams.get("topic") || "", + view: url.searchParams.get("view") || "table", + }; +} + +export function setState(state) { + const url = new URL(location.href); + url.searchParams.set("page", state.page); + url.searchParams.set("metric", state.metric); + url.searchParams.set("view", state.view); + state.q ? url.searchParams.set("q", state.q) : url.searchParams.delete("q"); + state.language ? + url.searchParams.set("language", state.language) : + url.searchParams.delete("language"); + state.topic ? + url.searchParams.set("topic", state.topic) : + url.searchParams.delete("topic"); + history.replaceState({}, "", url.toString()); +} diff --git a/static/style.css b/static/style.css index 2c063a2..30db4ae 100755 --- a/static/style.css +++ b/static/style.css @@ -8,7 +8,7 @@ --accent-hover: #79b8ff; --danger: #f85149; --mono: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; - --sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif; + --sans: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif; } * { box-sizing: border-box; @@ -78,7 +78,9 @@ h1 a:hover { font-size: 12px; color: var(--muted); } -input, select, button { +input, +select, +button { padding: 8px 12px; border-radius: 6px; border: 1px solid var(--border); @@ -87,7 +89,8 @@ input, select, button { font-size: 14px; outline: none; } -input:focus, select:focus { +input:focus, +select:focus { border-color: var(--accent); } input::placeholder { @@ -154,7 +157,8 @@ table { width: 100%; border-collapse: collapse; } -th, td { +th, +td { padding: 10px 12px; text-align: left; border-bottom: 1px solid var(--border); @@ -169,7 +173,7 @@ th { white-space: nowrap; } th[data-sort] { - cursor: pointer; + cursor: pointer; } th:hover { color: var(--text); @@ -321,14 +325,14 @@ tr.highlight td { gap: 8px; align-items: center; } -.pager-controls input[type="number"] { +.pager-controls input[type='number'] { width: 60px; text-align: center; -moz-appearance: textfield; appearance: textfield; } -.pager-controls input[type="number"]::-webkit-outer-spin-button, -.pager-controls input[type="number"]::-webkit-inner-spin-button { +.pager-controls input[type='number']::-webkit-outer-spin-button, +.pager-controls input[type='number']::-webkit-inner-spin-button { -webkit-appearance: none; margin: 0; } diff --git a/static/ui.js b/static/ui.js new file mode 100644 index 0000000..7c24c3f --- /dev/null +++ b/static/ui.js @@ -0,0 +1,118 @@ +import { loadLeaderboard } from "./api.js"; +import { $ } from "./lib.js"; +import { closeModal } from "./modal.js"; +import { getTotalPages, renderContent } from "./render.js"; +import { getState, setState } from "./state.js"; + +export const STATIC_DATA = { languages: [], topics: [] }; + +export function loadLanguages() { + const dl = $("languagesList"); + dl.innerHTML = ""; + (STATIC_DATA.languages || []).forEach((name) => { + const opt = document.createElement("option"); + opt.value = name; + dl.appendChild(opt); + }); +} + +export function loadTopics() { + const dl = $("topicsList"); + dl.innerHTML = ""; + (STATIC_DATA.topics || []).forEach((t) => { + const opt = document.createElement("option"); + opt.value = t.name; + opt.textContent = `${t.name} (${t.count})`; + dl.appendChild(opt); + }); +} + +export function goToPage() { + const val = parseInt($("pageInput").value, 10); + if (val >= 1 && val <= getTotalPages()) { + const state = getState(); + state.page = val; + setState(state); + loadLeaderboard(); + } else { + const state = getState(); + $("pageInput").value = state.page; + } +} + +export function wire() { + $("apply").addEventListener("click", () => { + const state = getState(); + state.page = 1; + state.metric = $("metric").value; + state.q = $("q").value.trim(); + state.language = $("language").value.trim(); + state.topic = $("topic").value.trim(); + setState(state); + loadLeaderboard(false); + }); + + $("appTitle").addEventListener("click", (e) => { + e.preventDefault(); + history.pushState({}, "", "/"); + loadLeaderboard(); + }); + + $("q").addEventListener("keydown", (e) => { + if (e.key === "Enter") $("apply").click(); + }); + + $("prev").addEventListener("click", () => { + const state = getState(); + if (state.page > 1) { + state.page--; + setState(state); + loadLeaderboard(); + } + }); + + $("next").addEventListener("click", () => { + const state = getState(); + if (state.page < getTotalPages()) { + state.page++; + setState(state); + loadLeaderboard(); + } + }); + + $("goPage").addEventListener("click", goToPage); + + $("pageInput").addEventListener("keydown", (e) => { + if (e.key === "Enter") goToPage(); + }); + + $("viewTable").addEventListener("click", () => { + const state = getState(); + state.view = "table"; + setState(state); + $("viewTable").classList.add("active"); + $("viewCards").classList.remove("active"); + renderContent(); + }); + + $("viewCards").addEventListener("click", () => { + const state = getState(); + state.view = "cards"; + setState(state); + $("viewCards").classList.add("active"); + $("viewTable").classList.remove("active"); + renderContent(); + }); + + $("modalOverlay").addEventListener("click", (e) => { + if (e.target === $("modalOverlay")) closeModal(); + }); + $("modalClose").addEventListener("click", closeModal); + document.addEventListener("keydown", (e) => { + if (e.key === "Escape" && !$("modalOverlay").classList.contains("hidden")) { + closeModal(); + } + }); +} + + From ea538864de08977f8a5d8cc118bdf7f86d32e5c2 Mon Sep 17 00:00:00 2001 From: ichigirl Date: Wed, 4 Mar 2026 20:58:58 +0100 Subject: [PATCH 2/5] updated crawler.py to be OOP --- crawler.py | 178 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 103 insertions(+), 75 deletions(-) diff --git a/crawler.py b/crawler.py index ab0b407..dd35f75 100755 --- a/crawler.py +++ b/crawler.py @@ -5,17 +5,14 @@ import subprocess import time from datetime import datetime, timedelta, timezone -from typing import Callable, List +from typing import List import requests from db import RepoDB TOKEN = "replace this" -MIN_STARS = 1_000 -DB_PATH = "github_repos.db" -LIVE_DB_PATH = "repos.db" -PM2_APP_NAME = "git_leaderboard" + GRAPHQL_QUERY = """ query($queryString: String!, $cursor: String) { rateLimit { @@ -127,75 +124,104 @@ def execute_query(self, query: str, cursor: str = None): raise Exception("Max retries exceeded.") -def deploy_site(crawled_db_path: str): - log("Preparing deployment...") - - db = RepoDB(crawled_db_path) - - row = db.conn.execute("SELECT COUNT(*) AS cnt FROM repo_latest").fetchone() - total_repos = int(row["cnt"]) if row else 0 - formatted_total = "{:,}".format(total_repos) - - lang_rows = db.conn.execute("SELECT name FROM language ORDER BY name LIMIT 5000").fetchall() - languages = [str(r["name"]) for r in lang_rows] - - topic_sql = """ - SELECT t.name, COUNT(rtl.repo_id) AS cnt - FROM topic t - JOIN repo_topic_latest rtl ON rtl.topic_id = t.id - GROUP BY t.id - ORDER BY cnt DESC - LIMIT 500 - """ - topic_rows = db.conn.execute(topic_sql).fetchall() - topics = [{"name": str(r["name"]), "count": int(r["cnt"])} for r in topic_rows] - - db.close() - - log("Stopping PM2 service...") - try: - subprocess.run(["pm2", "stop", PM2_APP_NAME], check=False, stdout=subprocess.DEVNULL) - except Exception as e: - log(f"Warning: PM2 stop failed (might not be running): {e}") - - log("Swapping Database...") - if os.path.exists(crawled_db_path): - if os.path.exists(LIVE_DB_PATH): - os.remove(LIVE_DB_PATH) - shutil.copy2(crawled_db_path, LIVE_DB_PATH) - - index_path = "index.html" - if os.path.exists(index_path): - with open(index_path, "r", encoding="utf-8") as f: - html_content = f.read() - - html_content = re.sub(r'(id="totalRepos"[^>]*>).*?()', f"\\g<1>{formatted_total}\\g<2>", html_content) - - with open(index_path, "w", encoding="utf-8") as f: - f.write(html_content) - log(f"Updated {index_path} with {formatted_total} repos.") - - app_js_path = "static/app.js" - if os.path.exists(app_js_path): - with open(app_js_path, "r", encoding="utf-8") as f: - js_content = f.read() - - static_data = {"languages": languages, "topics": topics} - injection_code1 = f"const STATIC_DATA = {json.dumps(static_data)};" - - if "const STATIC_DATA =" in js_content: - js_content = re.sub(r"const STATIC_DATA = \{.*?\};", injection_code1, js_content, flags=re.DOTALL) - - with open(app_js_path, "w", encoding="utf-8") as f: - f.write(js_content) - log(f"Updated {app_js_path} with static lists.") - - log("Restarting PM2 service...") - try: - subprocess.run(["pm2", "restart", PM2_APP_NAME], check=False, stdout=subprocess.DEVNULL) - except Exception as e: - log(f"Warning: PM2 restart failed (maybe not running): {e}") - log("Deployment complete.") +class Crawler: + def __init__( + self, + token: str, + min_stars: int = 1_000, + db_path: str = "github_repos.db", + live_db_path: str = "repos.db", + pm2_app_name: str = "git_leaderboard", + ): + self.token = token + self.min_stars = min_stars + self.db_path = db_path + self.live_db_path = live_db_path + self.pm2_app_name = pm2_app_name + + self.gh = GithubGraphQL(self.token) + self.db = RepoDB(self.db_path) + + self.current_min_stars = self.min_stars + self.total_fetched = 0 + # used by upsert logic to avoid duplicate processing in a run + self._processed_repo_ids: set[int] = set() + + def log(self, *args, **kwargs): + """Simple timestamped logger""" + timestamp = datetime.now().strftime("[%d:%H:%S]") + print(timestamp, *args, **kwargs) + + def deploy_site(self): + self.log("Preparing deployment...") + + # reopen a fresh connection so we don't interfere with crawling + db = RepoDB(self.db_path) + + row = db.conn.execute("SELECT COUNT(*) AS cnt FROM repo_latest").fetchone() + total_repos = int(row["cnt"]) if row else 0 + formatted_total = "{:,}".format(total_repos) + + lang_rows = db.conn.execute("SELECT name FROM language ORDER BY name LIMIT 5000").fetchall() + languages = [str(r["name"]) for r in lang_rows] + + topic_sql = """ + SELECT t.name, COUNT(rtl.repo_id) AS cnt + FROM topic t + JOIN repo_topic_latest rtl ON rtl.topic_id = t.id + GROUP BY t.id + ORDER BY cnt DESC + LIMIT 500 + """ + topic_rows = db.conn.execute(topic_sql).fetchall() + topics = [{"name": str(r["name"]), "count": int(r["cnt"])} for r in topic_rows] + + db.close() + + self.log("Stopping PM2 service...") + try: + subprocess.run(["pm2", "stop", self.pm2_app_name], check=False, stdout=subprocess.DEVNULL) + except Exception as e: + self.log(f"Warning: PM2 stop failed (might not be running): {e}") + + self.log("Swapping Database...") + if os.path.exists(self.db_path): + if os.path.exists(self.live_db_path): + os.remove(self.live_db_path) + shutil.copy2(self.db_path, self.live_db_path) + + index_path = "index.html" + if os.path.exists(index_path): + with open(index_path, "r", encoding="utf-8") as f: + html_content = f.read() + + html_content = re.sub(r'(id="totalRepos"[^>]*>).*?()', f"\\g<1>{formatted_total}\\g<2>", html_content) + + with open(index_path, "w", encoding="utf-8") as f: + f.write(html_content) + self.log(f"Updated {index_path} with {formatted_total} repos.") + + app_js_path = "static/app.js" + if os.path.exists(app_js_path): + with open(app_js_path, "r", encoding="utf-8") as f: + js_content = f.read() + + static_data = {"languages": languages, "topics": topics} + injection_code1 = f"const STATIC_DATA = {json.dumps(static_data)};" + + if "const STATIC_DATA =" in js_content: + js_content = re.sub(r"const STATIC_DATA = \{.*?\};", injection_code1, js_content, flags=re.DOTALL) + + with open(app_js_path, "w", encoding="utf-8") as f: + f.write(js_content) + self.log(f"Updated {app_js_path} with static lists.") + + self.log("Restarting PM2 service...") + try: + subprocess.run(["pm2", "restart", self.pm2_app_name], check=False, stdout=subprocess.DEVNULL) + except Exception as e: + self.log(f"Warning: PM2 restart failed (maybe not running): {e}") + self.log("Deployment complete.") def crawl(): @@ -285,4 +311,6 @@ def run_at_hours(func: Callable, hours_list: List[int]): if __name__ == "__main__": - run_at_hours(crawl, [0, 6, 12, 18]) + # token could come from an environment variable in real deployments + crawler = Crawler(TOKEN) + crawler.run_at_hours([0, 6, 12, 18]) From e71ac3bb36909207cacc5121b0ecbc24b6617f35 Mon Sep 17 00:00:00 2001 From: ichigirl Date: Wed, 4 Mar 2026 21:06:59 +0100 Subject: [PATCH 3/5] added ignored files --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b6aa6d7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +github_repos.db +github_repos.db-shm +github_repos.db-wal +__pycache__/ +venv/ \ No newline at end of file From 973ae993a681b55c642c2e7603c677723d9ffa50 Mon Sep 17 00:00:00 2001 From: ichigirl Date: Wed, 4 Mar 2026 21:07:09 +0100 Subject: [PATCH 4/5] finished OOP transition for crawler --- crawler.py | 112 +++++++++++------------------------------------------ db.py | 107 +++++++++++++++++++++++++------------------------- 2 files changed, 76 insertions(+), 143 deletions(-) diff --git a/crawler.py b/crawler.py index dd35f75..f675daa 100755 --- a/crawler.py +++ b/crawler.py @@ -123,7 +123,6 @@ def execute_query(self, query: str, cursor: str = None): raise Exception("Max retries exceeded.") - class Crawler: def __init__( self, @@ -222,95 +221,30 @@ def deploy_site(self): except Exception as e: self.log(f"Warning: PM2 restart failed (maybe not running): {e}") self.log("Deployment complete.") - - -def crawl(): - gh = GithubGraphQL(TOKEN) - db = RepoDB(DB_PATH) - current_min_stars = MIN_STARS - total_fetched = 0 - - log(f"Starting crawl for repos with >= {MIN_STARS} stars...") - - while True: - search_query = f"stars:>={current_min_stars} sort:stars-asc" - log(f"Querying batch: '{search_query}'") - - cursor = None - batch_repos = [] - has_next_page = True - - # Max 1000 results allowed by GitHub - while has_next_page: - time.sleep(0.1) - data = gh.execute_query(search_query, cursor) - search_data = data["search"] - - nodes = search_data["nodes"] - if not nodes: - break - - batch_repos.extend(nodes) - total_fetched += len(nodes) - - log(f" Fetched {len(nodes)} items. Total: {total_fetched}. Last star count: {nodes[-1]['stargazerCount']}") - - page_info = search_data["pageInfo"] - has_next_page = page_info["hasNextPage"] - cursor = page_info["endCursor"] - - if len(batch_repos) >= 1000: - break - - if not batch_repos: - log("No more results found.") - break - - last_repo_stars = batch_repos[-1]["stargazerCount"] - - if last_repo_stars == current_min_stars: - current_min_stars += 1 - else: - current_min_stars = last_repo_stars - db.upsert_from_github_nodes(batch_repos) - db.close() - deploy_site(DB_PATH) - - -def run_at_hours(func: Callable, hours_list: List[int]): - now = datetime.now(tz=timezone.utc) - current_h = now.hour - - if current_h in hours_list: - minutes_remaining = 0 - else: - sorted_hours = sorted(hours_list) - next_hour = next((h for h in sorted_hours if h > current_h), sorted_hours[0]) - - target = now.replace(hour=next_hour, minute=0, second=0, microsecond=0) - if next_hour <= current_h: - target += timedelta(days=1) - - minutes_remaining = int((target - now).total_seconds() / 60) - - log(f"Scheduler started for hours: {hours_list}") - log(f"Next crawl will start in approximately {minutes_remaining} minutes.") - last_run_hour = -1 - - while True: - current_hour = datetime.now(tz=timezone.utc).hour - - if current_hour in hours_list and current_hour != last_run_hour: - func() - last_run_hour = current_hour - - if current_hour not in hours_list: - last_run_hour = -1 - - time.sleep(30) + + def run_at_hours(self, hours: List[int]): + self.log(f"Starting crawler. Will run at hours: {hours}") + while True: + now = datetime.now() + if now.hour in hours: + self.log("Starting crawl cycle...") + try: + self.crawl_and_update() + self.deploy_site() + except Exception as e: + self.log(f"Error during crawl/deploy: {e}") + self.log("Cycle complete. Sleeping for 1 hour.") + time.sleep(3600) + else: + next_run = min((h for h in hours if h > now.hour), default=hours[0] + 24) + next_run_time = now.replace(hour=next_run % 24, minute=0, second=0, microsecond=0) + if next_run_time <= now: + next_run_time += timedelta(days=1) + sleep_seconds = (next_run_time - now).total_seconds() + self.log(f"Current hour {now.hour} not in target hours. Sleeping for {sleep_seconds/3600:.2f} hours until {next_run_time}.") + time.sleep(sleep_seconds) if __name__ == "__main__": - # token could come from an environment variable in real deployments crawler = Crawler(TOKEN) - crawler.run_at_hours([0, 6, 12, 18]) + crawler.run_at_hours([0, 6, 12, 18]) \ No newline at end of file diff --git a/db.py b/db.py index 3c9a219..07d0b0a 100755 --- a/db.py +++ b/db.py @@ -3,51 +3,6 @@ from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple -def iso_to_unix(ts: Optional[str]) -> Optional[int]: - if not ts: - return None - dt = datetime.fromisoformat(ts.replace("Z", "+00:00")) - return int(dt.timestamp()) - - -def unix_to_iso(ts: Optional[int]) -> Optional[str]: - if ts is None: - return None - return datetime.fromtimestamp(int(ts), tz=timezone.utc).isoformat().replace("+00:00", "Z") - - -def chunks(seq: Sequence[Any], n: int) -> Iterable[Sequence[Any]]: - for i in range(0, len(seq), n): - yield seq[i : i + n] - - -def row_to_obj(row: sqlite3.Row) -> Dict[str, Any]: - topics_concat = row["topicsConcat"] - topics = [] if topics_concat is None else str(topics_concat).split("\x1f") - topics = [t for t in topics if t] - - res = { - "n": row["nameWithOwner"], - "g": None if row["globalRank"] is None else int(row["globalRank"]), - "s": int(row["stargazerCount"]), - "f": int(row["forkCount"]), - "w": int(row["watchersCount"]), - "d": None if row["diskUsage"] is None else int(row["diskUsage"]), - "a": row["description"], - "h": row["homepageUrl"], - "c": unix_to_iso(row["createdAtUnix"]), - # "u": unix_to_iso(row["updatedAtUnix"]), - "p": unix_to_iso(row["pushedAtUnix"]), - "i": bool(int(row["isArchived"])), - "l": row["primaryLanguage"], - "t": topics, - # "x": unix_to_iso(row["fetchedAtUnix"]), - } - if "newStars" in row.keys() and row["newStars"]: - res["ns"] = int(row["newStars"]) - return res - - def select_latest_base_sql(include_global_rank: bool = True, extra_select: str = "") -> str: rank_select = "gr.globalRank AS globalRank," if include_global_rank else "NULL AS globalRank," rank_join = ( @@ -105,6 +60,50 @@ def count_base_sql() -> str: class RepoDB: + @staticmethod + def iso_to_unix(ts: Optional[str]) -> Optional[int]: + if not ts: + return None + dt = datetime.fromisoformat(ts.replace("Z", "+00:00")) + return int(dt.timestamp()) + + @staticmethod + def unix_to_iso(ts: Optional[int]) -> Optional[str]: + if ts is None: + return None + return datetime.fromtimestamp(int(ts), tz=timezone.utc).isoformat().replace("+00:00", "Z") + + @staticmethod + def chunks(seq: Sequence[Any], n: int) -> Iterable[Sequence[Any]]: + for i in range(0, len(seq), n): + yield seq[i : i + n] + + @staticmethod + def row_to_obj(row: sqlite3.Row) -> Dict[str, Any]: + topics_concat = row["topicsConcat"] + topics = [] if topics_concat is None else str(topics_concat).split("\x1f") + topics = [t for t in topics if t] + + res = { + "n": row["nameWithOwner"], + "g": None if row["globalRank"] is None else int(row["globalRank"]), + "s": int(row["stargazerCount"]), + "f": int(row["forkCount"]), + "w": int(row["watchersCount"]), + "d": None if row["diskUsage"] is None else int(row["diskUsage"]), + "a": row["description"], + "h": row["homepageUrl"], + "c": RepoDB.unix_to_iso(row["createdAtUnix"]), + # "u": RepoDB.unix_to_iso(row["updatedAtUnix"]), + "p": RepoDB.unix_to_iso(row["pushedAtUnix"]), + "i": bool(int(row["isArchived"])), + "l": row["primaryLanguage"], + "t": topics, + # "x": RepoDB.unix_to_iso(row["fetchedAtUnix"]), + } + if "newStars" in row.keys() and row["newStars"]: + res["ns"] = int(row["newStars"]) + return res def __init__(self, path: str) -> None: self.conn = sqlite3.connect(path, check_same_thread=False) self.conn.row_factory = sqlite3.Row @@ -252,7 +251,7 @@ def _fetch_repo_ids(self, names: List[str]) -> Dict[str, int]: out: Dict[str, int] = {} if not names: return out - for chunk in chunks(names, 500): + for chunk in self.chunks(names, 500): q = "SELECT id, name_with_owner FROM repo WHERE name_with_owner IN ({})".format(",".join(["?"] * len(chunk))) for row in self.conn.execute(q, tuple(chunk)).fetchall(): out[str(row["name_with_owner"])] = int(row["id"]) @@ -262,7 +261,7 @@ def _fetch_latest_metrics(self, repo_ids: List[int]) -> Dict[int, sqlite3.Row]: out: Dict[int, sqlite3.Row] = {} if not repo_ids: return out - for chunk in chunks(repo_ids, 500): + for chunk in self.chunks(repo_ids, 500): q = "SELECT repo_id, history_start_run_id, stars, forks, watchers, disk_usage FROM repo_latest WHERE repo_id IN ({})".format(",".join(["?"] * len(chunk))) for row in self.conn.execute(q, tuple(chunk)).fetchall(): out[int(row["repo_id"])] = row @@ -301,7 +300,7 @@ def upsert_from_github_nodes(self, nodes: List[Dict[str, Any]]) -> None: repo_id = int(n["databaseId"]) name = n["nameWithOwner"] repo_ids.append(repo_id) - repo_rows.append((repo_id, name, iso_to_unix(n.get("createdAt")), n.get("description"), n.get("homepageUrl"))) + repo_rows.append((repo_id, name, self.iso_to_unix(n.get("createdAt")), n.get("description"), n.get("homepageUrl"))) with self.conn: conflict_params = [(row[1], row[0]) for row in repo_rows] @@ -360,8 +359,8 @@ def upsert_from_github_nodes(self, nodes: List[Dict[str, Any]]) -> None: disk_usage = n.get("diskUsage") disk_usage_i: Optional[int] = None if disk_usage is None else int(disk_usage) - updated_at = iso_to_unix(n.get("updatedAt")) - pushed_at = iso_to_unix(n.get("pushedAt")) + updated_at = self.iso_to_unix(n.get("updatedAt")) + pushed_at = self.iso_to_unix(n.get("pushedAt")) is_archived = 1 if bool(n.get("isArchived")) else 0 pl_name = None @@ -473,7 +472,7 @@ def upsert_from_github_nodes(self, nodes: List[Dict[str, Any]]) -> None: ) if topic_repo_ids_to_refresh: - for chunk in chunks(topic_repo_ids_to_refresh, 500): + for chunk in self.chunks(topic_repo_ids_to_refresh, 500): q = "DELETE FROM repo_topic_latest WHERE repo_id IN ({})".format(",".join(["?"] * len(chunk))) self.conn.execute(q, tuple(chunk)) @@ -492,7 +491,7 @@ def get_repo_latest(self, name_with_owner: str) -> Optional[Dict[str, Any]]: """ ) row = self.conn.execute(q, (name_with_owner,)).fetchone() - return None if row is None else row_to_obj(row) + return None if row is None else self.row_to_obj(row) def _prepare_filter_conditions( self, @@ -693,8 +692,8 @@ def history_segments(self, name_with_owner: str, limit: int = 5000) -> List[Dict for r in rows: out.append( { - "startFetchedAt": unix_to_iso(r["startFetchedAtUnix"]), - "endFetchedAt": unix_to_iso(r["endFetchedAtUnix"]), + "startFetchedAt": self.unix_to_iso(r["startFetchedAtUnix"]), + "endFetchedAt": self.unix_to_iso(r["endFetchedAtUnix"]), "s": int(r["stars"]), "f": int(r["forks"]), "w": int(r["watchers"]), From 7b7f91d3a6a576f6dd01afd0eff6a19d00d9e818 Mon Sep 17 00:00:00 2001 From: ichigirl Date: Wed, 4 Mar 2026 21:08:03 +0100 Subject: [PATCH 5/5] finished OOP transition for db --- db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db.py b/db.py index 07d0b0a..9787b5d 100755 --- a/db.py +++ b/db.py @@ -590,7 +590,7 @@ def trending_leaderboard( all_params.extend([page_size, int(offset)]) rows = self.conn.execute(sql, tuple(all_params)).fetchall() - return [row_to_obj(r) for r in rows] + return [self.row_to_obj(r) for r in rows] def leaderboard( self, @@ -649,7 +649,7 @@ def leaderboard( params.extend([page_size, int(offset)]) rows = self.conn.execute(sql, tuple(params)).fetchall() - return [row_to_obj(r) for r in rows] + return [self.row_to_obj(r) for r in rows] def get_global_rank(self, name_with_owner: str) -> Optional[int]: sql = """