From dc4de29a7f0cefeda9a067a91eb837b49318db9f Mon Sep 17 00:00:00 2001 From: arena-agent Date: Sun, 11 Oct 2026 15:26:26 +0000 Subject: [PATCH 1/2] Auto-discover repos for profile totals; add CloakJS links - Count every public repo under johanbeytell1 and Infinitode automatically (no hand-maintained package/repo lists). - Downloads: PyPI packages verified as belonging to the repo, plus jsDelivr for every counted repo. - Add total stars, total forks and total clones badges. Clones are merged into data/clone-history.json because GitHub only exposes 14 days (needs TRAFFIC_TOKEN). - Link CloakJS in the changelog, status and projects sections. Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/workflows/update-download-badges.yml | 57 +- data/clone-history.json | 3 + profile/README.md | 10 +- scripts/test_update_download_badges.py | 745 +++++++++++-------- scripts/update-download-badges.py | 441 ++++++++--- 5 files changed, 834 insertions(+), 422 deletions(-) create mode 100644 data/clone-history.json diff --git a/.github/workflows/update-download-badges.yml b/.github/workflows/update-download-badges.yml index 649264a..ad4a284 100644 --- a/.github/workflows/update-download-badges.yml +++ b/.github/workflows/update-download-badges.yml @@ -1,21 +1,32 @@ -name: Update total downloads badge +name: Update profile totals -# Tallies PyPI + jsDelivr downloads and rewrites the TOTAL-DOWNLOADS badge -# block in profile/README.md. +# Tallies downloads, clones, stars and forks across the public repos of +# johanbeytell1 and Infinitode, and rewrites the TOTAL-DOWNLOADS badge block in +# profile/README.md. Repos are discovered automatically; nothing is listed by hand. # # Data sources (see scripts/update-download-badges.py for details): -# PyPI - pepy v2 API when PEPY_API_KEY is set (exact counts), otherwise -# the shields.io /pepy/dt badge (no key, rounded figures). -# jsDelivr - shields.io /jsdelivr/gh/hy badge = hits over the last 12 months. -# jsDelivr publishes period stats only; there is no all-time figure. +# Stars, forks - GitHub repo listing (exact, all-time). +# PyPI - packages that belong to a repo (verified by a link back to +# the repo or by the repo's packaging file). pepy v2 API when +# PEPY_API_KEY is set (exact), otherwise the shields.io +# /pepy/dt badge (rounded). +# jsDelivr - shields.io /jsdelivr/gh/hy badge for every repo; hits over +# the last 12 months (there is no all-time jsDelivr figure). +# Clones - GitHub traffic API, which only covers 14 days and needs +# push access. Daily counts are merged into +# data/clone-history.json so the total keeps growing. # -# Optional but recommended: add a repository secret named PEPY_API_KEY -# (free key: pepy.tech -> sign in -> API key). Without it the job still runs, -# using the rounded shields.io figures instead. +# Secrets (both optional; the job still runs without them): +# PEPY_API_KEY - exact PyPI counts (free key: pepy.tech -> sign in -> API key). +# TRAFFIC_TOKEN - a token with read access to repository traffic on every +# counted repo (fine-grained PAT: Administration read, on both +# owners' repos; or a classic PAT with the repo scope). Without +# it, clone totals are not refreshed. # -# Safety: a failed fetch never overwrites the badge. The job only writes when -# at least one source returned a number, and never publishes a total more than -# 5% below the figure already shown. +# Safety: a failed fetch never overwrites the published numbers. The job only +# writes when the repo listing succeeded and at least one download source +# returned a number. Download totals are never published more than 5% below the +# figure already shown. on: schedule: @@ -24,6 +35,7 @@ on: pull_request: paths: - "scripts/**" + - "data/**" - ".github/workflows/update-download-badges.yml" push: branches: [main] @@ -53,26 +65,31 @@ jobs: - name: Run the badge tests run: python3 -m unittest discover -s scripts -p 'test_*.py' -v - # Pull requests only run the tests above; the badge is rewritten on main. - - name: Tally downloads and rewrite the badge + # Pull requests only run the tests above; the badges are rewritten on main. + - name: Tally totals and rewrite the badges if: github.event_name != 'pull_request' env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PEPY_API_KEY: ${{ secrets.PEPY_API_KEY }} + TRAFFIC_TOKEN: ${{ secrets.TRAFFIC_TOKEN }} run: | if [ -z "$PEPY_API_KEY" ]; then echo "::warning::PEPY_API_KEY is not set; PyPI counts will be rounded shields.io figures." fi - python3 scripts/update-download-badges.py --readme profile/README.md + if [ -z "$TRAFFIC_TOKEN" ]; then + echo "::warning::TRAFFIC_TOKEN is not set; clone totals will not be refreshed." + fi + python3 scripts/update-download-badges.py --readme profile/README.md --history data/clone-history.json - - name: Commit the updated badge + - name: Commit the updated badges if: github.event_name != 'pull_request' run: | git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add profile/README.md + git add profile/README.md data/clone-history.json if git diff --cached --quiet; then - echo "Total downloads unchanged; nothing to commit." + echo "Totals unchanged; nothing to commit." else - git commit -m "chore: update total downloads badge [skip ci]" + git commit -m "chore: update profile totals [skip ci]" git push fi diff --git a/data/clone-history.json b/data/clone-history.json new file mode 100644 index 0000000..03be823 --- /dev/null +++ b/data/clone-history.json @@ -0,0 +1,3 @@ +{ + "repos": {} +} diff --git a/profile/README.md b/profile/README.md index 29943a3..846f948 100644 --- a/profile/README.md +++ b/profile/README.md @@ -10,6 +10,9 @@ ![](https://komarev.com/ghpvc/?username=infinitode&color=5d17eb) ![Total Downloads](https://img.shields.io/badge/Total%20Downloads-120.9k-5d17eb) +![Total Clones](https://img.shields.io/badge/Total%20Clones-n%2Fa-5d17eb) +![Total Stars](https://img.shields.io/badge/Total%20Stars-50-5d17eb) +![Total Forks](https://img.shields.io/badge/Total%20Forks-10-5d17eb) [![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/M4J423SRBM) @@ -30,7 +33,7 @@ Welcome to Infinitode, a cutting-edge small technology company dedicated to deve > Upcoming changes are planned changes. We can remove, cancel, or stage these changes for a later date. ### Latest changes -- Released `CloakJS`, a small JavaScript code obfuscator. +- Released [`CloakJS`](https://github.com/Infinitode/CloakJS), a small JavaScript code obfuscator. - Updated website with new pricing and MFA authentication available for verified emails and Google sign-ins. - Updated general website security and user account security. - Updated `Python Projects` with new projects. @@ -421,6 +424,10 @@ Welcome to Infinitode, a cutting-edge small technology company dedicated to deve ![jsDelivr hits (GitHub)](https://img.shields.io/jsdelivr/gh/hy/infinitode/blurjs) ![jsDelivr hits (GitHub)](https://img.shields.io/jsdelivr/gh/hm/infinitode/blurjs) ![GitHub Repo stars](https://img.shields.io/github/stars/infinitode/blurjs) ![GitHub last commit](https://img.shields.io/github/last-commit/infinitode/blurjs) +[**CloakJS**](https://github.com/Infinitode/CloakJS) + +![jsDelivr hits (GitHub)](https://img.shields.io/jsdelivr/gh/hy/infinitode/cloakjs) ![GitHub stars](https://img.shields.io/github/stars/infinitode/cloakjs) ![GitHub last commit](https://img.shields.io/github/last-commit/infinitode/cloakjs) + ## Our mission @@ -438,6 +445,7 @@ We're proud to showcase some of our current projects: * **CodeSafe**: An open-source Python library for code encryption, decryption, and safe evaluation using Python's built-in AST module, with allowed functions, variables, built-in imports, timeouts, and more. * **PyWebScrapr**: An open-source Python web scraping tool supporting image and text scraping for effortless data collection. * **FuncProfiler**: An open-source Python library for identifying bottlenecks in code, featuring function profiling, data exports, logging, and line-by-line profiling for more granular control. +* **CloakJS**: A single-file, dependency-free JavaScript obfuscator for the browser and Node.js, with identifier renaming, string encryption, and control-flow options. [View on GitHub](https://github.com/Infinitode/CloakJS). > [!TIP] > For more projects, see: https://github.com/Infinitode?tab=repositories diff --git a/scripts/test_update_download_badges.py b/scripts/test_update_download_badges.py index 25dbcec..1d79642 100644 --- a/scripts/test_update_download_badges.py +++ b/scripts/test_update_download_badges.py @@ -1,17 +1,19 @@ #!/usr/bin/env python3 """Offline tests for scripts/update-download-badges.py. -The download APIs are unreachable from CI sandboxes, so the network layer is -stubbed here with payloads shaped like the real pepy JSON and shields.io SVG -responses. Everything else -- count extraction, badge parsing, summing, -formatting, the regression guards and the README rewrite -- is production code. +The network layer is replaced with a router that answers with payloads shaped +like the real GitHub, PyPI, pepy and shields.io responses. Everything else -- +repo discovery and ownership checks, counting, clone merging, badge parsing, +the regression guards and the README rewrite -- is production code. Run with: python3 -m unittest discover -s scripts -p 'test_*.py' -v """ from __future__ import annotations +import base64 import io +import json import os import sys import tempfile @@ -27,6 +29,8 @@ udb = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(udb) +TOKEN_VARS = ("PEPY_API_KEY", "GITHUB_TOKEN", "TRAFFIC_TOKEN") + def shields_svg(label, value): """A shields.io flat-style badge, including the duplicated shadow text.""" @@ -41,348 +45,475 @@ def shields_svg(label, value): """ -class StubJson: - def __init__(self, responses): - self.responses = responses - self.calls = [] - - def __call__(self, url, headers=None, timeout=30): - self.calls.append((url, headers or {})) - for needle, payload in self.responses.items(): - if needle in url: - if isinstance(payload, Exception): - raise payload - return payload - raise urllib.error.URLError(f"unstubbed json url: {url}") - - -class StubText: - def __init__(self, responses): - self.responses = responses - self.calls = [] - - def __call__(self, url, headers=None, timeout=30): - self.calls.append((url, headers or {})) - for needle, payload in self.responses.items(): - if needle in url: - if isinstance(payload, Exception): - raise payload - return payload - raise urllib.error.URLError(f"unstubbed text url: {url}") - - -PEPY_JSON = {"name": "codesafe", "total_downloads": 1234, "recent_downloads": {"last_month": 90}} -BADGE_SVG = shields_svg("downloads", "1.2k") - - -class TestShieldsBadgeParsing(unittest.TestCase): - def test_reads_the_value_not_the_label(self): - self.assertEqual(udb.parse_shields_badge(BADGE_SVG), 1200) - - def test_thousands_separator(self): - self.assertEqual(udb.parse_shields_badge(shields_svg("hits", "1,234")), 1234) - - def test_mega_suffix(self): - self.assertEqual(udb.parse_shields_badge(shields_svg("downloads", "2.5M")), 2_500_000) - - def test_plain_integer(self): - self.assertEqual(udb.parse_shields_badge(shields_svg("hits", "999")), 999) - - def test_error_badge_yields_none(self): +def repo(owner, name, stars=0, forks=0, private=False): + """A repository object as the GitHub listing API returns it (trimmed).""" + return { + "name": name, + "full_name": f"{owner}/{name}", + "owner": {"login": owner}, + "private": private, + "visibility": "private" if private else "public", + "stargazers_count": stars, + "forks_count": forks, + } + + +def pypi_meta(home=None, description=""): + return { + "info": { + "home_page": home, + "download_url": None, + "project_url": None, + "project_urls": None, + "description": description, + } + } + + +def contents_file(name, text): + """What GET /repos/{o}/{r}/contents/{file} returns for a file.""" + return {"content": base64.b64encode(text.encode("utf-8")).decode("ascii"), "name": name} + + +def http_error(code): + return urllib.error.HTTPError("http://test", code, "test", None, None) + + +class FakeWeb: + """Routes each URL to the most specific fragment that matches it. + + A route value is a payload, an exception to raise, or a callable taking the + URL and returning either of those. + """ + + def __init__(self): + self.json = {} + self.text = {} + self.json_calls = [] + + def fetch_json(self, url, headers=None, timeout=30): + self.json_calls.append((url, headers or {})) + return self._route(self.json, url, headers) + + def fetch_text(self, url, headers=None, timeout=30): + return self._route(self.text, url, headers) + + @staticmethod + def _route(table, url, headers): + matches = [needle for needle in table if needle in url] + if not matches: + raise urllib.error.URLError(f"unstubbed url: {url}") + value = table[max(matches, key=len)] + if callable(value): + value = value(url, headers) + if isinstance(value, Exception): + raise value + return value + + +def page_of(url): + """The ``page=`` query value of a listing URL (1 when absent).""" + for part in url.split("?", 1)[-1].split("&"): + if part.startswith("page="): + return int(part.split("=", 1)[1]) + return 1 + + +def listing(repos): + """A repo listing that serves everything on page 1 and nothing after.""" + return lambda url, headers: repos if page_of(url) == 1 else [] + + +def standard_world(): + """Two owners, one private repo, one foreign repo, and packages of every kind.""" + web = FakeWeb() + web.json.update( + { + "/users/johanbeytell1/repos": listing( + [ + repo("johanbeytell1", "LocalShare", stars=1), # PyPI name taken by someone else + repo("johanbeytell1", "Alloy", stars=1), + repo("johanbeytell1", "Secret", stars=99, private=True), + ] + ), + "/orgs/Infinitode/repos": listing( + [ + repo("Infinitode", "Hued", stars=2, forks=1), + repo("Infinitode", "qrforge"), + repo("Infinitode", "CloakJS"), + repo("someone-else", "hued-fork", stars=50), # not ours, even if listed + ] + ), + # PyPI: hued links back to our repo; qrforge only declares its name in setup.py; + # localshare is someone else's project; alloy and cloakjs are not on PyPI. + "pypi.org/pypi/hued/json": pypi_meta(home="https://github.com/Infinitode/Hued"), + "pypi.org/pypi/qrforge/json": pypi_meta(description="# qrforge\nNo links here."), + "pypi.org/pypi/localshare/json": pypi_meta(home="https://github.com/sunu/localshare"), + "pypi.org/pypi/alloy/json": http_error(404), + "pypi.org/pypi/cloakjs/json": http_error(404), + "/repos/Infinitode/qrforge/contents/setup.py": contents_file( + "setup.py", "setup(name='qrforge', version='1.0.1')" + ), + "/repos/Infinitode/qrforge/contents": [ + {"type": "file", "name": "setup.py", "url": "https://api.github.com/repos/Infinitode/qrforge/contents/setup.py"}, + {"type": "dir", "name": "tests", "url": "https://api.github.com/x"}, + ], + "/repos/johanbeytell1/LocalShare/contents": [{"type": "file", "name": "readme.md", "url": "x"}], + "/repos/Infinitode/Hued/contents": http_error(404), + } + ) + web.json["/repos/johanbeytell1/Alloy/contents"] = [] + web.json["/repos/Infinitode/CloakJS/contents"] = [] + return web + + +def standard_text_world(web): + web.text.update( + { + "img.shields.io/pepy/dt/": lambda url, headers: shields_svg("downloads", "1.0k"), + "img.shields.io/jsdelivr/gh/hy/infinitode/cloakjs": shields_svg("hits", "250"), + "img.shields.io/jsdelivr/gh/hy/": shields_svg("hits", "0"), + } + ) + return web + + +class StubMixin: + """Points the module's network layer at a FakeWeb for the duration of a test.""" + + def install(self, web): + self._saved = (udb.fetch_json, udb.fetch_text) + udb.fetch_json, udb.fetch_text = web.fetch_json, web.fetch_text + self.addCleanup(self._restore) + + def _restore(self): + udb.fetch_json, udb.fetch_text = self._saved + + +# --- parsers and helpers ------------------------------------------------------ + + +class TestParsers(unittest.TestCase): + def test_shields_badge_reads_the_value(self): + self.assertEqual(udb.parse_shields_badge(shields_svg("hits", "5.6k")), 5600) + self.assertEqual(udb.parse_shields_badge(shields_svg("downloads", "1,234")), 1234) + + def test_shields_error_badge_has_no_value(self): self.assertIsNone(udb.parse_shields_badge(shields_svg("downloads", "unavailable"))) - - def test_empty_input_yields_none(self): self.assertIsNone(udb.parse_shields_badge("")) - self.assertIsNone(udb.parse_shields_badge(None)) - - def test_title_element_is_not_mistaken_for_the_value(self): - svg = shields_svg("downloads", "42") - self.assertIn("downloads: 42", svg) - self.assertEqual(udb.parse_shields_badge(svg), 42) - - -class TestCountExtraction(unittest.TestCase): - def test_finds_pepy_total_downloads(self): - self.assertEqual(udb.find_count(PEPY_JSON), 1234) - - def test_finds_jsdelivr_total_hits(self): - self.assertEqual(udb.find_count({"type": "github", "totalHits": 5678}), 5678) - def test_prefers_all_time_key_over_period_hits(self): - self.assertEqual(udb.find_count({"hits": {"lastMonth": 50}, "totalHits": 999}), 999) - - def test_returns_none_when_absent_or_non_numeric(self): - self.assertIsNone(udb.find_count({"name": "nothing"})) - self.assertIsNone(udb.find_count({"downloads": True})) - - -class TestFormatting(unittest.TestCase): - def test_boundaries(self): - cases = [ - (0, "0"), (999, "999"), (1_000, "1.0k"), (45_678, "45.7k"), - (999_949, "999.9k"), (999_950, "1.00M"), (1_234_567, "1.23M"), - (1_000_000_000, "1.00B"), (12_300_000_000, "12.30B"), - ] - for value, expected in cases: - self.assertEqual(udb.format_count(value), expected, f"format_count({value})") - - def test_never_renders_1000k(self): - self.assertNotIn("1000.0k", [udb.format_count(n) for n in range(990_000, 1_010_000, 997)]) - - def test_value_is_url_safe(self): - for value in (0, 999, 45_678, 999_950, 1_234_567): - self.assertRegex(udb.format_count(value), r"^[0-9.]+[kMB]?$") - - def test_round_trips_through_parse_badge_value(self): - for value in (0, 999, 45_678, 999_950, 1_234_567, 12_300_000_000): - badge = udb.BADGE.format(value=udb.format_count(value)) - parsed = udb.parse_badge_value(badge) - # rounded display loses precision, but must stay within 0.5% - self.assertAlmostEqual(parsed, value, delta=max(1, value * 0.005)) - - -class TestPypiTotals(unittest.TestCase): - def test_uses_the_pepy_api_when_a_key_is_set(self): - fetch_json = StubJson({"api.pepy.tech": PEPY_JSON}) - totals, failures, notes = udb.pypi_totals( - ["codesafe"], api_key="secret", fetch_json=fetch_json, fetch_text=StubText({}) - ) - self.assertEqual(totals, {"codesafe": 1234}) - self.assertEqual(failures, {}) - self.assertEqual(notes, []) - self.assertEqual(fetch_json.calls[0][1], {"X-Api-Key": "secret"}) - - def test_falls_back_to_shields_without_a_key(self): - fetch_text = StubText({"img.shields.io": BADGE_SVG}) - totals, failures, notes = udb.pypi_totals( - ["codesafe"], api_key=None, fetch_json=StubJson({}), fetch_text=fetch_text - ) - self.assertEqual(totals, {"codesafe": 1200}) - self.assertEqual(failures, {}) - self.assertTrue(any("PEPY_API_KEY" in n for n in notes)) - - def test_falls_back_to_shields_when_the_pepy_api_rejects_the_key(self): - fetch_json = StubJson({"api.pepy.tech": urllib.error.HTTPError("u", 403, "Forbidden", {}, None)}) - totals, _, _ = udb.pypi_totals( - ["codesafe"], api_key="bad", fetch_json=fetch_json, fetch_text=StubText({"img.shields.io": BADGE_SVG}) - ) - self.assertEqual(totals, {"codesafe": 1200}) - - def test_records_failure_when_both_sources_fail(self): - totals, failures, _ = udb.pypi_totals( - ["codesafe"], - api_key="k", - fetch_json=StubJson({"api.pepy.tech": urllib.error.HTTPError("u", 403, "Forbidden", {}, None)}), - fetch_text=StubText({"img.shields.io": shields_svg("downloads", "unavailable")}), - ) - self.assertEqual(totals, {}) - self.assertIn("codesafe", failures) - - -class TestJsdelivrTotals(unittest.TestCase): - def test_reads_hits_from_the_badge(self): - fetch_text = StubText({"img.shields.io": shields_svg("hits", "5.6k")}) - totals, failures = udb.jsdelivr_totals(["infinitode/blurjs"], fetch_text=fetch_text) - self.assertEqual(totals, {"infinitode/blurjs": 5600}) - self.assertEqual(failures, {}) - self.assertIn("/jsdelivr/gh/hy/infinitode/blurjs", fetch_text.calls[0][0]) - - def test_records_failure_for_an_error_badge(self): - totals, failures = udb.jsdelivr_totals( - ["infinitode/blurjs"], - fetch_text=StubText({"img.shields.io": shields_svg("hits", "unavailable")}), - ) - self.assertEqual(totals, {}) - self.assertIn("infinitode/blurjs", failures) + def test_format_count(self): + self.assertEqual(udb.format_count(999), "999") + self.assertEqual(udb.format_count(45_678), "45.7k") + self.assertEqual(udb.format_count(1_234_567), "1.23M") + self.assertEqual(udb.format_count(999_999), "1.00M") # never "1000.0k" + self.assertEqual(udb.format_count(2_000_000_000), "2.00B") + + def test_find_count_prefers_exact_all_time_keys(self): + self.assertEqual(udb.find_count({"downloads": {"last_month": 5}, "total_downloads": 9}), 9) + self.assertIsNone(udb.find_count({"nothing": "here"})) + + def test_parse_badge_value_reads_downloads_only(self): + text = udb.render_block([("Total Downloads", "5.17M"), ("Total Stars", "50")]) + self.assertEqual(udb.parse_badge_value(text), 5_170_000) + self.assertIsNone(udb.parse_badge_value("no badge here")) + + def test_normalize_name_follows_pep503(self): + self.assertEqual(udb.normalize_name("Py_AutoPlot"), "py-autoplot") + self.assertEqual(udb.normalize_name("LocalSiteMap"), "localsitemap") + + def test_is_counted_keeps_only_public_repos_of_the_owner(self): + self.assertTrue(udb.is_counted(repo("Infinitode", "Hued"), "infinitode")) + self.assertFalse(udb.is_counted(repo("Infinitode", "Hued", private=True), "Infinitode")) + self.assertFalse(udb.is_counted(repo("someone-else", "hued-fork"), "Infinitode")) + + def test_links_back_needs_the_exact_repo(self): + self.assertTrue(udb.links_back(pypi_meta(home="https://github.com/Infinitode/Hued"), "Infinitode", "Hued")) + self.assertFalse(udb.links_back(pypi_meta(home="https://github.com/sunu/localshare"), "johanbeytell1", "LocalShare")) + # a longer repo name must not count as a match for the shorter one + self.assertFalse(udb.links_back(pypi_meta(home="https://github.com/infinitode/localsitemap-tools"), "infinitode", "localsitemap")) + + def test_packaging_names(self): + self.assertEqual(udb.packaging_names("setup.py", "setup(name='qrforge', version='1')"), {"qrforge"}) + self.assertEqual(udb.packaging_names("pyproject.toml", '[project]\nname = "hued"\n'), {"hued"}) + self.assertEqual(udb.packaging_names("setup.cfg", "[metadata]\nname = valx\n"), {"valx"}) + + +class TestClones(unittest.TestCase): + def test_days_are_overwritten_not_added(self): + history = {"repos": {}} + udb.merge_clones(history, "Infinitode/Hued", {"2026-10-10": 3, "2026-10-11": 1}) + udb.merge_clones(history, "Infinitode/Hued", {"2026-10-10": 5, "2026-10-11": 1}) + self.assertEqual(udb.history_total(history), 6) + + def test_zero_days_are_dropped_and_empty_repos_removed(self): + history = {"repos": {}} + udb.merge_clones(history, "Infinitode/Hued", {"2026-10-10": 2}) + udb.merge_clones(history, "Infinitode/Hued", {"2026-10-10": 0}) + self.assertEqual(history, {"repos": {}}) + + def test_totals_accumulate_across_repos(self): + history = {"repos": {}} + udb.merge_clones(history, "Infinitode/Hued", {"2026-10-10": 4}) + udb.merge_clones(history, "johanbeytell1/Alloy", {"2026-10-09": 1}) + self.assertEqual(udb.history_total(history), 5) class TestReadmeRewrite(unittest.TestCase): def setUp(self): self.readme = f"# Profile\n\n{udb.ANCHOR}\n\nSome text.\n" + self.values = [ + ("Total Downloads", "1.23M"), + ("Total Clones", "n%2Fa"), + ("Total Stars", "50"), + ("Total Forks", "10"), + ] - def test_inserts_after_visitor_counter_on_first_run(self): - updated, changed = udb.update_readme(self.readme, 1_234_567) + def test_inserts_all_badges_after_visitor_counter_on_first_run(self): + updated, changed = udb.update_readme(self.readme, self.values) self.assertTrue(changed) self.assertIn("Total%20Downloads-1.23M-", updated) + self.assertIn("Total%20Clones-n%2Fa-", updated) + self.assertIn("Total%20Stars-50-", updated) + self.assertIn("Total%20Forks-10-", updated) self.assertLess(updated.index(udb.ANCHOR), updated.index(udb.START_MARK)) def test_replaces_in_place_on_later_runs(self): - once, _ = udb.update_readme(self.readme, 1_000) - twice, changed = udb.update_readme(once, 2_000_000) + once, _ = udb.update_readme(self.readme, self.values) + twice, changed = udb.update_readme(once, [("Total Downloads", "2.00M"), ("Total Stars", "51")]) self.assertTrue(changed) self.assertEqual(twice.count("TOTAL-DOWNLOADS:START"), 1) - self.assertIn("2.00M", twice) + self.assertIn("Total%20Downloads-2.00M-", twice) + self.assertIn("Total%20Stars-51-", twice) + self.assertNotIn("Total%20Forks", twice, "the block is rewritten as a whole") - def test_idempotent_for_same_value(self): - once, _ = udb.update_readme(self.readme, 1234) - twice, changed = udb.update_readme(once, 1234) + def test_idempotent_for_same_values(self): + once, _ = udb.update_readme(self.readme, self.values) + twice, changed = udb.update_readme(once, self.values) self.assertFalse(changed) self.assertEqual(once, twice) def test_no_anchor_and_no_marker_is_a_noop(self): - updated, changed = udb.update_readme("nothing here\n", 1) + updated, changed = udb.update_readme("nothing here\n", self.values) self.assertFalse(changed) + self.assertEqual(updated, "nothing here\n") + + def test_current_downloads_text_reads_the_published_value(self): + text, _ = udb.update_readme(self.readme, self.values) + self.assertEqual(udb.current_downloads_text(text), "1.23M") + self.assertIsNone(udb.current_downloads_text("no badge")) -class TestMain(unittest.TestCase): - """Drives the real main() against a throwaway copy of the README.""" +# --- discovery ---------------------------------------------------------------- + +class TestDiscovery(StubMixin, unittest.TestCase): def setUp(self): - self.readme = f"# Profile\n\n{udb.ANCHOR}\n" - self.path = self._write(self.readme) - - def _write(self, text): - handle = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8") - handle.write(text) - handle.close() - self.addCleanup(os.unlink, handle.name) - return handle.name - - def _read(self): - with open(self.path, encoding="utf-8") as handle: - return handle.read() + self.web = standard_world() + self.install(self.web) + + def test_lists_only_public_repos_of_the_two_owners(self): + names = sorted(r["full_name"] for r in udb.discover_repos()) + self.assertEqual( + names, + ["Infinitode/CloakJS", "Infinitode/Hued", "Infinitode/qrforge", "johanbeytell1/Alloy", "johanbeytell1/LocalShare"], + ) + + def test_listing_follows_pages(self): + many = [repo("Infinitode", f"r{i}") for i in range(100)] + self.web.json["/orgs/Infinitode/repos"] = lambda url, headers: many if page_of(url) == 1 else [repo("Infinitode", "last")] + repos = udb.list_public_repos("Infinitode", "org", fetch_json=self.web.fetch_json) + self.assertEqual(len(repos), 101) + + def test_sends_the_github_token_only_when_given(self): + udb.discover_repos(token="tok") + self.assertTrue(all(h.get("Authorization") == "Bearer tok" for _, h in self.web.json_calls)) + + def test_pypi_package_must_belong_to_the_repo(self): + pairs = { + "Infinitode/Hued": "hued", # linked from PyPI metadata + "Infinitode/qrforge": "qrforge", # declared by the repo's setup.py + "johanbeytell1/LocalShare": None, # PyPI "localshare" is someone else's + "johanbeytell1/Alloy": None, # not on PyPI at all + } + for full_name, expected in pairs.items(): + owner, name = full_name.split("/") + found = udb.pypi_package_for(repo(owner, name), fetch_json=self.web.fetch_json) + self.assertEqual(found, expected, full_name) + + def test_discover_pypi_maps_repos_to_packages(self): + repos = udb.discover_repos() + packages, notes = udb.discover_pypi(repos, fetch_json=self.web.fetch_json) + self.assertEqual(packages, {"Infinitode/Hued": "hued", "Infinitode/qrforge": "qrforge"}) + self.assertEqual(notes, []) + + def test_pypi_server_error_is_a_note_not_a_crash(self): + self.web.json["pypi.org/pypi/hued/json"] = http_error(500) + packages, notes = udb.discover_pypi(udb.discover_repos(), fetch_json=self.web.fetch_json) + self.assertNotIn("Infinitode/Hued", packages) + self.assertTrue(any("Infinitode/Hued" in n for n in notes)) + + +# --- collection --------------------------------------------------------------- + - def _run(self, argv, json_responses=None, text_responses=None, api_key=None): - # text_responses is keyed by URL fragment: "pepy" or "jsdelivr". - if api_key is None: - os.environ.pop("PEPY_API_KEY", None) - else: - os.environ["PEPY_API_KEY"] = api_key - self.addCleanup(os.environ.pop, "PEPY_API_KEY", None) +class TestCollect(StubMixin, unittest.TestCase): + def setUp(self): + self.web = standard_text_world(standard_world()) + self.web.json["api.pepy.tech"] = lambda url, headers: {"total_downloads": 1_000} + self.install(self.web) + + def test_stars_and_forks_are_summed_over_counted_repos(self): + result = udb.collect(api_key="k", fetch_json=self.web.fetch_json, fetch_text=self.web.fetch_text) + self.assertEqual(result["stars"], 1 + 1 + 2) # LocalShare, Alloy, Hued; Secret and the foreign fork excluded + self.assertEqual(result["forks"], 1) + + def test_downloads_come_from_pypi_and_jsdelivr(self): + result = udb.collect(api_key="k", fetch_json=self.web.fetch_json, fetch_text=self.web.fetch_text) + self.assertEqual(result["pypi"], {"hued": 1_000, "qrforge": 1_000}) + self.assertEqual(result["jsdelivr"]["infinitode/cloakjs"], 250) + self.assertEqual(result["downloads"], 1_000 + 1_000 + 250) + self.assertNotIn("localshare", result["pypi"]) + + def test_clones_stay_unknown_without_a_traffic_token(self): + history = {"repos": {}} + result = udb.collect(api_key="k", history=history, fetch_json=self.web.fetch_json, fetch_text=self.web.fetch_text) + self.assertIsNone(result["clones"]) + self.assertTrue(any("TRAFFIC_TOKEN" in n for n in result["notes"])) + + def test_clone_history_is_merged_per_repo(self): + self.web.json["traffic/clones"] = lambda url, headers: { + "count": 9, + "clones": [{"timestamp": "2026-10-10T00:00:00Z", "count": 3}, {"timestamp": "2026-10-11T00:00:00Z", "count": 0}], + } + history = {"repos": {}} + result = udb.collect(api_key="k", traffic_token="t", history=history, fetch_json=self.web.fetch_json, fetch_text=self.web.fetch_text) + self.assertEqual(result["clones"], 3 * len(result["repos"])) + self.assertEqual(history["repos"]["Infinitode/Hued"], {"2026-10-10": 3}) + traffic_calls = [h for url, h in self.web.json_calls if "traffic/clones" in url] + self.assertTrue(all(h.get("Authorization") == "Bearer t" for h in traffic_calls)) + + def test_one_repo_failing_its_clone_fetch_keeps_the_others(self): + def clones(url, headers): + if "/Hued/" in url: + raise http_error(403) + return {"clones": [{"timestamp": "2026-10-10T00:00:00Z", "count": 2}]} + + self.web.json["traffic/clones"] = clones + history = {"repos": {}} + result = udb.collect(traffic_token="t", history=history, fetch_json=self.web.fetch_json, fetch_text=self.web.fetch_text) + self.assertIn("Infinitode/Hued", result["failures"]) + self.assertNotIn("Infinitode/Hued", history["repos"]) + self.assertIn("Infinitode/CloakJS", history["repos"]) + + +# --- entry point -------------------------------------------------------------- + + +class TestMain(StubMixin, unittest.TestCase): + """Drives the real main() against a throwaway README and clone history.""" - udb.fetch_json = StubJson(json_responses or {}) - udb.fetch_text = StubText(text_responses or {}) + def setUp(self): + self.readme_text = f"# Profile\n\n{udb.ANCHOR}\n" + self.dir = tempfile.TemporaryDirectory() + self.addCleanup(self.dir.cleanup) + self.readme = os.path.join(self.dir.name, "README.md") + self.history = os.path.join(self.dir.name, "clone-history.json") + self._write(self.readme, self.readme_text) + self._write(self.history, json.dumps({"repos": {}})) + saved = {var: os.environ.pop(var) for var in TOKEN_VARS if var in os.environ} + self.addCleanup(os.environ.update, saved) + self.addCleanup(lambda: [os.environ.pop(var, None) for var in TOKEN_VARS]) + + @staticmethod + def _write(path, text): + with open(path, "w", encoding="utf-8") as handle: + handle.write(text) + + def _read(self, path): + with open(path, encoding="utf-8") as handle: + return handle.read() + + def _run(self, web, argv=(), env=None): + self.install(web) + for var, value in (env or {}).items(): + os.environ[var] = value buffer = io.StringIO() - original_stdout, sys.stdout = sys.stdout, buffer + original_stdout, original_stderr = sys.stdout, sys.stderr + sys.stdout = sys.stderr = buffer try: - code = udb.main(argv) + code = udb.main([*argv, "--readme", self.readme, "--history", self.history]) finally: - sys.stdout = original_stdout + sys.stdout, sys.stderr = original_stdout, original_stderr return code, buffer.getvalue() - def test_all_sources_failing_leaves_the_badge_untouched(self): - """The regression that shipped: a total wipeout must never publish 0.""" - dead = shields_svg("downloads", "unavailable") - code, _ = self._run( - ["--readme", self.path], - text_responses={"pepy": dead, "jsdelivr": shields_svg("hits", "unavailable")}, - ) + def _healthy_web(self): + web = standard_text_world(standard_world()) + web.json["api.pepy.tech"] = lambda url, headers: {"total_downloads": 1_000} + return web + + def test_listing_failure_changes_nothing(self): + web = self._healthy_web() + web.json["/orgs/Infinitode/repos"] = http_error(503) + code, _ = self._run(web) self.assertEqual(code, 1) - self.assertEqual(self._read(), self.readme, "README must stay byte-identical") - - def test_published_zero_is_treated_as_unpublished(self): - """A bogus 0 left by a failed run must not block the next good one.""" - with open(self.path, "w", encoding="utf-8") as handle: - handle.write(udb.update_readme(self.readme, 0)[0]) - self.assertIn("Total%20Downloads-0-", self._read()) - - code, _ = self._run( - ["--readme", self.path], - json_responses={pkg: {"total_downloads": 1_000} for pkg in udb.PYPI_PACKAGES}, - text_responses={"jsdelivr": shields_svg("hits", "0")}, - api_key="k", - ) - self.assertEqual(code, 0) - self.assertNotIn("Total%20Downloads-0-", self._read()) - self.assertIn(udb.format_count(1_000 * len(udb.PYPI_PACKAGES)), self._read()) - - def test_lower_total_is_refused(self): - with open(self.path, "w", encoding="utf-8") as handle: - handle.write(udb.update_readme(self.readme, 10_000_000)[0]) - code, _ = self._run( - ["--readme", self.path], - json_responses={pkg: {"total_downloads": 100} for pkg in udb.PYPI_PACKAGES}, - text_responses={"jsdelivr": shields_svg("hits", "0")}, - api_key="k", - ) - self.assertEqual(code, 0) - self.assertIn("10.00M", self._read()) - - def test_rounding_level_decrease_still_writes(self): - """"5.17M" stands for [5,165,000, 5,175,000); 5,168,400 is not a drop.""" - with open(self.path, "w", encoding="utf-8") as handle: - handle.write(udb.update_readme(self.readme, 5_170_000)[0]) - self.assertIn("5.17M", self._read()) - - per_package = 5_168_400 // len(udb.PYPI_PACKAGES) - total = per_package * len(udb.PYPI_PACKAGES) - self.assertLess(total, 5_170_000) # genuinely below the published figure - self.assertGreater(total, 5_170_000 * 0.95) # but inside the tolerance - - code, output = self._run( - ["--readme", self.path], - json_responses={pkg: {"total_downloads": per_package} for pkg in udb.PYPI_PACKAGES}, - text_responses={"jsdelivr": shields_svg("hits", "0")}, - api_key="k", - ) - self.assertEqual(code, 0) - self.assertNotIn("Refusing", output, "rounding noise must not trip the guard") - self.assertIn(udb.format_count(total), self._read()) - - def test_drop_beyond_tolerance_is_refused(self): - with open(self.path, "w", encoding="utf-8") as handle: - handle.write(udb.update_readme(self.readme, 10_000_000)[0]) - # ~40% drop -- one or more sources missing - code, _ = self._run( - ["--readme", self.path], - json_responses={pkg: {"total_downloads": 400_000} for pkg in udb.PYPI_PACKAGES}, - text_responses={"jsdelivr": shields_svg("hits", "0")}, - api_key="k", - ) - self.assertEqual(code, 0) - self.assertIn("10.00M", self._read()) - - def test_allow_decrease_overrides_the_guard(self): - with open(self.path, "w", encoding="utf-8") as handle: - handle.write(udb.update_readme(self.readme, 10_000_000)[0]) - code, _ = self._run( - ["--readme", self.path, "--allow-decrease"], - json_responses={pkg: {"total_downloads": 100} for pkg in udb.PYPI_PACKAGES}, - text_responses={"jsdelivr": shields_svg("hits", "0")}, - api_key="k", - ) + self.assertEqual(self._read(self.readme), self.readme_text) + + def test_every_download_source_failing_changes_nothing(self): + web = standard_world() + web.json["api.pepy.tech"] = http_error(500) + web.text["img.shields.io"] = urllib.error.URLError("offline") + code, _ = self._run(web, env={"PEPY_API_KEY": "k"}) + self.assertEqual(code, 1) + self.assertEqual(self._read(self.readme), self.readme_text) + + def test_happy_path_writes_all_four_badges(self): + web = self._healthy_web() + web.json["traffic/clones"] = {"clones": [{"timestamp": "2026-10-10T00:00:00Z", "count": 7}]} + code, output = self._run(web, env={"PEPY_API_KEY": "k", "TRAFFIC_TOKEN": "t"}) + self.assertEqual(code, 0, output) + readme = self._read(self.readme) + # 2 packages x 1,000 + jsDelivr 0 for each repo except CloakJS (1.0k shields figure is the default) + # PyPI: hued + qrforge at 1,000 each; jsDelivr: CloakJS 250, the rest 0. + self.assertIn(f"Total%20Downloads-{udb.format_count(2_250)}-", readme) + self.assertIn("Total%20Stars-4-", readme) # LocalShare 1 + Alloy 1 + Hued 2 + self.assertIn("Total%20Forks-1-", readme) + self.assertIn(f"Total%20Clones-{udb.format_count(7 * 5)}-", readme) # 7 clones x 5 repos + self.assertEqual(json.loads(self._read(self.history))["repos"]["Infinitode/Hued"], {"2026-10-10": 7}) + + def test_dry_run_changes_nothing(self): + code, output = self._run(self._healthy_web(), argv=["--dry-run"], env={"PEPY_API_KEY": "k"}) self.assertEqual(code, 0) - expected = 100 * len(udb.PYPI_PACKAGES) - self.assertIn(udb.format_count(expected), self._read()) - - def test_partial_failure_still_publishes_the_rest(self): - responses = {pkg: {"total_downloads": 1_000} for pkg in udb.PYPI_PACKAGES} - responses["qrforge"] = urllib.error.URLError("boom") - code, _ = self._run( - ["--readme", self.path], - json_responses=responses, - text_responses={ - "pepy": shields_svg("downloads", "unavailable"), # fallback also dead - "jsdelivr": shields_svg("hits", "4k"), - }, - api_key="k", - ) + self.assertIn("--dry-run", output) + self.assertEqual(self._read(self.readme), self.readme_text) + + def test_drop_beyond_tolerance_keeps_the_published_downloads(self): + self._write(self.readme, udb.update_readme(self.readme_text, [("Total Downloads", "10.00M")])[0]) + code, output = self._run(self._healthy_web(), env={"PEPY_API_KEY": "k"}) self.assertEqual(code, 0) - expected = 1_000 * (len(udb.PYPI_PACKAGES) - 1) + 4_000 - self.assertIn(udb.format_count(expected), self._read()) - - def test_happy_path_writes_the_tally(self): - code, output = self._run( - ["--readme", self.path], - json_responses={pkg: {"total_downloads": 1_000_000} for pkg in udb.PYPI_PACKAGES}, - text_responses={"jsdelivr": shields_svg("hits", "400k")}, - api_key="k", - ) + self.assertIn("Refusing", output) + readme = self._read(self.readme) + self.assertIn("Total%20Downloads-10.00M-", readme) + self.assertIn("Total%20Stars-4-", readme, "other figures still refresh") + + def test_allow_decrease_publishes_a_lower_total(self): + self._write(self.readme, udb.update_readme(self.readme_text, [("Total Downloads", "10.00M")])[0]) + code, _ = self._run(self._healthy_web(), argv=["--allow-decrease"], env={"PEPY_API_KEY": "k"}) self.assertEqual(code, 0) - expected = 1_000_000 * len(udb.PYPI_PACKAGES) + 400_000 - self.assertIn(f"{expected:>14,}", output) - self.assertIn(udb.format_count(expected), self._read()) + self.assertNotIn("Total%20Downloads-10.00M-", self._read(self.readme)) - def test_dry_run_changes_nothing(self): - code, output = self._run( - ["--readme", self.path, "--dry-run"], - json_responses={pkg: {"total_downloads": 7} for pkg in udb.PYPI_PACKAGES}, - text_responses={"jsdelivr": shields_svg("hits", "8")}, - api_key="k", - ) + def test_clones_show_not_available_without_a_token(self): + code, _ = self._run(self._healthy_web(), env={"PEPY_API_KEY": "k"}) self.assertEqual(code, 0) - self.assertIn("--dry-run", output) - self.assertEqual(self._read(), self.readme) + self.assertIn("Total%20Clones-n%2Fa-", self._read(self.readme)) + self.assertEqual(json.loads(self._read(self.history)), {"repos": {}}) + + def test_repos_are_discovered_without_any_hand_list(self): + code, output = self._run(self._healthy_web(), argv=["--dry-run"], env={"PEPY_API_KEY": "k"}) + self.assertIn("Infinitode/CloakJS", output) + self.assertIn("johanbeytell1/Alloy", output) + self.assertNotIn("hued-fork", output) + self.assertNotIn("Secret", output) if __name__ == "__main__": diff --git a/scripts/update-download-badges.py b/scripts/update-download-badges.py index f2bd16d..5f66154 100644 --- a/scripts/update-download-badges.py +++ b/scripts/update-download-badges.py @@ -1,37 +1,53 @@ #!/usr/bin/env python3 -"""Tally PyPI and jsDelivr downloads and refresh the profile badge. - -Data sources, in order of preference ------------------------------------- -PyPI (all-time) - 1. pepy v2 API -- ``https://api.pepy.tech/api/v2/projects/{pkg}`` with an - ``X-Api-Key`` header. Exact counts. Requires the ``PEPY_API_KEY`` - secret; pepy retired its keyless endpoint (it now returns 403). - 2. shields.io ``/pepy/dt/{pkg}`` -- the same endpoint the per-package - badges in this README already use. No key needed, but the figure is - rounded ("12.3k"), so the total is approximate. - -jsDelivr - shields.io ``/jsdelivr/gh/{period}/{owner}/{repo}``. jsDelivr publishes - period-based stats only (hour/day/week/month/year), so the default - period is ``hy`` = last 12 months. There is no all-time jsDelivr figure. +"""Tally downloads, clones, stars and forks across Infinitode's public repos. + +Nothing is hand-maintained. Every public repository owned by one of OWNERS is +discovered from the GitHub API and counted; anything else is ignored. Packages +are only counted when they are public and belong to one of those repositories. + +What is measured +---------------- +Stars and forks (exact, all-time) + ``stargazers_count`` and ``forks_count`` from the GitHub repo listing. + +Downloads + PyPI (all-time). A PyPI package belongs to a repo when it has the repo's + name AND either its PyPI metadata links back to the repo, or the repo's own + packaging file (pyproject.toml / setup.py / setup.cfg) declares that name. + Same-named packages by other people fail both checks, so they are skipped. + 1. pepy v2 API when PEPY_API_KEY is set (exact counts), otherwise + 2. shields.io ``/pepy/dt`` badge (rounded figures). + jsDelivr (last 12 months). shields.io ``/jsdelivr/gh/hy`` for every counted + repo. jsDelivr publishes period stats only; there is no all-time figure. + +Clones + GitHub traffic API ``/traffic/clones``. It only returns the last 14 days and + needs push access to the repository, so it requires the TRAFFIC_TOKEN secret. + Daily counts are merged into CLONE_HISTORY, which is the all-time record. + The total therefore starts from the first run that has a token. Safety ------ -A failed fetch never overwrites the badge. The badge is only rewritten when -at least one source returned a number, and never to a value lower than the -one already published (download totals are monotonically increasing, so a -drop means we are looking at partial data). +A failed fetch never overwrites published numbers. If the repo listing fails, +nothing is written. If every download source fails, nothing is written. A +download total is only published when it is not more than 5% below the figure +already shown (download totals only increase, so a drop means partial data). +Stars, forks and clones are not guarded, because they can legitimately fall. Usage:: python3 scripts/update-download-badges.py --readme profile/README.md python3 scripts/update-download-badges.py --dry-run + +Environment: PEPY_API_KEY (optional), GITHUB_TOKEN (optional, raises the GitHub +rate limit), TRAFFIC_TOKEN (optional, enables clone counts). """ from __future__ import annotations import argparse +import base64 +import copy import gzip import json import os @@ -42,27 +58,21 @@ # --- what gets counted ------------------------------------------------------- -PYPI_PACKAGES = [ - "qrforge", - "localsitemap", - "pyautoplot", - "valx", - "hued", - "deepdefend", - "codesafe", - "pywebscrapr", - "funcprofiler", - "duplipy", -] - -JSDELIVR_GH = [ - "infinitode/blurjs", - # "johanbeytell1/alloy", # uncomment once it is served through jsDelivr -] +# (login, account type). Only public repositories owned by these accounts count. +OWNERS = (("johanbeytell1", "user"), ("Infinitode", "org")) # hy = hits per year, the longest window jsDelivr exposes. JSDELIVR_PERIOD = "hy" +# Clone history, kept in the repo so the all-time total survives the 14-day window. +CLONE_HISTORY = "data/clone-history.json" + +GITHUB_API = "https://api.github.com" +GITHUB_HEADERS = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"} +MAX_PAGES = 20 +PACKAGING_FILES = ("pyproject.toml", "setup.py", "setup.cfg") + +PYPI_JSON = "https://pypi.org/pypi/{pkg}/json" PEPY_V2 = "https://api.pepy.tech/api/v2/projects/{pkg}" SHIELDS_PYPI = "https://img.shields.io/pepy/dt/{pkg}" SHIELDS_JSDELIVR = "https://img.shields.io/jsdelivr/gh/{period}/{repo}" @@ -75,19 +85,24 @@ # Keys that hold an all-time count, highest confidence first. COUNT_KEYS = ("total_downloads", "totaldownloads", "totalhits", "downloads", "hits") +START_MARK = "" +END_MARK = "" START_RE = re.compile( r"", re.S ) -START_MARK = "" -END_MARK = "" -BADGE = "![Total Downloads](https://img.shields.io/badge/Total%20Downloads-{value}-5d17eb)" -BADGE_VALUE_RE = re.compile(r"Total%20Downloads-([0-9][0-9.,]*)([kMB]?)-") ANCHOR = "![](https://komarev.com/ghpvc/?username=infinitode&color=5d17eb)" +BADGE_COLOR = "5d17eb" +NOT_AVAILABLE = "n%2Fa" # shields.io path form of "n/a" +BADGE_VALUE_RE = re.compile(r"Total%20Downloads-([0-9][0-9.,]*)([kMB]?)-") +CURRENT_DOWNLOADS_RE = re.compile(r"Total%20Downloads-([^-\s)]+)-") TEXT_RE = re.compile(r"]*>([^<]*)") SHIELDS_VALUE_RE = re.compile(r"^([0-9][0-9.,]*)\s*([kMB]?)$") _SUFFIX = {"": 1, "k": 1_000, "M": 1_000_000, "B": 1_000_000_000} +_PY_NAME_RE = re.compile(r"""\bname\s*=\s*["']([A-Za-z0-9][A-Za-z0-9._-]*)["']""") +_CFG_NAME_RE = re.compile(r"^name\s*=\s*([A-Za-z0-9][A-Za-z0-9._-]*)\s*$", re.M) + # --- network (the only I/O in this module) ----------------------------------- @@ -117,7 +132,14 @@ def fetch_text(url, headers=None, timeout=30): return fetch(url, headers, timeout)[0].decode("utf-8", "replace") -# --- parsers (pure) ---------------------------------------------------------- +def github_headers(token=None): + headers = dict(GITHUB_HEADERS) + if token: + headers["Authorization"] = f"Bearer {token}" + return headers + + +# --- parsers and pure helpers (no I/O) --------------------------------------- def _search(node, key): @@ -173,13 +195,167 @@ def format_count(total): def parse_badge_value(text): - """Read the number currently published in the badge, or ``None``.""" + """Read the download total currently published in the badge, or ``None``.""" match = BADGE_VALUE_RE.search(text or "") if not match: return None return int(round(float(match.group(1).replace(",", "")) * _SUFFIX[match.group(2)])) +def normalize_name(name): + """PyPI's name normalisation (PEP 503): case-insensitive, runs of -_. equal.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def is_counted(repo, owner): + """True for a public repository that is owned by *owner*.""" + login = (repo.get("owner") or {}).get("login", "") + return ( + login.lower() == owner.lower() + and not repo.get("private") + and repo.get("visibility", "public") == "public" + ) + + +def links_back(meta, owner, repo): + """True if the PyPI project's metadata links to github.com/owner/repo.""" + info = meta.get("info") or {} + urls = [info.get(key) for key in ("home_page", "download_url", "project_url")] + urls += list((info.get("project_urls") or {}).values()) + text = " ".join(str(url) for url in urls if url) + "\n" + (info.get("description") or "") + pattern = rf"github\.com/{re.escape(owner)}/{re.escape(repo)}(?![\w-]|\.\w)" + return re.search(pattern, text, re.I) is not None + + +def packaging_names(filename, text): + """Package names a packaging file declares.""" + if filename == "setup.cfg": + return set(_CFG_NAME_RE.findall(text)) + return set(_PY_NAME_RE.findall(text)) + + +def merge_clones(history, full_name, daily): + """Fold one repo's daily clone counts into *history* (in place). + + GitHub reports the last 14 days in full on every call, so a day is simply + overwritten with the latest count. Days that are zero are not stored. + """ + days = history.setdefault("repos", {}).setdefault(full_name, {}) + for day, count in daily.items(): + if count > 0: + days[day] = count + else: + days.pop(day, None) + if not days: + history["repos"].pop(full_name, None) + + +def history_total(history): + return sum(sum(days.values()) for days in history.get("repos", {}).values()) + + +def render_block(values): + """The badge block. *values* is a list of ``(label, badge_text)`` pairs.""" + lines = [START_MARK] + for label, text in values: + label_path = label.replace(" ", "%20") + lines.append(f"![{label}](https://img.shields.io/badge/{label_path}-{text}-{BADGE_COLOR})") + lines.append(END_MARK) + return "\n".join(lines) + + +def update_readme(text, values): + """Replace (or insert) the badge block. Returns ``(new_text, changed)``.""" + block = render_block(values) + if START_RE.search(text): + new_text, _ = START_RE.subn(lambda _match: block, text, count=1) + return new_text, new_text != text + if ANCHOR in text: # first run: drop the badges under the visitor counter + return text.replace(ANCHOR, f"{ANCHOR}\n{block}", 1), True + return text, False + + +def current_downloads_text(text): + """The badge text currently shown for downloads, or ``None``.""" + match = CURRENT_DOWNLOADS_RE.search(text or "") + return match.group(1) if match else None + + +# --- discovery --------------------------------------------------------------- + + +def list_public_repos(owner, kind, fetch_json=None, token=None): + """Every public repository owned by *owner*. Raises on API failure.""" + fetch_json = fetch_json or globals()["fetch_json"] + base = f"{GITHUB_API}/{'orgs' if kind == 'org' else 'users'}/{owner}/repos" + listing = "public" if kind == "org" else "owner" + repos = [] + for page in range(1, MAX_PAGES + 1): + batch = fetch_json(f"{base}?type={listing}&per_page=100&page={page}", github_headers(token)) + if not isinstance(batch, list): + raise ValueError(f"unexpected repository listing for {owner}") + repos.extend(repo for repo in batch if is_counted(repo, owner)) + if len(batch) < 100: + break + return repos + + +def discover_repos(fetch_json=None, token=None): + repos = [] + for owner, kind in OWNERS: + repos.extend(list_public_repos(owner, kind, fetch_json=fetch_json, token=token)) + return repos + + +def declared_names(owner, repo, fetch_json=None, token=None): + """Package names declared by the repo's own packaging files.""" + fetch_json = fetch_json or globals()["fetch_json"] + try: + listing = fetch_json(f"{GITHUB_API}/repos/{owner}/{repo}/contents", github_headers(token)) + except urllib.error.HTTPError as error: + if error.code == 404: + return set() + raise + names = set() + for item in listing if isinstance(listing, list) else []: + if item.get("type") == "file" and item.get("name") in PACKAGING_FILES: + blob = fetch_json(item["url"], github_headers(token)) + text = base64.b64decode(blob.get("content", "")).decode("utf-8", "replace") + names |= packaging_names(item["name"], text) + return names + + +def pypi_package_for(repo, fetch_json=None, token=None): + """The PyPI package that belongs to *repo*, or ``None``.""" + fetch_json = fetch_json or globals()["fetch_json"] + owner, name = repo["owner"]["login"], repo["name"] + package = normalize_name(name) + try: + meta = fetch_json(PYPI_JSON.format(pkg=package)) + except urllib.error.HTTPError as error: + if error.code == 404: + return None + raise + if links_back(meta, owner, name): + return package + declared = {normalize_name(n) for n in declared_names(owner, name, fetch_json, token)} + return package if package in declared else None + + +def discover_pypi(repos, fetch_json=None, token=None): + """Map ``full_name -> PyPI package`` for every repo that has one.""" + packages, notes = {}, [] + for repo in repos: + try: + package = pypi_package_for(repo, fetch_json=fetch_json, token=token) + except (urllib.error.URLError, ValueError) as error: + notes.append(f"{repo['full_name']}: PyPI lookup skipped ({error})") + continue + if package: + packages[repo["full_name"]] = package + return packages, notes + + # --- source lookups ---------------------------------------------------------- @@ -217,7 +393,7 @@ def pypi_totals(packages, api_key=None, fetch_json=None, fetch_text=None): def jsdelivr_totals(repos, period=JSDELIVR_PERIOD, fetch_text=None): - """jsDelivr hits per repo for *period* (default: last 12 months).""" + """jsDelivr hits per repo (``owner/repo``) for *period* (default: 12 months).""" fetch_text = fetch_text or globals()["fetch_text"] totals, failures = {}, {} for repo in repos: @@ -234,104 +410,181 @@ def jsdelivr_totals(repos, period=JSDELIVR_PERIOD, fetch_text=None): return totals, failures -def collect(api_key=None, fetch_json=None, fetch_text=None): +def clone_counts(full_name, token, fetch_json=None): + """Daily clone counts for the last 14 days: ``{"YYYY-MM-DD": count}``.""" + fetch_json = fetch_json or globals()["fetch_json"] + payload = fetch_json(f"{GITHUB_API}/repos/{full_name}/traffic/clones", github_headers(token)) + return {entry["timestamp"][:10]: int(entry.get("count", 0)) for entry in payload.get("clones", [])} + + +def load_history(path): + if not os.path.exists(path): + return {"repos": {}} + with open(path, encoding="utf-8") as handle: + history = json.load(handle) + history.setdefault("repos", {}) + return history + + +def save_history(path, history): + with open(path, "w", encoding="utf-8") as handle: + json.dump(history, handle, indent=2, sort_keys=True) + handle.write("\n") + + +# --- collection -------------------------------------------------------------- + + +def collect( + api_key=None, + github_token=None, + traffic_token=None, + history=None, + fetch_json=None, + fetch_text=None, +): + """Gather every figure. *history* is updated in place with clone data.""" fetch_json = fetch_json or globals()["fetch_json"] fetch_text = fetch_text or globals()["fetch_text"] - pypi, pypi_failures, notes = pypi_totals( - PYPI_PACKAGES, api_key=api_key, fetch_json=fetch_json, fetch_text=fetch_text - ) - jsdelivr, jsdelivr_failures = jsdelivr_totals(JSDELIVR_GH, fetch_text=fetch_text) - return { - "pypi": pypi, - "jsdelivr": jsdelivr, - "failures": {**pypi_failures, **jsdelivr_failures}, - "notes": notes, - "sources": len(pypi) + len(jsdelivr), - "total": sum(pypi.values()) + sum(jsdelivr.values()), - } + history = history if history is not None else {"repos": {}} + repos = discover_repos(fetch_json=fetch_json, token=github_token) + notes = [] -# --- README rewrite ---------------------------------------------------------- + stars = sum(int(repo.get("stargazers_count") or 0) for repo in repos) + forks = sum(int(repo.get("forks_count") or 0) for repo in repos) + packages, pypi_notes = discover_pypi(repos, fetch_json=fetch_json, token=github_token) + notes += pypi_notes + pypi, pypi_failures, pypi_notes = pypi_totals( + sorted(set(packages.values())), api_key=api_key, fetch_json=fetch_json, fetch_text=fetch_text + ) + notes += pypi_notes -def render_block(total): - return f"{START_MARK}\n{BADGE.format(value=format_count(total))}\n{END_MARK}" + jsdelivr, jsdelivr_failures = jsdelivr_totals( + [repo["full_name"].lower() for repo in repos], fetch_text=fetch_text + ) + clone_failures = {} + if traffic_token: + for repo in repos: + full_name = repo["full_name"] + try: + daily = clone_counts(full_name, traffic_token, fetch_json=fetch_json) + except (urllib.error.URLError, ValueError) as error: + clone_failures[full_name] = f"traffic/clones -> {error}" + continue + merge_clones(history, full_name, daily) + else: + notes.append("TRAFFIC_TOKEN not set; clone counts were not refreshed") -def update_readme(text, total): - """Replace (or insert) the badge block. Returns ``(new_text, changed)``.""" - block = render_block(total) - if START_RE.search(text): - new_text, _ = START_RE.subn(lambda _match: block, text, count=1) - return new_text, new_text != text - if ANCHOR in text: # first run: drop the badge under the visitor counter - return text.replace(ANCHOR, f"{ANCHOR}\n{block}", 1), True - return text, False + return { + "repos": [repo["full_name"] for repo in repos], + "stars": stars, + "forks": forks, + "packages": packages, + "pypi": pypi, + "jsdelivr": jsdelivr, + "failures": {**pypi_failures, **jsdelivr_failures, **clone_failures}, + "notes": sorted(set(notes)), + "download_sources": len(pypi) + len(jsdelivr), + "downloads": sum(pypi.values()) + sum(jsdelivr.values()), + "clones": history_total(history) if history.get("repos") else None, + } # --- entry point ------------------------------------------------------------- def main(argv=None): - parser = argparse.ArgumentParser(description="Refresh the total downloads badge.") + parser = argparse.ArgumentParser(description="Refresh the profile totals badges.") parser.add_argument("--readme", default="profile/README.md") + parser.add_argument("--history", default=CLONE_HISTORY, help="clone history JSON") parser.add_argument("--dry-run", action="store_true", help="print the result, change nothing") parser.add_argument( "--allow-decrease", action="store_true", - help="write the total even if it is lower than the published one", + help="publish the download total even if it is lower than the published one", ) args = parser.parse_args(argv) - result = collect(api_key=os.environ.get("PEPY_API_KEY")) + history = load_history(args.history) + before = copy.deepcopy(history) + try: + result = collect( + api_key=os.environ.get("PEPY_API_KEY") or None, + github_token=os.environ.get("GITHUB_TOKEN") or None, + traffic_token=os.environ.get("TRAFFIC_TOKEN") or None, + history=history, + ) + except (urllib.error.URLError, ValueError) as error: + print(f"Could not list repositories ({error}); nothing was changed.", file=sys.stderr) + return 1 for note in result["notes"]: print(f" note {note}") + print(f" repos {len(result['repos'])} public repos counted: {', '.join(sorted(result['repos']))}") + for repo, package in sorted(result["packages"].items()): + print(f" package {repo:<40} -> pypi/{package}") for name, count in sorted(result["pypi"].items()): - print(f" pypi {name:<20} {count:>14,}") + print(f" pypi {name:<29} {count:>14,}") for name, count in sorted(result["jsdelivr"].items()): print(f" jsdelivr {name:<29} {count:>14,}") for name, reason in sorted(result["failures"].items()): print(f" FAILED {name}: {reason}", file=sys.stderr) - print(f" {'TOTAL':<38} {result['total']:>14,} ({format_count(result['total'])})") - - # Never publish a number we could not actually measure. - if result["sources"] == 0: - print("Every source failed; the badge is left untouched.", file=sys.stderr) + clones = result["clones"] + print(f" {'DOWNLOADS':<38} {result['downloads']:>14,} ({format_count(result['downloads'])})") + print(f" {'CLONES':<38} {'n/a' if clones is None else f'{clones:,}'}") + print(f" {'STARS':<38} {result['stars']:>14,}") + print(f" {'FORKS':<38} {result['forks']:>14,}") + + # Never publish a download total we could not actually measure. + if result["download_sources"] == 0: + print("Every download source failed; the README and clone history are left untouched.", file=sys.stderr) return 1 with open(args.readme, encoding="utf-8") as handle: original = handle.read() + if not START_RE.search(original) and ANCHOR not in original: + print(f"No badge block or anchor found in {args.readme}.", file=sys.stderr) + return 1 + # Download totals only increase, so a drop means partial data. A published + # 0 is a leftover from a failed run, not a real measurement. published = parse_badge_value(original) - # A published 0 is a leftover from a failed run, not a real measurement. - # Allow for display rounding: "5.17M" can stand for anything in - # [5,165,000, 5,175,000), so a tiny apparent drop is noise, not data loss. - if ( - published - and result["total"] < published * (1 - DROP_TOLERANCE) - and not args.allow_decrease - ): + downloads_text = current_downloads_text(original) or NOT_AVAILABLE + if published and result["downloads"] < published * (1 - DROP_TOLERANCE) and not args.allow_decrease: print( - f"Refusing to publish {result['total']:,} over {published:,} " + f"Refusing to publish {result['downloads']:,} downloads over {published:,} " f"(more than {DROP_TOLERANCE:.0%} below the published figure; looks like " - "partial data -- pass --allow-decrease to override).", + "partial data -- pass --allow-decrease to override). Downloads badge kept.", file=sys.stderr, ) - return 0 + else: + downloads_text = format_count(result["downloads"]) + + values = [ + ("Total Downloads", downloads_text), + ("Total Clones", NOT_AVAILABLE if clones is None else format_count(clones)), + ("Total Stars", format_count(result["stars"])), + ("Total Forks", format_count(result["forks"])), + ] if args.dry_run: - print("--dry-run: README left untouched") + print("--dry-run: nothing written") return 0 - updated, changed = update_readme(original, result["total"]) + if history != before: + save_history(args.history, history) + print(f"Updated clone history in {args.history}.") + + updated, changed = update_readme(original, values) if not changed: - print(f"Badge already up to date in {args.readme}.") + print(f"Badges already up to date in {args.readme}.") return 0 - with open(args.readme, "w", encoding="utf-8") as handle: handle.write(updated) - print(f"Updated the badge in {args.readme}.") + print(f"Updated the badges in {args.readme}.") return 0 From 317e2b642b42ef8115b0d78c06c1fb3c2940c6b5 Mon Sep 17 00:00:00 2001 From: arena-agent Date: Sun, 11 Oct 2026 15:31:27 +0000 Subject: [PATCH 2/2] Drop clone counting; run tokenless - Remove the traffic/clones totals (their 14-day window and push-access requirement needed a personal token and a history file). - Badges now: Total Downloads, Total Stars, Total Forks. - Workflow needs no secrets beyond the built-in GITHUB_TOKEN; PEPY_API_KEY stays optional for exact PyPI counts. Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/workflows/update-download-badges.yml | 33 ++---- data/clone-history.json | 3 - profile/README.md | 1 - scripts/test_update_download_badges.py | 87 +++------------- scripts/update-download-badges.py | 102 ++----------------- 5 files changed, 32 insertions(+), 194 deletions(-) delete mode 100644 data/clone-history.json diff --git a/.github/workflows/update-download-badges.yml b/.github/workflows/update-download-badges.yml index ad4a284..6b57637 100644 --- a/.github/workflows/update-download-badges.yml +++ b/.github/workflows/update-download-badges.yml @@ -1,27 +1,17 @@ name: Update profile totals -# Tallies downloads, clones, stars and forks across the public repos of -# johanbeytell1 and Infinitode, and rewrites the TOTAL-DOWNLOADS badge block in +# Tallies downloads, stars and forks across the public repos of johanbeytell1 +# and Infinitode, and rewrites the TOTAL-DOWNLOADS badge block in # profile/README.md. Repos are discovered automatically; nothing is listed by hand. +# No personal access tokens are needed. # # Data sources (see scripts/update-download-badges.py for details): # Stars, forks - GitHub repo listing (exact, all-time). # PyPI - packages that belong to a repo (verified by a link back to -# the repo or by the repo's packaging file). pepy v2 API when -# PEPY_API_KEY is set (exact), otherwise the shields.io -# /pepy/dt badge (rounded). +# the repo or by the repo's packaging file). Rounded shields.io +# figures by default; set PEPY_API_KEY for exact counts. # jsDelivr - shields.io /jsdelivr/gh/hy badge for every repo; hits over # the last 12 months (there is no all-time jsDelivr figure). -# Clones - GitHub traffic API, which only covers 14 days and needs -# push access. Daily counts are merged into -# data/clone-history.json so the total keeps growing. -# -# Secrets (both optional; the job still runs without them): -# PEPY_API_KEY - exact PyPI counts (free key: pepy.tech -> sign in -> API key). -# TRAFFIC_TOKEN - a token with read access to repository traffic on every -# counted repo (fine-grained PAT: Administration read, on both -# owners' repos; or a classic PAT with the repo scope). Without -# it, clone totals are not refreshed. # # Safety: a failed fetch never overwrites the published numbers. The job only # writes when the repo listing succeeded and at least one download source @@ -35,7 +25,6 @@ on: pull_request: paths: - "scripts/**" - - "data/**" - ".github/workflows/update-download-badges.yml" push: branches: [main] @@ -69,24 +58,22 @@ jobs: - name: Tally totals and rewrite the badges if: github.event_name != 'pull_request' env: + # Built into every workflow run; only used to raise the GitHub rate limit. GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Optional. Without it PyPI counts are rounded shields.io figures. PEPY_API_KEY: ${{ secrets.PEPY_API_KEY }} - TRAFFIC_TOKEN: ${{ secrets.TRAFFIC_TOKEN }} run: | if [ -z "$PEPY_API_KEY" ]; then - echo "::warning::PEPY_API_KEY is not set; PyPI counts will be rounded shields.io figures." - fi - if [ -z "$TRAFFIC_TOKEN" ]; then - echo "::warning::TRAFFIC_TOKEN is not set; clone totals will not be refreshed." + echo "::notice::PEPY_API_KEY is not set; PyPI counts will be rounded shields.io figures." fi - python3 scripts/update-download-badges.py --readme profile/README.md --history data/clone-history.json + python3 scripts/update-download-badges.py --readme profile/README.md - name: Commit the updated badges if: github.event_name != 'pull_request' run: | git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add profile/README.md data/clone-history.json + git add profile/README.md if git diff --cached --quiet; then echo "Totals unchanged; nothing to commit." else diff --git a/data/clone-history.json b/data/clone-history.json deleted file mode 100644 index 03be823..0000000 --- a/data/clone-history.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "repos": {} -} diff --git a/profile/README.md b/profile/README.md index 846f948..5404562 100644 --- a/profile/README.md +++ b/profile/README.md @@ -10,7 +10,6 @@ ![](https://komarev.com/ghpvc/?username=infinitode&color=5d17eb) ![Total Downloads](https://img.shields.io/badge/Total%20Downloads-120.9k-5d17eb) -![Total Clones](https://img.shields.io/badge/Total%20Clones-n%2Fa-5d17eb) ![Total Stars](https://img.shields.io/badge/Total%20Stars-50-5d17eb) ![Total Forks](https://img.shields.io/badge/Total%20Forks-10-5d17eb) diff --git a/scripts/test_update_download_badges.py b/scripts/test_update_download_badges.py index 1d79642..9f521ba 100644 --- a/scripts/test_update_download_badges.py +++ b/scripts/test_update_download_badges.py @@ -3,7 +3,7 @@ The network layer is replaced with a router that answers with payloads shaped like the real GitHub, PyPI, pepy and shields.io responses. Everything else -- -repo discovery and ownership checks, counting, clone merging, badge parsing, +repo discovery and ownership checks, counting, badge parsing, the regression guards and the README rewrite -- is production code. Run with: python3 -m unittest discover -s scripts -p 'test_*.py' -v @@ -29,7 +29,7 @@ udb = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(udb) -TOKEN_VARS = ("PEPY_API_KEY", "GITHUB_TOKEN", "TRAFFIC_TOKEN") +TOKEN_VARS = ("PEPY_API_KEY", "GITHUB_TOKEN") def shields_svg(label, value): @@ -239,32 +239,11 @@ def test_packaging_names(self): self.assertEqual(udb.packaging_names("setup.cfg", "[metadata]\nname = valx\n"), {"valx"}) -class TestClones(unittest.TestCase): - def test_days_are_overwritten_not_added(self): - history = {"repos": {}} - udb.merge_clones(history, "Infinitode/Hued", {"2026-10-10": 3, "2026-10-11": 1}) - udb.merge_clones(history, "Infinitode/Hued", {"2026-10-10": 5, "2026-10-11": 1}) - self.assertEqual(udb.history_total(history), 6) - - def test_zero_days_are_dropped_and_empty_repos_removed(self): - history = {"repos": {}} - udb.merge_clones(history, "Infinitode/Hued", {"2026-10-10": 2}) - udb.merge_clones(history, "Infinitode/Hued", {"2026-10-10": 0}) - self.assertEqual(history, {"repos": {}}) - - def test_totals_accumulate_across_repos(self): - history = {"repos": {}} - udb.merge_clones(history, "Infinitode/Hued", {"2026-10-10": 4}) - udb.merge_clones(history, "johanbeytell1/Alloy", {"2026-10-09": 1}) - self.assertEqual(udb.history_total(history), 5) - - class TestReadmeRewrite(unittest.TestCase): def setUp(self): self.readme = f"# Profile\n\n{udb.ANCHOR}\n\nSome text.\n" self.values = [ ("Total Downloads", "1.23M"), - ("Total Clones", "n%2Fa"), ("Total Stars", "50"), ("Total Forks", "10"), ] @@ -273,7 +252,6 @@ def test_inserts_all_badges_after_visitor_counter_on_first_run(self): updated, changed = udb.update_readme(self.readme, self.values) self.assertTrue(changed) self.assertIn("Total%20Downloads-1.23M-", updated) - self.assertIn("Total%20Clones-n%2Fa-", updated) self.assertIn("Total%20Stars-50-", updated) self.assertIn("Total%20Forks-10-", updated) self.assertLess(updated.index(udb.ANCHOR), updated.index(udb.START_MARK)) @@ -375,52 +353,20 @@ def test_downloads_come_from_pypi_and_jsdelivr(self): self.assertEqual(result["downloads"], 1_000 + 1_000 + 250) self.assertNotIn("localshare", result["pypi"]) - def test_clones_stay_unknown_without_a_traffic_token(self): - history = {"repos": {}} - result = udb.collect(api_key="k", history=history, fetch_json=self.web.fetch_json, fetch_text=self.web.fetch_text) - self.assertIsNone(result["clones"]) - self.assertTrue(any("TRAFFIC_TOKEN" in n for n in result["notes"])) - - def test_clone_history_is_merged_per_repo(self): - self.web.json["traffic/clones"] = lambda url, headers: { - "count": 9, - "clones": [{"timestamp": "2026-10-10T00:00:00Z", "count": 3}, {"timestamp": "2026-10-11T00:00:00Z", "count": 0}], - } - history = {"repos": {}} - result = udb.collect(api_key="k", traffic_token="t", history=history, fetch_json=self.web.fetch_json, fetch_text=self.web.fetch_text) - self.assertEqual(result["clones"], 3 * len(result["repos"])) - self.assertEqual(history["repos"]["Infinitode/Hued"], {"2026-10-10": 3}) - traffic_calls = [h for url, h in self.web.json_calls if "traffic/clones" in url] - self.assertTrue(all(h.get("Authorization") == "Bearer t" for h in traffic_calls)) - - def test_one_repo_failing_its_clone_fetch_keeps_the_others(self): - def clones(url, headers): - if "/Hued/" in url: - raise http_error(403) - return {"clones": [{"timestamp": "2026-10-10T00:00:00Z", "count": 2}]} - - self.web.json["traffic/clones"] = clones - history = {"repos": {}} - result = udb.collect(traffic_token="t", history=history, fetch_json=self.web.fetch_json, fetch_text=self.web.fetch_text) - self.assertIn("Infinitode/Hued", result["failures"]) - self.assertNotIn("Infinitode/Hued", history["repos"]) - self.assertIn("Infinitode/CloakJS", history["repos"]) # --- entry point -------------------------------------------------------------- class TestMain(StubMixin, unittest.TestCase): - """Drives the real main() against a throwaway README and clone history.""" + """Drives the real main() against a throwaway README.""" def setUp(self): self.readme_text = f"# Profile\n\n{udb.ANCHOR}\n" self.dir = tempfile.TemporaryDirectory() self.addCleanup(self.dir.cleanup) self.readme = os.path.join(self.dir.name, "README.md") - self.history = os.path.join(self.dir.name, "clone-history.json") self._write(self.readme, self.readme_text) - self._write(self.history, json.dumps({"repos": {}})) saved = {var: os.environ.pop(var) for var in TOKEN_VARS if var in os.environ} self.addCleanup(os.environ.update, saved) self.addCleanup(lambda: [os.environ.pop(var, None) for var in TOKEN_VARS]) @@ -442,7 +388,7 @@ def _run(self, web, argv=(), env=None): original_stdout, original_stderr = sys.stdout, sys.stderr sys.stdout = sys.stderr = buffer try: - code = udb.main([*argv, "--readme", self.readme, "--history", self.history]) + code = udb.main([*argv, "--readme", self.readme]) finally: sys.stdout, sys.stderr = original_stdout, original_stderr return code, buffer.getvalue() @@ -467,19 +413,20 @@ def test_every_download_source_failing_changes_nothing(self): self.assertEqual(code, 1) self.assertEqual(self._read(self.readme), self.readme_text) - def test_happy_path_writes_all_four_badges(self): - web = self._healthy_web() - web.json["traffic/clones"] = {"clones": [{"timestamp": "2026-10-10T00:00:00Z", "count": 7}]} - code, output = self._run(web, env={"PEPY_API_KEY": "k", "TRAFFIC_TOKEN": "t"}) + def test_tokenless_run_writes_downloads_stars_and_forks(self): + code, output = self._run(self._healthy_web()) self.assertEqual(code, 0, output) readme = self._read(self.readme) - # 2 packages x 1,000 + jsDelivr 0 for each repo except CloakJS (1.0k shields figure is the default) - # PyPI: hued + qrforge at 1,000 each; jsDelivr: CloakJS 250, the rest 0. - self.assertIn(f"Total%20Downloads-{udb.format_count(2_250)}-", readme) + self.assertIn("Total%20Downloads-", readme) self.assertIn("Total%20Stars-4-", readme) # LocalShare 1 + Alloy 1 + Hued 2 self.assertIn("Total%20Forks-1-", readme) - self.assertIn(f"Total%20Clones-{udb.format_count(7 * 5)}-", readme) # 7 clones x 5 repos - self.assertEqual(json.loads(self._read(self.history))["repos"]["Infinitode/Hued"], {"2026-10-10": 7}) + self.assertNotIn("Clones", readme) + + def test_happy_path_with_pepy_key_is_exact(self): + code, output = self._run(self._healthy_web(), env={"PEPY_API_KEY": "k"}) + self.assertEqual(code, 0, output) + # PyPI: hued + qrforge at 1,000 each (pepy); jsDelivr: CloakJS 250, the rest 0. + self.assertIn(f"Total%20Downloads-{udb.format_count(2_250)}-", self._read(self.readme)) def test_dry_run_changes_nothing(self): code, output = self._run(self._healthy_web(), argv=["--dry-run"], env={"PEPY_API_KEY": "k"}) @@ -502,12 +449,6 @@ def test_allow_decrease_publishes_a_lower_total(self): self.assertEqual(code, 0) self.assertNotIn("Total%20Downloads-10.00M-", self._read(self.readme)) - def test_clones_show_not_available_without_a_token(self): - code, _ = self._run(self._healthy_web(), env={"PEPY_API_KEY": "k"}) - self.assertEqual(code, 0) - self.assertIn("Total%20Clones-n%2Fa-", self._read(self.readme)) - self.assertEqual(json.loads(self._read(self.history)), {"repos": {}}) - def test_repos_are_discovered_without_any_hand_list(self): code, output = self._run(self._healthy_web(), argv=["--dry-run"], env={"PEPY_API_KEY": "k"}) self.assertIn("Infinitode/CloakJS", output) diff --git a/scripts/update-download-badges.py b/scripts/update-download-badges.py index 5f66154..5b3fb1c 100644 --- a/scripts/update-download-badges.py +++ b/scripts/update-download-badges.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Tally downloads, clones, stars and forks across Infinitode's public repos. +"""Tally downloads, stars and forks across Infinitode's public repos. Nothing is hand-maintained. Every public repository owned by one of OWNERS is discovered from the GitHub API and counted; anything else is ignored. Packages @@ -20,34 +20,27 @@ jsDelivr (last 12 months). shields.io ``/jsdelivr/gh/hy`` for every counted repo. jsDelivr publishes period stats only; there is no all-time figure. -Clones - GitHub traffic API ``/traffic/clones``. It only returns the last 14 days and - needs push access to the repository, so it requires the TRAFFIC_TOKEN secret. - Daily counts are merged into CLONE_HISTORY, which is the all-time record. - The total therefore starts from the first run that has a token. - Safety ------ A failed fetch never overwrites published numbers. If the repo listing fails, nothing is written. If every download source fails, nothing is written. A download total is only published when it is not more than 5% below the figure already shown (download totals only increase, so a drop means partial data). -Stars, forks and clones are not guarded, because they can legitimately fall. +Stars and forks are not guarded, because they can legitimately fall. Usage:: python3 scripts/update-download-badges.py --readme profile/README.md python3 scripts/update-download-badges.py --dry-run -Environment: PEPY_API_KEY (optional), GITHUB_TOKEN (optional, raises the GitHub -rate limit), TRAFFIC_TOKEN (optional, enables clone counts). +Environment: PEPY_API_KEY (optional, exact PyPI counts). GITHUB_TOKEN is also +read if present, which only raises the GitHub rate limit; nothing needs it. """ from __future__ import annotations import argparse import base64 -import copy import gzip import json import os @@ -64,9 +57,6 @@ # hy = hits per year, the longest window jsDelivr exposes. JSDELIVR_PERIOD = "hy" -# Clone history, kept in the repo so the all-time total survives the 14-day window. -CLONE_HISTORY = "data/clone-history.json" - GITHUB_API = "https://api.github.com" GITHUB_HEADERS = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"} MAX_PAGES = 20 @@ -234,26 +224,6 @@ def packaging_names(filename, text): return set(_PY_NAME_RE.findall(text)) -def merge_clones(history, full_name, daily): - """Fold one repo's daily clone counts into *history* (in place). - - GitHub reports the last 14 days in full on every call, so a day is simply - overwritten with the latest count. Days that are zero are not stored. - """ - days = history.setdefault("repos", {}).setdefault(full_name, {}) - for day, count in daily.items(): - if count > 0: - days[day] = count - else: - days.pop(day, None) - if not days: - history["repos"].pop(full_name, None) - - -def history_total(history): - return sum(sum(days.values()) for days in history.get("repos", {}).values()) - - def render_block(values): """The badge block. *values* is a list of ``(label, badge_text)`` pairs.""" lines = [START_MARK] @@ -410,43 +380,13 @@ def jsdelivr_totals(repos, period=JSDELIVR_PERIOD, fetch_text=None): return totals, failures -def clone_counts(full_name, token, fetch_json=None): - """Daily clone counts for the last 14 days: ``{"YYYY-MM-DD": count}``.""" - fetch_json = fetch_json or globals()["fetch_json"] - payload = fetch_json(f"{GITHUB_API}/repos/{full_name}/traffic/clones", github_headers(token)) - return {entry["timestamp"][:10]: int(entry.get("count", 0)) for entry in payload.get("clones", [])} - - -def load_history(path): - if not os.path.exists(path): - return {"repos": {}} - with open(path, encoding="utf-8") as handle: - history = json.load(handle) - history.setdefault("repos", {}) - return history - - -def save_history(path, history): - with open(path, "w", encoding="utf-8") as handle: - json.dump(history, handle, indent=2, sort_keys=True) - handle.write("\n") - - # --- collection -------------------------------------------------------------- -def collect( - api_key=None, - github_token=None, - traffic_token=None, - history=None, - fetch_json=None, - fetch_text=None, -): - """Gather every figure. *history* is updated in place with clone data.""" +def collect(api_key=None, github_token=None, fetch_json=None, fetch_text=None): + """Gather every download, star and fork figure.""" fetch_json = fetch_json or globals()["fetch_json"] fetch_text = fetch_text or globals()["fetch_text"] - history = history if history is not None else {"repos": {}} repos = discover_repos(fetch_json=fetch_json, token=github_token) notes = [] @@ -465,19 +405,6 @@ def collect( [repo["full_name"].lower() for repo in repos], fetch_text=fetch_text ) - clone_failures = {} - if traffic_token: - for repo in repos: - full_name = repo["full_name"] - try: - daily = clone_counts(full_name, traffic_token, fetch_json=fetch_json) - except (urllib.error.URLError, ValueError) as error: - clone_failures[full_name] = f"traffic/clones -> {error}" - continue - merge_clones(history, full_name, daily) - else: - notes.append("TRAFFIC_TOKEN not set; clone counts were not refreshed") - return { "repos": [repo["full_name"] for repo in repos], "stars": stars, @@ -485,11 +412,10 @@ def collect( "packages": packages, "pypi": pypi, "jsdelivr": jsdelivr, - "failures": {**pypi_failures, **jsdelivr_failures, **clone_failures}, + "failures": {**pypi_failures, **jsdelivr_failures}, "notes": sorted(set(notes)), "download_sources": len(pypi) + len(jsdelivr), "downloads": sum(pypi.values()) + sum(jsdelivr.values()), - "clones": history_total(history) if history.get("repos") else None, } @@ -499,7 +425,6 @@ def collect( def main(argv=None): parser = argparse.ArgumentParser(description="Refresh the profile totals badges.") parser.add_argument("--readme", default="profile/README.md") - parser.add_argument("--history", default=CLONE_HISTORY, help="clone history JSON") parser.add_argument("--dry-run", action="store_true", help="print the result, change nothing") parser.add_argument( "--allow-decrease", @@ -508,14 +433,10 @@ def main(argv=None): ) args = parser.parse_args(argv) - history = load_history(args.history) - before = copy.deepcopy(history) try: result = collect( api_key=os.environ.get("PEPY_API_KEY") or None, github_token=os.environ.get("GITHUB_TOKEN") or None, - traffic_token=os.environ.get("TRAFFIC_TOKEN") or None, - history=history, ) except (urllib.error.URLError, ValueError) as error: print(f"Could not list repositories ({error}); nothing was changed.", file=sys.stderr) @@ -532,15 +453,13 @@ def main(argv=None): print(f" jsdelivr {name:<29} {count:>14,}") for name, reason in sorted(result["failures"].items()): print(f" FAILED {name}: {reason}", file=sys.stderr) - clones = result["clones"] print(f" {'DOWNLOADS':<38} {result['downloads']:>14,} ({format_count(result['downloads'])})") - print(f" {'CLONES':<38} {'n/a' if clones is None else f'{clones:,}'}") print(f" {'STARS':<38} {result['stars']:>14,}") print(f" {'FORKS':<38} {result['forks']:>14,}") # Never publish a download total we could not actually measure. if result["download_sources"] == 0: - print("Every download source failed; the README and clone history are left untouched.", file=sys.stderr) + print("Every download source failed; the README is left untouched.", file=sys.stderr) return 1 with open(args.readme, encoding="utf-8") as handle: @@ -565,7 +484,6 @@ def main(argv=None): values = [ ("Total Downloads", downloads_text), - ("Total Clones", NOT_AVAILABLE if clones is None else format_count(clones)), ("Total Stars", format_count(result["stars"])), ("Total Forks", format_count(result["forks"])), ] @@ -574,10 +492,6 @@ def main(argv=None): print("--dry-run: nothing written") return 0 - if history != before: - save_history(args.history, history) - print(f"Updated clone history in {args.history}.") - updated, changed = update_readme(original, values) if not changed: print(f"Badges already up to date in {args.readme}.")