diff --git a/.github/workflows/update-download-badges.yml b/.github/workflows/update-download-badges.yml
index 649264a..6b57637 100644
--- a/.github/workflows/update-download-badges.yml
+++ b/.github/workflows/update-download-badges.yml
@@ -1,21 +1,22 @@
-name: Update total downloads badge
+name: Update profile totals
-# Tallies PyPI + jsDelivr downloads and rewrites the TOTAL-DOWNLOADS badge
-# block in profile/README.md.
+# Tallies downloads, stars and forks across the public repos of johanbeytell1
+# and Infinitode, and rewrites the TOTAL-DOWNLOADS badge block in
+# profile/README.md. Repos are discovered automatically; nothing is listed by hand.
+# No personal access tokens are needed.
#
# Data sources (see scripts/update-download-badges.py for details):
-# PyPI - pepy v2 API when PEPY_API_KEY is set (exact counts), otherwise
-# the shields.io /pepy/dt badge (no key, rounded figures).
-# jsDelivr - shields.io /jsdelivr/gh/hy badge = hits over the last 12 months.
-# jsDelivr publishes period stats only; there is no all-time figure.
+# Stars, forks - GitHub repo listing (exact, all-time).
+# PyPI - packages that belong to a repo (verified by a link back to
+# the repo or by the repo's packaging file). Rounded shields.io
+# figures by default; set PEPY_API_KEY for exact counts.
+# jsDelivr - shields.io /jsdelivr/gh/hy badge for every repo; hits over
+# the last 12 months (there is no all-time jsDelivr figure).
#
-# Optional but recommended: add a repository secret named PEPY_API_KEY
-# (free key: pepy.tech -> sign in -> API key). Without it the job still runs,
-# using the rounded shields.io figures instead.
-#
-# Safety: a failed fetch never overwrites the badge. The job only writes when
-# at least one source returned a number, and never publishes a total more than
-# 5% below the figure already shown.
+# Safety: a failed fetch never overwrites the published numbers. The job only
+# writes when the repo listing succeeded and at least one download source
+# returned a number. Download totals are never published more than 5% below the
+# figure already shown.
on:
schedule:
@@ -53,26 +54,29 @@ jobs:
- name: Run the badge tests
run: python3 -m unittest discover -s scripts -p 'test_*.py' -v
- # Pull requests only run the tests above; the badge is rewritten on main.
- - name: Tally downloads and rewrite the badge
+ # Pull requests only run the tests above; the badges are rewritten on main.
+ - name: Tally totals and rewrite the badges
if: github.event_name != 'pull_request'
env:
+ # Built into every workflow run; only used to raise the GitHub rate limit.
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ # Optional. Without it PyPI counts are rounded shields.io figures.
PEPY_API_KEY: ${{ secrets.PEPY_API_KEY }}
run: |
if [ -z "$PEPY_API_KEY" ]; then
- echo "::warning::PEPY_API_KEY is not set; PyPI counts will be rounded shields.io figures."
+ echo "::notice::PEPY_API_KEY is not set; PyPI counts will be rounded shields.io figures."
fi
python3 scripts/update-download-badges.py --readme profile/README.md
- - name: Commit the updated badge
+ - name: Commit the updated badges
if: github.event_name != 'pull_request'
run: |
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git add profile/README.md
if git diff --cached --quiet; then
- echo "Total downloads unchanged; nothing to commit."
+ echo "Totals unchanged; nothing to commit."
else
- git commit -m "chore: update total downloads badge [skip ci]"
+ git commit -m "chore: update profile totals [skip ci]"
git push
fi
diff --git a/profile/README.md b/profile/README.md
index 29943a3..5404562 100644
--- a/profile/README.md
+++ b/profile/README.md
@@ -10,6 +10,8 @@
+
+
[](https://ko-fi.com/M4J423SRBM)
@@ -30,7 +32,7 @@ Welcome to Infinitode, a cutting-edge small technology company dedicated to deve
> Upcoming changes are planned changes. We can remove, cancel, or stage these changes for a later date.
### Latest changes
-- Released `CloakJS`, a small JavaScript code obfuscator.
+- Released [`CloakJS`](https://github.com/Infinitode/CloakJS), a small JavaScript code obfuscator.
- Updated website with new pricing and MFA authentication available for verified emails and Google sign-ins.
- Updated general website security and user account security.
- Updated `Python Projects` with new projects.
@@ -421,6 +423,10 @@ Welcome to Infinitode, a cutting-edge small technology company dedicated to deve
   
+[**CloakJS**](https://github.com/Infinitode/CloakJS)
+
+  
+
## Our mission
@@ -438,6 +444,7 @@ We're proud to showcase some of our current projects:
* **CodeSafe**: An open-source Python library for code encryption, decryption, and safe evaluation using Python's built-in AST module, with allowed functions, variables, built-in imports, timeouts, and more.
* **PyWebScrapr**: An open-source Python web scraping tool supporting image and text scraping for effortless data collection.
* **FuncProfiler**: An open-source Python library for identifying bottlenecks in code, featuring function profiling, data exports, logging, and line-by-line profiling for more granular control.
+* **CloakJS**: A single-file, dependency-free JavaScript obfuscator for the browser and Node.js, with identifier renaming, string encryption, and control-flow options. [View on GitHub](https://github.com/Infinitode/CloakJS).
> [!TIP]
> For more projects, see: https://github.com/Infinitode?tab=repositories
diff --git a/scripts/test_update_download_badges.py b/scripts/test_update_download_badges.py
index 25dbcec..9f521ba 100644
--- a/scripts/test_update_download_badges.py
+++ b/scripts/test_update_download_badges.py
@@ -1,17 +1,19 @@
#!/usr/bin/env python3
"""Offline tests for scripts/update-download-badges.py.
-The download APIs are unreachable from CI sandboxes, so the network layer is
-stubbed here with payloads shaped like the real pepy JSON and shields.io SVG
-responses. Everything else -- count extraction, badge parsing, summing,
-formatting, the regression guards and the README rewrite -- is production code.
+The network layer is replaced with a router that answers with payloads shaped
+like the real GitHub, PyPI, pepy and shields.io responses. Everything else --
+repo discovery and ownership checks, counting, badge parsing,
+the regression guards and the README rewrite -- is production code.
Run with: python3 -m unittest discover -s scripts -p 'test_*.py' -v
"""
from __future__ import annotations
+import base64
import io
+import json
import os
import sys
import tempfile
@@ -27,6 +29,8 @@
udb = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(udb)
+TOKEN_VARS = ("PEPY_API_KEY", "GITHUB_TOKEN")
+
def shields_svg(label, value):
"""A shields.io flat-style badge, including the duplicated shadow text."""
@@ -41,348 +45,416 @@ def shields_svg(label, value):
"""
-class StubJson:
- def __init__(self, responses):
- self.responses = responses
- self.calls = []
-
- def __call__(self, url, headers=None, timeout=30):
- self.calls.append((url, headers or {}))
- for needle, payload in self.responses.items():
- if needle in url:
- if isinstance(payload, Exception):
- raise payload
- return payload
- raise urllib.error.URLError(f"unstubbed json url: {url}")
-
-
-class StubText:
- def __init__(self, responses):
- self.responses = responses
- self.calls = []
-
- def __call__(self, url, headers=None, timeout=30):
- self.calls.append((url, headers or {}))
- for needle, payload in self.responses.items():
- if needle in url:
- if isinstance(payload, Exception):
- raise payload
- return payload
- raise urllib.error.URLError(f"unstubbed text url: {url}")
-
+def repo(owner, name, stars=0, forks=0, private=False):
+ """A repository object as the GitHub listing API returns it (trimmed)."""
+ return {
+ "name": name,
+ "full_name": f"{owner}/{name}",
+ "owner": {"login": owner},
+ "private": private,
+ "visibility": "private" if private else "public",
+ "stargazers_count": stars,
+ "forks_count": forks,
+ }
+
+
+def pypi_meta(home=None, description=""):
+ return {
+ "info": {
+ "home_page": home,
+ "download_url": None,
+ "project_url": None,
+ "project_urls": None,
+ "description": description,
+ }
+ }
+
+
+def contents_file(name, text):
+ """What GET /repos/{o}/{r}/contents/{file} returns for a file."""
+ return {"content": base64.b64encode(text.encode("utf-8")).decode("ascii"), "name": name}
+
+
+def http_error(code):
+ return urllib.error.HTTPError("http://test", code, "test", None, None)
+
+
+class FakeWeb:
+ """Routes each URL to the most specific fragment that matches it.
+
+ A route value is a payload, an exception to raise, or a callable taking the
+ URL and returning either of those.
+ """
+
+ def __init__(self):
+ self.json = {}
+ self.text = {}
+ self.json_calls = []
+
+ def fetch_json(self, url, headers=None, timeout=30):
+ self.json_calls.append((url, headers or {}))
+ return self._route(self.json, url, headers)
+
+ def fetch_text(self, url, headers=None, timeout=30):
+ return self._route(self.text, url, headers)
+
+ @staticmethod
+ def _route(table, url, headers):
+ matches = [needle for needle in table if needle in url]
+ if not matches:
+ raise urllib.error.URLError(f"unstubbed url: {url}")
+ value = table[max(matches, key=len)]
+ if callable(value):
+ value = value(url, headers)
+ if isinstance(value, Exception):
+ raise value
+ return value
+
+
+def page_of(url):
+ """The ``page=`` query value of a listing URL (1 when absent)."""
+ for part in url.split("?", 1)[-1].split("&"):
+ if part.startswith("page="):
+ return int(part.split("=", 1)[1])
+ return 1
+
+
+def listing(repos):
+ """A repo listing that serves everything on page 1 and nothing after."""
+ return lambda url, headers: repos if page_of(url) == 1 else []
+
+
+def standard_world():
+ """Two owners, one private repo, one foreign repo, and packages of every kind."""
+ web = FakeWeb()
+ web.json.update(
+ {
+ "/users/johanbeytell1/repos": listing(
+ [
+ repo("johanbeytell1", "LocalShare", stars=1), # PyPI name taken by someone else
+ repo("johanbeytell1", "Alloy", stars=1),
+ repo("johanbeytell1", "Secret", stars=99, private=True),
+ ]
+ ),
+ "/orgs/Infinitode/repos": listing(
+ [
+ repo("Infinitode", "Hued", stars=2, forks=1),
+ repo("Infinitode", "qrforge"),
+ repo("Infinitode", "CloakJS"),
+ repo("someone-else", "hued-fork", stars=50), # not ours, even if listed
+ ]
+ ),
+ # PyPI: hued links back to our repo; qrforge only declares its name in setup.py;
+ # localshare is someone else's project; alloy and cloakjs are not on PyPI.
+ "pypi.org/pypi/hued/json": pypi_meta(home="https://github.com/Infinitode/Hued"),
+ "pypi.org/pypi/qrforge/json": pypi_meta(description="# qrforge\nNo links here."),
+ "pypi.org/pypi/localshare/json": pypi_meta(home="https://github.com/sunu/localshare"),
+ "pypi.org/pypi/alloy/json": http_error(404),
+ "pypi.org/pypi/cloakjs/json": http_error(404),
+ "/repos/Infinitode/qrforge/contents/setup.py": contents_file(
+ "setup.py", "setup(name='qrforge', version='1.0.1')"
+ ),
+ "/repos/Infinitode/qrforge/contents": [
+ {"type": "file", "name": "setup.py", "url": "https://api.github.com/repos/Infinitode/qrforge/contents/setup.py"},
+ {"type": "dir", "name": "tests", "url": "https://api.github.com/x"},
+ ],
+ "/repos/johanbeytell1/LocalShare/contents": [{"type": "file", "name": "readme.md", "url": "x"}],
+ "/repos/Infinitode/Hued/contents": http_error(404),
+ }
+ )
+ web.json["/repos/johanbeytell1/Alloy/contents"] = []
+ web.json["/repos/Infinitode/CloakJS/contents"] = []
+ return web
+
+
+def standard_text_world(web):
+ web.text.update(
+ {
+ "img.shields.io/pepy/dt/": lambda url, headers: shields_svg("downloads", "1.0k"),
+ "img.shields.io/jsdelivr/gh/hy/infinitode/cloakjs": shields_svg("hits", "250"),
+ "img.shields.io/jsdelivr/gh/hy/": shields_svg("hits", "0"),
+ }
+ )
+ return web
+
+
+class StubMixin:
+ """Points the module's network layer at a FakeWeb for the duration of a test."""
+
+ def install(self, web):
+ self._saved = (udb.fetch_json, udb.fetch_text)
+ udb.fetch_json, udb.fetch_text = web.fetch_json, web.fetch_text
+ self.addCleanup(self._restore)
+
+ def _restore(self):
+ udb.fetch_json, udb.fetch_text = self._saved
+
+
+# --- parsers and helpers ------------------------------------------------------
+
+
+class TestParsers(unittest.TestCase):
+ def test_shields_badge_reads_the_value(self):
+ self.assertEqual(udb.parse_shields_badge(shields_svg("hits", "5.6k")), 5600)
+ self.assertEqual(udb.parse_shields_badge(shields_svg("downloads", "1,234")), 1234)
+
+ def test_shields_error_badge_has_no_value(self):
+ self.assertIsNone(udb.parse_shields_badge(shields_svg("downloads", "unavailable")))
+ self.assertIsNone(udb.parse_shields_badge(""))
-PEPY_JSON = {"name": "codesafe", "total_downloads": 1234, "recent_downloads": {"last_month": 90}}
-BADGE_SVG = shields_svg("downloads", "1.2k")
+ def test_format_count(self):
+ self.assertEqual(udb.format_count(999), "999")
+ self.assertEqual(udb.format_count(45_678), "45.7k")
+ self.assertEqual(udb.format_count(1_234_567), "1.23M")
+ self.assertEqual(udb.format_count(999_999), "1.00M") # never "1000.0k"
+ self.assertEqual(udb.format_count(2_000_000_000), "2.00B")
+ def test_find_count_prefers_exact_all_time_keys(self):
+ self.assertEqual(udb.find_count({"downloads": {"last_month": 5}, "total_downloads": 9}), 9)
+ self.assertIsNone(udb.find_count({"nothing": "here"}))
-class TestShieldsBadgeParsing(unittest.TestCase):
- def test_reads_the_value_not_the_label(self):
- self.assertEqual(udb.parse_shields_badge(BADGE_SVG), 1200)
+ def test_parse_badge_value_reads_downloads_only(self):
+ text = udb.render_block([("Total Downloads", "5.17M"), ("Total Stars", "50")])
+ self.assertEqual(udb.parse_badge_value(text), 5_170_000)
+ self.assertIsNone(udb.parse_badge_value("no badge here"))
- def test_thousands_separator(self):
- self.assertEqual(udb.parse_shields_badge(shields_svg("hits", "1,234")), 1234)
+ def test_normalize_name_follows_pep503(self):
+ self.assertEqual(udb.normalize_name("Py_AutoPlot"), "py-autoplot")
+ self.assertEqual(udb.normalize_name("LocalSiteMap"), "localsitemap")
- def test_mega_suffix(self):
- self.assertEqual(udb.parse_shields_badge(shields_svg("downloads", "2.5M")), 2_500_000)
+ def test_is_counted_keeps_only_public_repos_of_the_owner(self):
+ self.assertTrue(udb.is_counted(repo("Infinitode", "Hued"), "infinitode"))
+ self.assertFalse(udb.is_counted(repo("Infinitode", "Hued", private=True), "Infinitode"))
+ self.assertFalse(udb.is_counted(repo("someone-else", "hued-fork"), "Infinitode"))
- def test_plain_integer(self):
- self.assertEqual(udb.parse_shields_badge(shields_svg("hits", "999")), 999)
+ def test_links_back_needs_the_exact_repo(self):
+ self.assertTrue(udb.links_back(pypi_meta(home="https://github.com/Infinitode/Hued"), "Infinitode", "Hued"))
+ self.assertFalse(udb.links_back(pypi_meta(home="https://github.com/sunu/localshare"), "johanbeytell1", "LocalShare"))
+ # a longer repo name must not count as a match for the shorter one
+ self.assertFalse(udb.links_back(pypi_meta(home="https://github.com/infinitode/localsitemap-tools"), "infinitode", "localsitemap"))
- def test_error_badge_yields_none(self):
- self.assertIsNone(udb.parse_shields_badge(shields_svg("downloads", "unavailable")))
+ def test_packaging_names(self):
+ self.assertEqual(udb.packaging_names("setup.py", "setup(name='qrforge', version='1')"), {"qrforge"})
+ self.assertEqual(udb.packaging_names("pyproject.toml", '[project]\nname = "hued"\n'), {"hued"})
+ self.assertEqual(udb.packaging_names("setup.cfg", "[metadata]\nname = valx\n"), {"valx"})
- def test_empty_input_yields_none(self):
- self.assertIsNone(udb.parse_shields_badge(""))
- self.assertIsNone(udb.parse_shields_badge(None))
- def test_title_element_is_not_mistaken_for_the_value(self):
- svg = shields_svg("downloads", "42")
- self.assertIn("downloads: 42", svg)
- self.assertEqual(udb.parse_shields_badge(svg), 42)
+class TestReadmeRewrite(unittest.TestCase):
+ def setUp(self):
+ self.readme = f"# Profile\n\n{udb.ANCHOR}\n\nSome text.\n"
+ self.values = [
+ ("Total Downloads", "1.23M"),
+ ("Total Stars", "50"),
+ ("Total Forks", "10"),
+ ]
+ def test_inserts_all_badges_after_visitor_counter_on_first_run(self):
+ updated, changed = udb.update_readme(self.readme, self.values)
+ self.assertTrue(changed)
+ self.assertIn("Total%20Downloads-1.23M-", updated)
+ self.assertIn("Total%20Stars-50-", updated)
+ self.assertIn("Total%20Forks-10-", updated)
+ self.assertLess(updated.index(udb.ANCHOR), updated.index(udb.START_MARK))
-class TestCountExtraction(unittest.TestCase):
- def test_finds_pepy_total_downloads(self):
- self.assertEqual(udb.find_count(PEPY_JSON), 1234)
+ def test_replaces_in_place_on_later_runs(self):
+ once, _ = udb.update_readme(self.readme, self.values)
+ twice, changed = udb.update_readme(once, [("Total Downloads", "2.00M"), ("Total Stars", "51")])
+ self.assertTrue(changed)
+ self.assertEqual(twice.count("TOTAL-DOWNLOADS:START"), 1)
+ self.assertIn("Total%20Downloads-2.00M-", twice)
+ self.assertIn("Total%20Stars-51-", twice)
+ self.assertNotIn("Total%20Forks", twice, "the block is rewritten as a whole")
- def test_finds_jsdelivr_total_hits(self):
- self.assertEqual(udb.find_count({"type": "github", "totalHits": 5678}), 5678)
+ def test_idempotent_for_same_values(self):
+ once, _ = udb.update_readme(self.readme, self.values)
+ twice, changed = udb.update_readme(once, self.values)
+ self.assertFalse(changed)
+ self.assertEqual(once, twice)
- def test_prefers_all_time_key_over_period_hits(self):
- self.assertEqual(udb.find_count({"hits": {"lastMonth": 50}, "totalHits": 999}), 999)
+ def test_no_anchor_and_no_marker_is_a_noop(self):
+ updated, changed = udb.update_readme("nothing here\n", self.values)
+ self.assertFalse(changed)
+ self.assertEqual(updated, "nothing here\n")
- def test_returns_none_when_absent_or_non_numeric(self):
- self.assertIsNone(udb.find_count({"name": "nothing"}))
- self.assertIsNone(udb.find_count({"downloads": True}))
+ def test_current_downloads_text_reads_the_published_value(self):
+ text, _ = udb.update_readme(self.readme, self.values)
+ self.assertEqual(udb.current_downloads_text(text), "1.23M")
+ self.assertIsNone(udb.current_downloads_text("no badge"))
-class TestFormatting(unittest.TestCase):
- def test_boundaries(self):
- cases = [
- (0, "0"), (999, "999"), (1_000, "1.0k"), (45_678, "45.7k"),
- (999_949, "999.9k"), (999_950, "1.00M"), (1_234_567, "1.23M"),
- (1_000_000_000, "1.00B"), (12_300_000_000, "12.30B"),
- ]
- for value, expected in cases:
- self.assertEqual(udb.format_count(value), expected, f"format_count({value})")
+# --- discovery ----------------------------------------------------------------
- def test_never_renders_1000k(self):
- self.assertNotIn("1000.0k", [udb.format_count(n) for n in range(990_000, 1_010_000, 997)])
- def test_value_is_url_safe(self):
- for value in (0, 999, 45_678, 999_950, 1_234_567):
- self.assertRegex(udb.format_count(value), r"^[0-9.]+[kMB]?$")
+class TestDiscovery(StubMixin, unittest.TestCase):
+ def setUp(self):
+ self.web = standard_world()
+ self.install(self.web)
+
+ def test_lists_only_public_repos_of_the_two_owners(self):
+ names = sorted(r["full_name"] for r in udb.discover_repos())
+ self.assertEqual(
+ names,
+ ["Infinitode/CloakJS", "Infinitode/Hued", "Infinitode/qrforge", "johanbeytell1/Alloy", "johanbeytell1/LocalShare"],
+ )
- def test_round_trips_through_parse_badge_value(self):
- for value in (0, 999, 45_678, 999_950, 1_234_567, 12_300_000_000):
- badge = udb.BADGE.format(value=udb.format_count(value))
- parsed = udb.parse_badge_value(badge)
- # rounded display loses precision, but must stay within 0.5%
- self.assertAlmostEqual(parsed, value, delta=max(1, value * 0.005))
+ def test_listing_follows_pages(self):
+ many = [repo("Infinitode", f"r{i}") for i in range(100)]
+ self.web.json["/orgs/Infinitode/repos"] = lambda url, headers: many if page_of(url) == 1 else [repo("Infinitode", "last")]
+ repos = udb.list_public_repos("Infinitode", "org", fetch_json=self.web.fetch_json)
+ self.assertEqual(len(repos), 101)
+
+ def test_sends_the_github_token_only_when_given(self):
+ udb.discover_repos(token="tok")
+ self.assertTrue(all(h.get("Authorization") == "Bearer tok" for _, h in self.web.json_calls))
+
+ def test_pypi_package_must_belong_to_the_repo(self):
+ pairs = {
+ "Infinitode/Hued": "hued", # linked from PyPI metadata
+ "Infinitode/qrforge": "qrforge", # declared by the repo's setup.py
+ "johanbeytell1/LocalShare": None, # PyPI "localshare" is someone else's
+ "johanbeytell1/Alloy": None, # not on PyPI at all
+ }
+ for full_name, expected in pairs.items():
+ owner, name = full_name.split("/")
+ found = udb.pypi_package_for(repo(owner, name), fetch_json=self.web.fetch_json)
+ self.assertEqual(found, expected, full_name)
+
+ def test_discover_pypi_maps_repos_to_packages(self):
+ repos = udb.discover_repos()
+ packages, notes = udb.discover_pypi(repos, fetch_json=self.web.fetch_json)
+ self.assertEqual(packages, {"Infinitode/Hued": "hued", "Infinitode/qrforge": "qrforge"})
+ self.assertEqual(notes, [])
+ def test_pypi_server_error_is_a_note_not_a_crash(self):
+ self.web.json["pypi.org/pypi/hued/json"] = http_error(500)
+ packages, notes = udb.discover_pypi(udb.discover_repos(), fetch_json=self.web.fetch_json)
+ self.assertNotIn("Infinitode/Hued", packages)
+ self.assertTrue(any("Infinitode/Hued" in n for n in notes))
-class TestPypiTotals(unittest.TestCase):
- def test_uses_the_pepy_api_when_a_key_is_set(self):
- fetch_json = StubJson({"api.pepy.tech": PEPY_JSON})
- totals, failures, notes = udb.pypi_totals(
- ["codesafe"], api_key="secret", fetch_json=fetch_json, fetch_text=StubText({})
- )
- self.assertEqual(totals, {"codesafe": 1234})
- self.assertEqual(failures, {})
- self.assertEqual(notes, [])
- self.assertEqual(fetch_json.calls[0][1], {"X-Api-Key": "secret"})
- def test_falls_back_to_shields_without_a_key(self):
- fetch_text = StubText({"img.shields.io": BADGE_SVG})
- totals, failures, notes = udb.pypi_totals(
- ["codesafe"], api_key=None, fetch_json=StubJson({}), fetch_text=fetch_text
- )
- self.assertEqual(totals, {"codesafe": 1200})
- self.assertEqual(failures, {})
- self.assertTrue(any("PEPY_API_KEY" in n for n in notes))
-
- def test_falls_back_to_shields_when_the_pepy_api_rejects_the_key(self):
- fetch_json = StubJson({"api.pepy.tech": urllib.error.HTTPError("u", 403, "Forbidden", {}, None)})
- totals, _, _ = udb.pypi_totals(
- ["codesafe"], api_key="bad", fetch_json=fetch_json, fetch_text=StubText({"img.shields.io": BADGE_SVG})
- )
- self.assertEqual(totals, {"codesafe": 1200})
-
- def test_records_failure_when_both_sources_fail(self):
- totals, failures, _ = udb.pypi_totals(
- ["codesafe"],
- api_key="k",
- fetch_json=StubJson({"api.pepy.tech": urllib.error.HTTPError("u", 403, "Forbidden", {}, None)}),
- fetch_text=StubText({"img.shields.io": shields_svg("downloads", "unavailable")}),
- )
- self.assertEqual(totals, {})
- self.assertIn("codesafe", failures)
-
-
-class TestJsdelivrTotals(unittest.TestCase):
- def test_reads_hits_from_the_badge(self):
- fetch_text = StubText({"img.shields.io": shields_svg("hits", "5.6k")})
- totals, failures = udb.jsdelivr_totals(["infinitode/blurjs"], fetch_text=fetch_text)
- self.assertEqual(totals, {"infinitode/blurjs": 5600})
- self.assertEqual(failures, {})
- self.assertIn("/jsdelivr/gh/hy/infinitode/blurjs", fetch_text.calls[0][0])
-
- def test_records_failure_for_an_error_badge(self):
- totals, failures = udb.jsdelivr_totals(
- ["infinitode/blurjs"],
- fetch_text=StubText({"img.shields.io": shields_svg("hits", "unavailable")}),
- )
- self.assertEqual(totals, {})
- self.assertIn("infinitode/blurjs", failures)
+# --- collection ---------------------------------------------------------------
-class TestReadmeRewrite(unittest.TestCase):
+class TestCollect(StubMixin, unittest.TestCase):
def setUp(self):
- self.readme = f"# Profile\n\n{udb.ANCHOR}\n\nSome text.\n"
+ self.web = standard_text_world(standard_world())
+ self.web.json["api.pepy.tech"] = lambda url, headers: {"total_downloads": 1_000}
+ self.install(self.web)
- def test_inserts_after_visitor_counter_on_first_run(self):
- updated, changed = udb.update_readme(self.readme, 1_234_567)
- self.assertTrue(changed)
- self.assertIn("Total%20Downloads-1.23M-", updated)
- self.assertLess(updated.index(udb.ANCHOR), updated.index(udb.START_MARK))
+ def test_stars_and_forks_are_summed_over_counted_repos(self):
+ result = udb.collect(api_key="k", fetch_json=self.web.fetch_json, fetch_text=self.web.fetch_text)
+ self.assertEqual(result["stars"], 1 + 1 + 2) # LocalShare, Alloy, Hued; Secret and the foreign fork excluded
+ self.assertEqual(result["forks"], 1)
- def test_replaces_in_place_on_later_runs(self):
- once, _ = udb.update_readme(self.readme, 1_000)
- twice, changed = udb.update_readme(once, 2_000_000)
- self.assertTrue(changed)
- self.assertEqual(twice.count("TOTAL-DOWNLOADS:START"), 1)
- self.assertIn("2.00M", twice)
+ def test_downloads_come_from_pypi_and_jsdelivr(self):
+ result = udb.collect(api_key="k", fetch_json=self.web.fetch_json, fetch_text=self.web.fetch_text)
+ self.assertEqual(result["pypi"], {"hued": 1_000, "qrforge": 1_000})
+ self.assertEqual(result["jsdelivr"]["infinitode/cloakjs"], 250)
+ self.assertEqual(result["downloads"], 1_000 + 1_000 + 250)
+ self.assertNotIn("localshare", result["pypi"])
- def test_idempotent_for_same_value(self):
- once, _ = udb.update_readme(self.readme, 1234)
- twice, changed = udb.update_readme(once, 1234)
- self.assertFalse(changed)
- self.assertEqual(once, twice)
- def test_no_anchor_and_no_marker_is_a_noop(self):
- updated, changed = udb.update_readme("nothing here\n", 1)
- self.assertFalse(changed)
+
+# --- entry point --------------------------------------------------------------
-class TestMain(unittest.TestCase):
- """Drives the real main() against a throwaway copy of the README."""
+class TestMain(StubMixin, unittest.TestCase):
+ """Drives the real main() against a throwaway README."""
def setUp(self):
- self.readme = f"# Profile\n\n{udb.ANCHOR}\n"
- self.path = self._write(self.readme)
-
- def _write(self, text):
- handle = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8")
- handle.write(text)
- handle.close()
- self.addCleanup(os.unlink, handle.name)
- return handle.name
-
- def _read(self):
- with open(self.path, encoding="utf-8") as handle:
+ self.readme_text = f"# Profile\n\n{udb.ANCHOR}\n"
+ self.dir = tempfile.TemporaryDirectory()
+ self.addCleanup(self.dir.cleanup)
+ self.readme = os.path.join(self.dir.name, "README.md")
+ self._write(self.readme, self.readme_text)
+ saved = {var: os.environ.pop(var) for var in TOKEN_VARS if var in os.environ}
+ self.addCleanup(os.environ.update, saved)
+ self.addCleanup(lambda: [os.environ.pop(var, None) for var in TOKEN_VARS])
+
+ @staticmethod
+ def _write(path, text):
+ with open(path, "w", encoding="utf-8") as handle:
+ handle.write(text)
+
+ def _read(self, path):
+ with open(path, encoding="utf-8") as handle:
return handle.read()
- def _run(self, argv, json_responses=None, text_responses=None, api_key=None):
- # text_responses is keyed by URL fragment: "pepy" or "jsdelivr".
- if api_key is None:
- os.environ.pop("PEPY_API_KEY", None)
- else:
- os.environ["PEPY_API_KEY"] = api_key
- self.addCleanup(os.environ.pop, "PEPY_API_KEY", None)
-
- udb.fetch_json = StubJson(json_responses or {})
- udb.fetch_text = StubText(text_responses or {})
+ def _run(self, web, argv=(), env=None):
+ self.install(web)
+ for var, value in (env or {}).items():
+ os.environ[var] = value
buffer = io.StringIO()
- original_stdout, sys.stdout = sys.stdout, buffer
+ original_stdout, original_stderr = sys.stdout, sys.stderr
+ sys.stdout = sys.stderr = buffer
try:
- code = udb.main(argv)
+ code = udb.main([*argv, "--readme", self.readme])
finally:
- sys.stdout = original_stdout
+ sys.stdout, sys.stderr = original_stdout, original_stderr
return code, buffer.getvalue()
- def test_all_sources_failing_leaves_the_badge_untouched(self):
- """The regression that shipped: a total wipeout must never publish 0."""
- dead = shields_svg("downloads", "unavailable")
- code, _ = self._run(
- ["--readme", self.path],
- text_responses={"pepy": dead, "jsdelivr": shields_svg("hits", "unavailable")},
- )
+ def _healthy_web(self):
+ web = standard_text_world(standard_world())
+ web.json["api.pepy.tech"] = lambda url, headers: {"total_downloads": 1_000}
+ return web
+
+ def test_listing_failure_changes_nothing(self):
+ web = self._healthy_web()
+ web.json["/orgs/Infinitode/repos"] = http_error(503)
+ code, _ = self._run(web)
self.assertEqual(code, 1)
- self.assertEqual(self._read(), self.readme, "README must stay byte-identical")
-
- def test_published_zero_is_treated_as_unpublished(self):
- """A bogus 0 left by a failed run must not block the next good one."""
- with open(self.path, "w", encoding="utf-8") as handle:
- handle.write(udb.update_readme(self.readme, 0)[0])
- self.assertIn("Total%20Downloads-0-", self._read())
-
- code, _ = self._run(
- ["--readme", self.path],
- json_responses={pkg: {"total_downloads": 1_000} for pkg in udb.PYPI_PACKAGES},
- text_responses={"jsdelivr": shields_svg("hits", "0")},
- api_key="k",
- )
- self.assertEqual(code, 0)
- self.assertNotIn("Total%20Downloads-0-", self._read())
- self.assertIn(udb.format_count(1_000 * len(udb.PYPI_PACKAGES)), self._read())
-
- def test_lower_total_is_refused(self):
- with open(self.path, "w", encoding="utf-8") as handle:
- handle.write(udb.update_readme(self.readme, 10_000_000)[0])
- code, _ = self._run(
- ["--readme", self.path],
- json_responses={pkg: {"total_downloads": 100} for pkg in udb.PYPI_PACKAGES},
- text_responses={"jsdelivr": shields_svg("hits", "0")},
- api_key="k",
- )
- self.assertEqual(code, 0)
- self.assertIn("10.00M", self._read())
-
- def test_rounding_level_decrease_still_writes(self):
- """"5.17M" stands for [5,165,000, 5,175,000); 5,168,400 is not a drop."""
- with open(self.path, "w", encoding="utf-8") as handle:
- handle.write(udb.update_readme(self.readme, 5_170_000)[0])
- self.assertIn("5.17M", self._read())
-
- per_package = 5_168_400 // len(udb.PYPI_PACKAGES)
- total = per_package * len(udb.PYPI_PACKAGES)
- self.assertLess(total, 5_170_000) # genuinely below the published figure
- self.assertGreater(total, 5_170_000 * 0.95) # but inside the tolerance
-
- code, output = self._run(
- ["--readme", self.path],
- json_responses={pkg: {"total_downloads": per_package} for pkg in udb.PYPI_PACKAGES},
- text_responses={"jsdelivr": shields_svg("hits", "0")},
- api_key="k",
- )
- self.assertEqual(code, 0)
- self.assertNotIn("Refusing", output, "rounding noise must not trip the guard")
- self.assertIn(udb.format_count(total), self._read())
-
- def test_drop_beyond_tolerance_is_refused(self):
- with open(self.path, "w", encoding="utf-8") as handle:
- handle.write(udb.update_readme(self.readme, 10_000_000)[0])
- # ~40% drop -- one or more sources missing
- code, _ = self._run(
- ["--readme", self.path],
- json_responses={pkg: {"total_downloads": 400_000} for pkg in udb.PYPI_PACKAGES},
- text_responses={"jsdelivr": shields_svg("hits", "0")},
- api_key="k",
- )
- self.assertEqual(code, 0)
- self.assertIn("10.00M", self._read())
-
- def test_allow_decrease_overrides_the_guard(self):
- with open(self.path, "w", encoding="utf-8") as handle:
- handle.write(udb.update_readme(self.readme, 10_000_000)[0])
- code, _ = self._run(
- ["--readme", self.path, "--allow-decrease"],
- json_responses={pkg: {"total_downloads": 100} for pkg in udb.PYPI_PACKAGES},
- text_responses={"jsdelivr": shields_svg("hits", "0")},
- api_key="k",
- )
- self.assertEqual(code, 0)
- expected = 100 * len(udb.PYPI_PACKAGES)
- self.assertIn(udb.format_count(expected), self._read())
-
- def test_partial_failure_still_publishes_the_rest(self):
- responses = {pkg: {"total_downloads": 1_000} for pkg in udb.PYPI_PACKAGES}
- responses["qrforge"] = urllib.error.URLError("boom")
- code, _ = self._run(
- ["--readme", self.path],
- json_responses=responses,
- text_responses={
- "pepy": shields_svg("downloads", "unavailable"), # fallback also dead
- "jsdelivr": shields_svg("hits", "4k"),
- },
- api_key="k",
- )
- self.assertEqual(code, 0)
- expected = 1_000 * (len(udb.PYPI_PACKAGES) - 1) + 4_000
- self.assertIn(udb.format_count(expected), self._read())
-
- def test_happy_path_writes_the_tally(self):
- code, output = self._run(
- ["--readme", self.path],
- json_responses={pkg: {"total_downloads": 1_000_000} for pkg in udb.PYPI_PACKAGES},
- text_responses={"jsdelivr": shields_svg("hits", "400k")},
- api_key="k",
- )
- self.assertEqual(code, 0)
- expected = 1_000_000 * len(udb.PYPI_PACKAGES) + 400_000
- self.assertIn(f"{expected:>14,}", output)
- self.assertIn(udb.format_count(expected), self._read())
+ self.assertEqual(self._read(self.readme), self.readme_text)
+
+ def test_every_download_source_failing_changes_nothing(self):
+ web = standard_world()
+ web.json["api.pepy.tech"] = http_error(500)
+ web.text["img.shields.io"] = urllib.error.URLError("offline")
+ code, _ = self._run(web, env={"PEPY_API_KEY": "k"})
+ self.assertEqual(code, 1)
+ self.assertEqual(self._read(self.readme), self.readme_text)
+
+ def test_tokenless_run_writes_downloads_stars_and_forks(self):
+ code, output = self._run(self._healthy_web())
+ self.assertEqual(code, 0, output)
+ readme = self._read(self.readme)
+ self.assertIn("Total%20Downloads-", readme)
+ self.assertIn("Total%20Stars-4-", readme) # LocalShare 1 + Alloy 1 + Hued 2
+ self.assertIn("Total%20Forks-1-", readme)
+ self.assertNotIn("Clones", readme)
+
+ def test_happy_path_with_pepy_key_is_exact(self):
+ code, output = self._run(self._healthy_web(), env={"PEPY_API_KEY": "k"})
+ self.assertEqual(code, 0, output)
+ # PyPI: hued + qrforge at 1,000 each (pepy); jsDelivr: CloakJS 250, the rest 0.
+ self.assertIn(f"Total%20Downloads-{udb.format_count(2_250)}-", self._read(self.readme))
def test_dry_run_changes_nothing(self):
- code, output = self._run(
- ["--readme", self.path, "--dry-run"],
- json_responses={pkg: {"total_downloads": 7} for pkg in udb.PYPI_PACKAGES},
- text_responses={"jsdelivr": shields_svg("hits", "8")},
- api_key="k",
- )
+ code, output = self._run(self._healthy_web(), argv=["--dry-run"], env={"PEPY_API_KEY": "k"})
self.assertEqual(code, 0)
self.assertIn("--dry-run", output)
- self.assertEqual(self._read(), self.readme)
+ self.assertEqual(self._read(self.readme), self.readme_text)
+
+ def test_drop_beyond_tolerance_keeps_the_published_downloads(self):
+ self._write(self.readme, udb.update_readme(self.readme_text, [("Total Downloads", "10.00M")])[0])
+ code, output = self._run(self._healthy_web(), env={"PEPY_API_KEY": "k"})
+ self.assertEqual(code, 0)
+ self.assertIn("Refusing", output)
+ readme = self._read(self.readme)
+ self.assertIn("Total%20Downloads-10.00M-", readme)
+ self.assertIn("Total%20Stars-4-", readme, "other figures still refresh")
+
+ def test_allow_decrease_publishes_a_lower_total(self):
+ self._write(self.readme, udb.update_readme(self.readme_text, [("Total Downloads", "10.00M")])[0])
+ code, _ = self._run(self._healthy_web(), argv=["--allow-decrease"], env={"PEPY_API_KEY": "k"})
+ self.assertEqual(code, 0)
+ self.assertNotIn("Total%20Downloads-10.00M-", self._read(self.readme))
+
+ def test_repos_are_discovered_without_any_hand_list(self):
+ code, output = self._run(self._healthy_web(), argv=["--dry-run"], env={"PEPY_API_KEY": "k"})
+ self.assertIn("Infinitode/CloakJS", output)
+ self.assertIn("johanbeytell1/Alloy", output)
+ self.assertNotIn("hued-fork", output)
+ self.assertNotIn("Secret", output)
if __name__ == "__main__":
diff --git a/scripts/update-download-badges.py b/scripts/update-download-badges.py
index f2bd16d..5b3fb1c 100644
--- a/scripts/update-download-badges.py
+++ b/scripts/update-download-badges.py
@@ -1,37 +1,46 @@
#!/usr/bin/env python3
-"""Tally PyPI and jsDelivr downloads and refresh the profile badge.
-
-Data sources, in order of preference
-------------------------------------
-PyPI (all-time)
- 1. pepy v2 API -- ``https://api.pepy.tech/api/v2/projects/{pkg}`` with an
- ``X-Api-Key`` header. Exact counts. Requires the ``PEPY_API_KEY``
- secret; pepy retired its keyless endpoint (it now returns 403).
- 2. shields.io ``/pepy/dt/{pkg}`` -- the same endpoint the per-package
- badges in this README already use. No key needed, but the figure is
- rounded ("12.3k"), so the total is approximate.
-
-jsDelivr
- shields.io ``/jsdelivr/gh/{period}/{owner}/{repo}``. jsDelivr publishes
- period-based stats only (hour/day/week/month/year), so the default
- period is ``hy`` = last 12 months. There is no all-time jsDelivr figure.
+"""Tally downloads, stars and forks across Infinitode's public repos.
+
+Nothing is hand-maintained. Every public repository owned by one of OWNERS is
+discovered from the GitHub API and counted; anything else is ignored. Packages
+are only counted when they are public and belong to one of those repositories.
+
+What is measured
+----------------
+Stars and forks (exact, all-time)
+ ``stargazers_count`` and ``forks_count`` from the GitHub repo listing.
+
+Downloads
+ PyPI (all-time). A PyPI package belongs to a repo when it has the repo's
+ name AND either its PyPI metadata links back to the repo, or the repo's own
+ packaging file (pyproject.toml / setup.py / setup.cfg) declares that name.
+ Same-named packages by other people fail both checks, so they are skipped.
+ 1. pepy v2 API when PEPY_API_KEY is set (exact counts), otherwise
+ 2. shields.io ``/pepy/dt`` badge (rounded figures).
+ jsDelivr (last 12 months). shields.io ``/jsdelivr/gh/hy`` for every counted
+ repo. jsDelivr publishes period stats only; there is no all-time figure.
Safety
------
-A failed fetch never overwrites the badge. The badge is only rewritten when
-at least one source returned a number, and never to a value lower than the
-one already published (download totals are monotonically increasing, so a
-drop means we are looking at partial data).
+A failed fetch never overwrites published numbers. If the repo listing fails,
+nothing is written. If every download source fails, nothing is written. A
+download total is only published when it is not more than 5% below the figure
+already shown (download totals only increase, so a drop means partial data).
+Stars and forks are not guarded, because they can legitimately fall.
Usage::
python3 scripts/update-download-badges.py --readme profile/README.md
python3 scripts/update-download-badges.py --dry-run
+
+Environment: PEPY_API_KEY (optional, exact PyPI counts). GITHUB_TOKEN is also
+read if present, which only raises the GitHub rate limit; nothing needs it.
"""
from __future__ import annotations
import argparse
+import base64
import gzip
import json
import os
@@ -42,27 +51,18 @@
# --- what gets counted -------------------------------------------------------
-PYPI_PACKAGES = [
- "qrforge",
- "localsitemap",
- "pyautoplot",
- "valx",
- "hued",
- "deepdefend",
- "codesafe",
- "pywebscrapr",
- "funcprofiler",
- "duplipy",
-]
-
-JSDELIVR_GH = [
- "infinitode/blurjs",
- # "johanbeytell1/alloy", # uncomment once it is served through jsDelivr
-]
+# (login, account type). Only public repositories owned by these accounts count.
+OWNERS = (("johanbeytell1", "user"), ("Infinitode", "org"))
# hy = hits per year, the longest window jsDelivr exposes.
JSDELIVR_PERIOD = "hy"
+GITHUB_API = "https://api.github.com"
+GITHUB_HEADERS = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"}
+MAX_PAGES = 20
+PACKAGING_FILES = ("pyproject.toml", "setup.py", "setup.cfg")
+
+PYPI_JSON = "https://pypi.org/pypi/{pkg}/json"
PEPY_V2 = "https://api.pepy.tech/api/v2/projects/{pkg}"
SHIELDS_PYPI = "https://img.shields.io/pepy/dt/{pkg}"
SHIELDS_JSDELIVR = "https://img.shields.io/jsdelivr/gh/{period}/{repo}"
@@ -75,19 +75,24 @@
# Keys that hold an all-time count, highest confidence first.
COUNT_KEYS = ("total_downloads", "totaldownloads", "totalhits", "downloads", "hits")
+START_MARK = ""
+END_MARK = ""
START_RE = re.compile(
r"", re.S
)
-START_MARK = ""
-END_MARK = ""
-BADGE = ""
-BADGE_VALUE_RE = re.compile(r"Total%20Downloads-([0-9][0-9.,]*)([kMB]?)-")
ANCHOR = ""
+BADGE_COLOR = "5d17eb"
+NOT_AVAILABLE = "n%2Fa" # shields.io path form of "n/a"
+BADGE_VALUE_RE = re.compile(r"Total%20Downloads-([0-9][0-9.,]*)([kMB]?)-")
+CURRENT_DOWNLOADS_RE = re.compile(r"Total%20Downloads-([^-\s)]+)-")
TEXT_RE = re.compile(r"]*>([^<]*)")
SHIELDS_VALUE_RE = re.compile(r"^([0-9][0-9.,]*)\s*([kMB]?)$")
_SUFFIX = {"": 1, "k": 1_000, "M": 1_000_000, "B": 1_000_000_000}
+_PY_NAME_RE = re.compile(r"""\bname\s*=\s*["']([A-Za-z0-9][A-Za-z0-9._-]*)["']""")
+_CFG_NAME_RE = re.compile(r"^name\s*=\s*([A-Za-z0-9][A-Za-z0-9._-]*)\s*$", re.M)
+
# --- network (the only I/O in this module) -----------------------------------
@@ -117,7 +122,14 @@ def fetch_text(url, headers=None, timeout=30):
return fetch(url, headers, timeout)[0].decode("utf-8", "replace")
-# --- parsers (pure) ----------------------------------------------------------
+def github_headers(token=None):
+ headers = dict(GITHUB_HEADERS)
+ if token:
+ headers["Authorization"] = f"Bearer {token}"
+ return headers
+
+
+# --- parsers and pure helpers (no I/O) ---------------------------------------
def _search(node, key):
@@ -173,13 +185,147 @@ def format_count(total):
def parse_badge_value(text):
- """Read the number currently published in the badge, or ``None``."""
+ """Read the download total currently published in the badge, or ``None``."""
match = BADGE_VALUE_RE.search(text or "")
if not match:
return None
return int(round(float(match.group(1).replace(",", "")) * _SUFFIX[match.group(2)]))
+def normalize_name(name):
+ """PyPI's name normalisation (PEP 503): case-insensitive, runs of -_. equal."""
+ return re.sub(r"[-_.]+", "-", name).lower()
+
+
+def is_counted(repo, owner):
+ """True for a public repository that is owned by *owner*."""
+ login = (repo.get("owner") or {}).get("login", "")
+ return (
+ login.lower() == owner.lower()
+ and not repo.get("private")
+ and repo.get("visibility", "public") == "public"
+ )
+
+
+def links_back(meta, owner, repo):
+ """True if the PyPI project's metadata links to github.com/owner/repo."""
+ info = meta.get("info") or {}
+ urls = [info.get(key) for key in ("home_page", "download_url", "project_url")]
+ urls += list((info.get("project_urls") or {}).values())
+ text = " ".join(str(url) for url in urls if url) + "\n" + (info.get("description") or "")
+ pattern = rf"github\.com/{re.escape(owner)}/{re.escape(repo)}(?![\w-]|\.\w)"
+ return re.search(pattern, text, re.I) is not None
+
+
+def packaging_names(filename, text):
+ """Package names a packaging file declares."""
+ if filename == "setup.cfg":
+ return set(_CFG_NAME_RE.findall(text))
+ return set(_PY_NAME_RE.findall(text))
+
+
+def render_block(values):
+ """The badge block. *values* is a list of ``(label, badge_text)`` pairs."""
+ lines = [START_MARK]
+ for label, text in values:
+ label_path = label.replace(" ", "%20")
+ lines.append(f"")
+ lines.append(END_MARK)
+ return "\n".join(lines)
+
+
+def update_readme(text, values):
+ """Replace (or insert) the badge block. Returns ``(new_text, changed)``."""
+ block = render_block(values)
+ if START_RE.search(text):
+ new_text, _ = START_RE.subn(lambda _match: block, text, count=1)
+ return new_text, new_text != text
+ if ANCHOR in text: # first run: drop the badges under the visitor counter
+ return text.replace(ANCHOR, f"{ANCHOR}\n{block}", 1), True
+ return text, False
+
+
+def current_downloads_text(text):
+ """The badge text currently shown for downloads, or ``None``."""
+ match = CURRENT_DOWNLOADS_RE.search(text or "")
+ return match.group(1) if match else None
+
+
+# --- discovery ---------------------------------------------------------------
+
+
+def list_public_repos(owner, kind, fetch_json=None, token=None):
+ """Every public repository owned by *owner*. Raises on API failure."""
+ fetch_json = fetch_json or globals()["fetch_json"]
+ base = f"{GITHUB_API}/{'orgs' if kind == 'org' else 'users'}/{owner}/repos"
+ listing = "public" if kind == "org" else "owner"
+ repos = []
+ for page in range(1, MAX_PAGES + 1):
+ batch = fetch_json(f"{base}?type={listing}&per_page=100&page={page}", github_headers(token))
+ if not isinstance(batch, list):
+ raise ValueError(f"unexpected repository listing for {owner}")
+ repos.extend(repo for repo in batch if is_counted(repo, owner))
+ if len(batch) < 100:
+ break
+ return repos
+
+
+def discover_repos(fetch_json=None, token=None):
+ repos = []
+ for owner, kind in OWNERS:
+ repos.extend(list_public_repos(owner, kind, fetch_json=fetch_json, token=token))
+ return repos
+
+
+def declared_names(owner, repo, fetch_json=None, token=None):
+ """Package names declared by the repo's own packaging files."""
+ fetch_json = fetch_json or globals()["fetch_json"]
+ try:
+ listing = fetch_json(f"{GITHUB_API}/repos/{owner}/{repo}/contents", github_headers(token))
+ except urllib.error.HTTPError as error:
+ if error.code == 404:
+ return set()
+ raise
+ names = set()
+ for item in listing if isinstance(listing, list) else []:
+ if item.get("type") == "file" and item.get("name") in PACKAGING_FILES:
+ blob = fetch_json(item["url"], github_headers(token))
+ text = base64.b64decode(blob.get("content", "")).decode("utf-8", "replace")
+ names |= packaging_names(item["name"], text)
+ return names
+
+
+def pypi_package_for(repo, fetch_json=None, token=None):
+ """The PyPI package that belongs to *repo*, or ``None``."""
+ fetch_json = fetch_json or globals()["fetch_json"]
+ owner, name = repo["owner"]["login"], repo["name"]
+ package = normalize_name(name)
+ try:
+ meta = fetch_json(PYPI_JSON.format(pkg=package))
+ except urllib.error.HTTPError as error:
+ if error.code == 404:
+ return None
+ raise
+ if links_back(meta, owner, name):
+ return package
+ declared = {normalize_name(n) for n in declared_names(owner, name, fetch_json, token)}
+ return package if package in declared else None
+
+
+def discover_pypi(repos, fetch_json=None, token=None):
+ """Map ``full_name -> PyPI package`` for every repo that has one."""
+ packages, notes = {}, []
+ for repo in repos:
+ try:
+ package = pypi_package_for(repo, fetch_json=fetch_json, token=token)
+ except (urllib.error.URLError, ValueError) as error:
+ notes.append(f"{repo['full_name']}: PyPI lookup skipped ({error})")
+ continue
+ if package:
+ packages[repo["full_name"]] = package
+ return packages, notes
+
+
# --- source lookups ----------------------------------------------------------
@@ -217,7 +363,7 @@ def pypi_totals(packages, api_key=None, fetch_json=None, fetch_text=None):
def jsdelivr_totals(repos, period=JSDELIVR_PERIOD, fetch_text=None):
- """jsDelivr hits per repo for *period* (default: last 12 months)."""
+ """jsDelivr hits per repo (``owner/repo``) for *period* (default: 12 months)."""
fetch_text = fetch_text or globals()["fetch_text"]
totals, failures = {}, {}
for repo in repos:
@@ -234,104 +380,125 @@ def jsdelivr_totals(repos, period=JSDELIVR_PERIOD, fetch_text=None):
return totals, failures
-def collect(api_key=None, fetch_json=None, fetch_text=None):
+# --- collection --------------------------------------------------------------
+
+
+def collect(api_key=None, github_token=None, fetch_json=None, fetch_text=None):
+ """Gather every download, star and fork figure."""
fetch_json = fetch_json or globals()["fetch_json"]
fetch_text = fetch_text or globals()["fetch_text"]
- pypi, pypi_failures, notes = pypi_totals(
- PYPI_PACKAGES, api_key=api_key, fetch_json=fetch_json, fetch_text=fetch_text
+
+ repos = discover_repos(fetch_json=fetch_json, token=github_token)
+ notes = []
+
+ stars = sum(int(repo.get("stargazers_count") or 0) for repo in repos)
+ forks = sum(int(repo.get("forks_count") or 0) for repo in repos)
+
+ packages, pypi_notes = discover_pypi(repos, fetch_json=fetch_json, token=github_token)
+ notes += pypi_notes
+ pypi, pypi_failures, pypi_notes = pypi_totals(
+ sorted(set(packages.values())), api_key=api_key, fetch_json=fetch_json, fetch_text=fetch_text
)
- jsdelivr, jsdelivr_failures = jsdelivr_totals(JSDELIVR_GH, fetch_text=fetch_text)
+ notes += pypi_notes
+
+ jsdelivr, jsdelivr_failures = jsdelivr_totals(
+ [repo["full_name"].lower() for repo in repos], fetch_text=fetch_text
+ )
+
return {
+ "repos": [repo["full_name"] for repo in repos],
+ "stars": stars,
+ "forks": forks,
+ "packages": packages,
"pypi": pypi,
"jsdelivr": jsdelivr,
"failures": {**pypi_failures, **jsdelivr_failures},
- "notes": notes,
- "sources": len(pypi) + len(jsdelivr),
- "total": sum(pypi.values()) + sum(jsdelivr.values()),
+ "notes": sorted(set(notes)),
+ "download_sources": len(pypi) + len(jsdelivr),
+ "downloads": sum(pypi.values()) + sum(jsdelivr.values()),
}
-# --- README rewrite ----------------------------------------------------------
-
-
-def render_block(total):
- return f"{START_MARK}\n{BADGE.format(value=format_count(total))}\n{END_MARK}"
-
-
-def update_readme(text, total):
- """Replace (or insert) the badge block. Returns ``(new_text, changed)``."""
- block = render_block(total)
- if START_RE.search(text):
- new_text, _ = START_RE.subn(lambda _match: block, text, count=1)
- return new_text, new_text != text
- if ANCHOR in text: # first run: drop the badge under the visitor counter
- return text.replace(ANCHOR, f"{ANCHOR}\n{block}", 1), True
- return text, False
-
-
# --- entry point -------------------------------------------------------------
def main(argv=None):
- parser = argparse.ArgumentParser(description="Refresh the total downloads badge.")
+ parser = argparse.ArgumentParser(description="Refresh the profile totals badges.")
parser.add_argument("--readme", default="profile/README.md")
parser.add_argument("--dry-run", action="store_true", help="print the result, change nothing")
parser.add_argument(
"--allow-decrease",
action="store_true",
- help="write the total even if it is lower than the published one",
+ help="publish the download total even if it is lower than the published one",
)
args = parser.parse_args(argv)
- result = collect(api_key=os.environ.get("PEPY_API_KEY"))
+ try:
+ result = collect(
+ api_key=os.environ.get("PEPY_API_KEY") or None,
+ github_token=os.environ.get("GITHUB_TOKEN") or None,
+ )
+ except (urllib.error.URLError, ValueError) as error:
+ print(f"Could not list repositories ({error}); nothing was changed.", file=sys.stderr)
+ return 1
for note in result["notes"]:
print(f" note {note}")
+ print(f" repos {len(result['repos'])} public repos counted: {', '.join(sorted(result['repos']))}")
+ for repo, package in sorted(result["packages"].items()):
+ print(f" package {repo:<40} -> pypi/{package}")
for name, count in sorted(result["pypi"].items()):
- print(f" pypi {name:<20} {count:>14,}")
+ print(f" pypi {name:<29} {count:>14,}")
for name, count in sorted(result["jsdelivr"].items()):
print(f" jsdelivr {name:<29} {count:>14,}")
for name, reason in sorted(result["failures"].items()):
print(f" FAILED {name}: {reason}", file=sys.stderr)
- print(f" {'TOTAL':<38} {result['total']:>14,} ({format_count(result['total'])})")
+ print(f" {'DOWNLOADS':<38} {result['downloads']:>14,} ({format_count(result['downloads'])})")
+ print(f" {'STARS':<38} {result['stars']:>14,}")
+ print(f" {'FORKS':<38} {result['forks']:>14,}")
- # Never publish a number we could not actually measure.
- if result["sources"] == 0:
- print("Every source failed; the badge is left untouched.", file=sys.stderr)
+ # Never publish a download total we could not actually measure.
+ if result["download_sources"] == 0:
+ print("Every download source failed; the README is left untouched.", file=sys.stderr)
return 1
with open(args.readme, encoding="utf-8") as handle:
original = handle.read()
+ if not START_RE.search(original) and ANCHOR not in original:
+ print(f"No badge block or anchor found in {args.readme}.", file=sys.stderr)
+ return 1
+ # Download totals only increase, so a drop means partial data. A published
+ # 0 is a leftover from a failed run, not a real measurement.
published = parse_badge_value(original)
- # A published 0 is a leftover from a failed run, not a real measurement.
- # Allow for display rounding: "5.17M" can stand for anything in
- # [5,165,000, 5,175,000), so a tiny apparent drop is noise, not data loss.
- if (
- published
- and result["total"] < published * (1 - DROP_TOLERANCE)
- and not args.allow_decrease
- ):
+ downloads_text = current_downloads_text(original) or NOT_AVAILABLE
+ if published and result["downloads"] < published * (1 - DROP_TOLERANCE) and not args.allow_decrease:
print(
- f"Refusing to publish {result['total']:,} over {published:,} "
+ f"Refusing to publish {result['downloads']:,} downloads over {published:,} "
f"(more than {DROP_TOLERANCE:.0%} below the published figure; looks like "
- "partial data -- pass --allow-decrease to override).",
+ "partial data -- pass --allow-decrease to override). Downloads badge kept.",
file=sys.stderr,
)
- return 0
+ else:
+ downloads_text = format_count(result["downloads"])
+
+ values = [
+ ("Total Downloads", downloads_text),
+ ("Total Stars", format_count(result["stars"])),
+ ("Total Forks", format_count(result["forks"])),
+ ]
if args.dry_run:
- print("--dry-run: README left untouched")
+ print("--dry-run: nothing written")
return 0
- updated, changed = update_readme(original, result["total"])
+ updated, changed = update_readme(original, values)
if not changed:
- print(f"Badge already up to date in {args.readme}.")
+ print(f"Badges already up to date in {args.readme}.")
return 0
-
with open(args.readme, "w", encoding="utf-8") as handle:
handle.write(updated)
- print(f"Updated the badge in {args.readme}.")
+ print(f"Updated the badges in {args.readme}.")
return 0