From 3080b914ea627082411e5d916360c52efa154806 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 09:14:35 +0000 Subject: [PATCH 1/2] fix(wikis): wait out BWIKI's EdgeOne rate limiting instead of giving up after 3 s The scheduled source check of 2026-10-19 (run 39224451765) failed in discover_bwiki(): BWIKI answered a pages() batch with HTTP 567, EdgeOne's rate limiting, three times in a row. 567 was already retried, but on the same schedule as a server error, 3 attempts 1 s and 2 s apart, so the run stopped about 3 s after the first 567, well inside any rate-limit window. The rate-limit statuses (RATE_LIMITED: 429 and 567) now get RATE_LIMIT_ATTEMPTS (6) calls, waiting 5, 10, 20, 30 and 30 s when the server sends no Retry-After, about 95 s in all. Connection errors, timeouts and 500, 502, 503 and 504 keep 3 attempts 1 s and 2 s apart, so a host that is down still fails fast. A Retry-After still replaces any wait, capped at 30 s. Attempts count across statuses and the latest error decides the limit. tests/test_fetch_wikis.py pins both schedules, the mixed case, and the sentence in docs/sources-wikis.md and its Chinese mirror, computed from retry_delay() and the constants. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AZypKz9iupFY3Dhsd8Q1rK --- docs/sources-wikis.md | 2 +- docs/sources-wikis.zh-CN.md | 2 +- scripts/fetch_wikis.py | 21 ++++++--- tests/test_fetch_wikis.py | 89 +++++++++++++++++++++++++++++++++---- 4 files changed, 98 insertions(+), 16 deletions(-) diff --git a/docs/sources-wikis.md b/docs/sources-wikis.md index 558195b..3cdf8f7 100644 --- a/docs/sources-wikis.md +++ b/docs/sources-wikis.md @@ -50,7 +50,7 @@ python -I scripts/fetch_wikis.py --resume --workers 4 `--resume` never drops a record from `wikis.json` and never deletes a file. A record whose candidate is no longer in the snapshot is kept with `"upstream_status": "removed"`. When a re-uploaded file is downloaded, the new version keeps the asset ID, so deep links stay stable, and the previous version keeps its file and record under the ID `:`, with `"upstream_status": "superseded"` and `"superseded_by"` naming the current ID. If a re-download fails, the previous record stays, marked `"upstream_status": "fetch_failed"`, next to its entry in `failures`; the next `--resume` retries it. -The script allows at most 4 concurrent requests, sets timeouts and limited retries (at most 3 attempts on connection errors, timeouts, HTTP 429, 500, 502, 503 and 504, and BWIKI's EdgeOne 567, honouring `Retry-After` up to 30 s; any other HTTP error fails at once), and saves the manifest after every 25 results; every save replaces the file atomically and still lists every previous record, so an interrupted run loses nothing. An image download is refused, with nothing written and an entry in `failures` like any other failed download, unless every URL it requests, each redirect target included, is https under the wiki's own image path (`static.wikia.nocookie.net/deemo/images/` or `patchwiki.biligame.com/images/deemo/`, the list in `scripts/check_sources.py` that the source check also applies to a proposed snapshot) and the response is at most 64 MiB. The other requests, the MediaWiki API queries, the DeemoSongs mapping and the illustrator index, likewise follow a redirect only while it stays on https on the same host and read at most 64 MiB; otherwise they fail with an error, which for the illustrator index is recorded in `failures`. No request reads the body of a redirect. Discovery follows the continuation of each MediaWiki list for at most 1,000 continuation requests after its first request, 1,001 requests in all; page contents and `imageinfo` are requested for 30 and 50 titles at a time, and each such batch counts as one list. If a list is still continued after those requests, or a continuation is not an object or would repeat an earlier request, discovery fails with an error naming the list and writes no snapshot, so a list cut short never passes for a complete one. `--metadata-only` downloads no images and rewrites only `wiki-discovery.json` (candidates and statistics) and `wiki-song-index.json`; it keeps an existing `wikis.json` and leaves `song-mapping.json` alone, because the DeemoSongs mapping is fetched only by a full run (without `--resume` or `--metadata-only`). +The script allows at most 4 concurrent requests, sets timeouts and limited retries (at most 3 attempts on connection errors, timeouts and HTTP 500, 502, 503 and 504, waiting 1 and 2 s between them; at most 6 attempts on the rate-limit statuses HTTP 429 and BWIKI's EdgeOne 567, waiting 5, 10, 20, 30 and 30 s; the server's `Retry-After` replaces any of these waits, capped at 30 s; any other HTTP error fails at once), and saves the manifest after every 25 results; every save replaces the file atomically and still lists every previous record, so an interrupted run loses nothing. An image download is refused, with nothing written and an entry in `failures` like any other failed download, unless every URL it requests, each redirect target included, is https under the wiki's own image path (`static.wikia.nocookie.net/deemo/images/` or `patchwiki.biligame.com/images/deemo/`, the list in `scripts/check_sources.py` that the source check also applies to a proposed snapshot) and the response is at most 64 MiB. The other requests, the MediaWiki API queries, the DeemoSongs mapping and the illustrator index, likewise follow a redirect only while it stays on https on the same host and read at most 64 MiB; otherwise they fail with an error, which for the illustrator index is recorded in `failures`. No request reads the body of a redirect. Discovery follows the continuation of each MediaWiki list for at most 1,000 continuation requests after its first request, 1,001 requests in all; page contents and `imageinfo` are requested for 30 and 50 titles at a time, and each such batch counts as one list. If a list is still continued after those requests, or a continuation is not an object or would repeat an earlier request, discovery fails with an error naming the list and writes no snapshot, so a list cut short never passes for a complete one. `--metadata-only` downloads no images and rewrites only `wiki-discovery.json` (candidates and statistics) and `wiki-song-index.json`; it keeps an existing `wikis.json` and leaves `song-mapping.json` alone, because the DeemoSongs mapping is fetched only by a full run (without `--resume` or `--metadata-only`). ## Access limits in this run diff --git a/docs/sources-wikis.zh-CN.md b/docs/sources-wikis.zh-CN.md index 6058f36..f2b304f 100644 --- a/docs/sources-wikis.zh-CN.md +++ b/docs/sources-wikis.zh-CN.md @@ -50,7 +50,7 @@ python -I scripts/fetch_wikis.py --resume --workers 4 `--resume` 不会从 `wikis.json` 删去任何记录，也不会删除文件。候选已不在快照中的记录会保留，并标记 `"upstream_status": "removed"`。重新上传的文件下载成功后，新版本沿用原资产 ID，深链接保持不变；旧版本连同文件与记录改用 `<资产 ID>:<其 SHA-256 前 12 位十六进制>` 作为 ID，标记 `"upstream_status": "superseded"`，并以 `"superseded_by"` 指向当前 ID。重新下载失败时保留旧记录并标记 `"upstream_status": "fetch_failed"`，同时在 `failures` 中记录；下次 `--resume` 会重试。 -脚本限制最多 4 个并发请求，设置超时和有限重试（连接错误、超时、HTTP 429、500、502、503、504 与 BWIKI 的 EdgeOne 567 最多尝试 3 次，遵循 `Retry-After`，上限 30 秒；其他 HTTP 错误立即失败），逐 25 个结果保存 manifest；每次保存都以原子方式替换文件，并仍列出全部已有记录，中断的运行不会丢失记录。图片下载请求的每个 URL（包括每次重定向的目标）都必须是 https 且位于该 Wiki 自己的图片路径下（`static.wikia.nocookie.net/deemo/images/` 或 `patchwiki.biligame.com/images/deemo/`，即 `scripts/check_sources.py` 中的列表，来源检查也按它审查提议的快照），且响应不超过 64 MiB；否则拒绝下载，不写入任何文件，并像其他失败的下载一样记入 `failures`。其他请求（MediaWiki API 查询、DeemoSongs 映射与画师目录）同样只在重定向仍为 https 且在同一主机上时才跟随，响应最多读取 64 MiB；否则报错失败，画师目录的失败会记入 `failures`。任何请求都不读取重定向响应的正文。发现阶段跟随每个 MediaWiki 列表的续查（`continue`），在首次请求之后最多再发出 1000 次续查请求，即共 1001 次请求；页面内容与 `imageinfo` 分别按每批 30 个与 50 个标题请求，每一批各算作一个列表。若这些请求之后列表仍有续查，或续查参数不是对象、或会重复先前的某次请求，发现阶段会报错并指明是哪个列表，且不写入快照，因此被截断的列表不会被当作完整列表。`--metadata-only` 不下载图片，只重写 `wiki-discovery.json`（候选与统计）与 `wiki-song-index.json`；已有的 `wikis.json` 保持不变，`song-mapping.json` 也不受影响，因为 DeemoSongs 映射只在完整运行（不带 `--resume` 或 `--metadata-only`）时抓取。 +脚本限制最多 4 个并发请求，设置超时和有限重试（连接错误、超时与 HTTP 500、502、503、504 最多尝试 3 次，间隔 1、2 秒；限流状态 HTTP 429 与 BWIKI 的 EdgeOne 567 最多尝试 6 次，间隔 5、10、20、30、30 秒；服务器给出 `Retry-After` 时改按它等待，上限 30 秒；其他 HTTP 错误立即失败），逐 25 个结果保存 manifest；每次保存都以原子方式替换文件，并仍列出全部已有记录，中断的运行不会丢失记录。图片下载请求的每个 URL（包括每次重定向的目标）都必须是 https 且位于该 Wiki 自己的图片路径下（`static.wikia.nocookie.net/deemo/images/` 或 `patchwiki.biligame.com/images/deemo/`，即 `scripts/check_sources.py` 中的列表，来源检查也按它审查提议的快照），且响应不超过 64 MiB；否则拒绝下载，不写入任何文件，并像其他失败的下载一样记入 `failures`。其他请求（MediaWiki API 查询、DeemoSongs 映射与画师目录）同样只在重定向仍为 https 且在同一主机上时才跟随，响应最多读取 64 MiB；否则报错失败，画师目录的失败会记入 `failures`。任何请求都不读取重定向响应的正文。发现阶段跟随每个 MediaWiki 列表的续查（`continue`），在首次请求之后最多再发出 1000 次续查请求，即共 1001 次请求；页面内容与 `imageinfo` 分别按每批 30 个与 50 个标题请求，每一批各算作一个列表。若这些请求之后列表仍有续查，或续查参数不是对象、或会重复先前的某次请求，发现阶段会报错并指明是哪个列表，且不写入快照，因此被截断的列表不会被当作完整列表。`--metadata-only` 不下载图片，只重写 `wiki-discovery.json`（候选与统计）与 `wiki-song-index.json`；已有的 `wikis.json` 保持不变，`song-mapping.json` 也不受影响，因为 DeemoSongs 映射只在完整运行（不带 `--resume` 或 `--metadata-only`）时抓取。 ## 本次访问限制 diff --git a/scripts/fetch_wikis.py b/scripts/fetch_wikis.py index b135b7f..cba9885 100644 --- a/scripts/fetch_wikis.py +++ b/scripts/fetch_wikis.py @@ -11,6 +11,7 @@ import argparse import hashlib import importlib.util +import itertools import json import math import os @@ -52,6 +53,11 @@ # Transient statuses worth another attempt; 567 is BWIKI's EdgeOne rate limiting. RETRY_STATUSES = {429, 500, 502, 503, 504, 567} ATTEMPTS = 3 +# Rate limiting outlasts a server hiccup, so these statuses get RATE_LIMIT_ATTEMPTS calls, and without a Retry-After +# the waits start at RATE_LIMIT_DELAY seconds and double up to MAX_RETRY_AFTER (5, 10, 20, 30 and 30 s). +RATE_LIMITED = {429, 567} +RATE_LIMIT_ATTEMPTS = 6 +RATE_LIMIT_DELAY = 5 MAX_RETRY_AFTER = 30 # seconds; a longer Retry-After is capped so a run stays bounded TIMEOUT = (10, 25) # seconds to connect, and to wait for each read # A request follows at most this many redirects, each checked like the first URL (see download_allowed() and get()). @@ -118,7 +124,8 @@ def now(): def retry_delay(error, attempt): """Seconds to wait before the next attempt: the server's Retry-After (seconds or an HTTP date), capped at - MAX_RETRY_AFTER, or else exponential backoff.""" + MAX_RETRY_AFTER, or else exponential backoff, from RATE_LIMIT_DELAY seconds for a RATE_LIMITED status and from + 1 second for any other error.""" response = getattr(error, "response", None) value = response.headers.get("Retry-After") if response is not None else None if value: @@ -131,19 +138,23 @@ def retry_delay(error, attempt): seconds = math.nan if not math.isnan(seconds): return min(max(seconds, 0.0), MAX_RETRY_AFTER) + if getattr(response, "status_code", None) in RATE_LIMITED: + return min(RATE_LIMIT_DELAY * 2**attempt, MAX_RETRY_AFTER) return min(2**attempt, 8) def retried(call, *args, **kwargs): """call(*args, **kwargs), called again after a connection error, a timeout or an HTTP error whose status is in - RETRY_STATUSES (ATTEMPTS calls in all, waiting retry_delay() in between); any other HTTP error is raised at once.""" - for attempt in range(ATTEMPTS): + RETRY_STATUSES, waiting retry_delay() in between: ATTEMPTS calls in all, or RATE_LIMIT_ATTEMPTS when the latest + error is a RATE_LIMITED status. Any other HTTP error is raised at once.""" + for attempt in itertools.count(): try: return call(*args, **kwargs) except requests.RequestException as error: - if isinstance(error, requests.HTTPError) and error.response.status_code not in RETRY_STATUSES: + status = error.response.status_code if isinstance(error, requests.HTTPError) else None + if status is not None and status not in RETRY_STATUSES: raise - if attempt == ATTEMPTS - 1: + if attempt + 1 >= (RATE_LIMIT_ATTEMPTS if status in RATE_LIMITED else ATTEMPTS): raise time.sleep(retry_delay(error, attempt)) diff --git a/tests/test_fetch_wikis.py b/tests/test_fetch_wikis.py index 2698683..4a1e5a2 100644 --- a/tests/test_fetch_wikis.py +++ b/tests/test_fetch_wikis.py @@ -836,10 +836,23 @@ def test_a_url_on_another_host_as_written_is_refused_though_requests_sends_it_to class RetryTests(FetcherRun): URL = "https://wiki.biligame.com/deemo/api.php" + def restart(self, route): + self.web.calls.clear() + self.sleeps.clear() + self.web.routes[self.URL] = route + def test_bwiki_edgeone_567_is_retried(self): self.web.routes[self.URL] = [(567, b"", {}), (200, b"{}", {})] self.assertEqual(fetch.get(self.URL).content, b"{}") - self.assertEqual((len(self.web.calls), self.sleeps), (2, [1])) + self.assertEqual((len(self.web.calls), self.sleeps), (2, [5])) + + def test_rate_limiting_is_waited_out_longer_than_a_server_error(self): + # The source check of 2026-10-19 stopped at a third 567 in a row from BWIKI, 1 s and 2 s after the first. + for status in (429, 567): + with self.subTest(status=status): + self.restart([(status, b"", {})] * 5 + [(200, b"{}", {})]) + self.assertEqual(fetch.get(self.URL).content, b"{}") + self.assertEqual((len(self.web.calls), self.sleeps), (6, [5, 10, 20, 30, 30])) def test_retry_after_is_honoured_up_to_a_cap(self): self.web.routes[self.URL] = [ @@ -857,16 +870,30 @@ def test_retry_after_may_be_an_http_date(self): self.assertTrue(5 <= self.sleeps[0] <= 20, self.sleeps) def test_attempts_are_bounded_and_client_errors_fail_at_once(self): - self.web.routes[self.URL] = (567, b"", {}) - with self.assertRaises(requests.HTTPError): - fetch.get(self.URL) - self.assertEqual((len(self.web.calls), len(self.sleeps)), (3, 2)) - self.web.calls.clear() - self.sleeps.clear() - self.web.routes[self.URL] = (404, b"", {}) + for route, error, sleeps in ( + ((503, b"", {}), requests.HTTPError, [1, 2]), + (None, requests.ConnectionError, [1, 2]), # FakeWeb answers a URL without a route as if offline + ((429, b"", {}), requests.HTTPError, [5, 10, 20, 30, 30]), + ((567, b"", {}), requests.HTTPError, [5, 10, 20, 30, 30]), + ((404, b"", {}), requests.HTTPError, []), + ): + with self.subTest(route=route): + self.restart(route) + with self.assertRaises(error): + fetch.get(self.URL) + self.assertEqual((len(self.web.calls), self.sleeps), (len(sleeps) + 1, sleeps)) + + def test_the_latest_error_decides_how_many_attempts_a_request_gets(self): + """Attempts count across statuses: a server error after three rate-limited ones ends the request (a fourth + attempt is past the three a server error gets), and rate limiting after two server errors still gets six.""" + self.web.limit = 10 + self.restart([(567, b"", {})] * 3 + [(503, b"", {})]) with self.assertRaises(requests.HTTPError): fetch.get(self.URL) - self.assertEqual((len(self.web.calls), self.sleeps), (1, [])) + self.assertEqual((len(self.web.calls), self.sleeps), (4, [5, 10, 20])) + self.restart([(503, b"", {})] * 2 + [(567, b"", {})] * 3 + [(200, b"{}", {})]) + self.assertEqual(fetch.get(self.URL).content, b"{}") + self.assertEqual((len(self.web.calls), self.sleeps), (6, [1, 2, 20, 30, 30])) class DownloadIntegrityTests(FetcherRun): @@ -1899,6 +1926,50 @@ def api(source, **params): with self.subTest(document=document, phrase=phrase): self.assertTrue(phrase in text, f"{document} does not say: {phrase}") + def test_the_retries_are_documented_as_they_are_made(self): + """docs/sources-wikis.md and its Chinese mirror state which errors are retried, how many attempts each gets + and the waits retry_delay() puts between them when the server sends no Retry-After.""" + + def waits(status, attempts): + response = requests.Response() + response.status_code = status + error = requests.HTTPError(response=response) + return [f"{fetch.retry_delay(error, attempt):g}" for attempt in range(attempts - 1)] + + def listed(items): + return ", ".join(items[:-1]) + " and " + items[-1] + + server = [str(status) for status in sorted(fetch.RETRY_STATUSES - fetch.RATE_LIMITED)] + self.assertEqual(sorted(fetch.RATE_LIMITED), [429, 567]) + server_waits, limit_waits = waits(503, fetch.ATTEMPTS), waits(567, fetch.RATE_LIMIT_ATTEMPTS) + cap = fetch.MAX_RETRY_AFTER + for document, phrases in ( + ( + "docs/sources-wikis.md", + ( + f"at most {fetch.ATTEMPTS} attempts on connection errors, timeouts and HTTP {listed(server)}, " + f"waiting {listed(server_waits)} s between them", + f"at most {fetch.RATE_LIMIT_ATTEMPTS} attempts on the rate-limit statuses HTTP 429 and BWIKI's " + f"EdgeOne 567, waiting {listed(limit_waits)} s", + f"the server's `Retry-After` replaces any of these waits, capped at {cap} s", + ), + ), + ( + "docs/sources-wikis.zh-CN.md", + ( + f"连接错误、超时与 HTTP {'、'.join(server)} 最多尝试 {fetch.ATTEMPTS} 次，" + f"间隔 {'、'.join(server_waits)} 秒", + f"限流状态 HTTP 429 与 BWIKI 的 EdgeOne 567 最多尝试 {fetch.RATE_LIMIT_ATTEMPTS} 次，" + f"间隔 {'、'.join(limit_waits)} 秒", + f"服务器给出 `Retry-After` 时改按它等待，上限 {cap} 秒", + ), + ), + ): + text = (REPO / document).read_text(encoding="utf-8") + for phrase in phrases: + with self.subTest(document=document, phrase=phrase): + self.assertTrue(phrase in text, f"{document} does not say: {phrase}") + class EncodingTests(unittest.TestCase): def test_resume_reads_utf8_manifests_under_an_ascii_locale(self): From 79eaabf81da5e2f2544cab4258e42ccbe54e5bdb Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 09:29:32 +0000 Subject: [PATCH 2/2] fix(wikis): count rate limiting apart from other errors; try no other URL after it Review of the previous commit found two faults in retried(): one attempt counter served both kinds of error, so a 503 after two 567s was not retried at all, and the rate-limit waits were indexed by the overall attempt, so 503, 503, 567 waited 20 s after the first 567 instead of the documented 5 s. Each kind now has its own counter, its own attempt limit and its own waits; a request is made at most ATTEMPTS + RATE_LIMIT_ATTEMPTS - 1 (8) times, and every failure advances one bounded counter, so the loop always ends. download() tried a Fandom candidate's next URL after any requests error, and each URL now has six rate-limited attempts: a rate-limited candidate would have cost about 3 x 95 s. Rate limiting now fails the candidate at once, as a RefusedDownload does; its URLs are all on the host that limits the rate. A server error still moves on to the next URL. http_status() reads the status for retried(), retry_delay() and download() alike. RetryTests caps the fake web at 20 requests so a broken bound fails instead of hanging; DocumentationTests shares one assertDocumented() helper. Both docs say the kinds are counted apart and that rate limiting ends an image download, and the doc test pins it. Undoing either fix fails the new tests. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AZypKz9iupFY3Dhsd8Q1rK --- docs/sources-wikis.md | 4 +- docs/sources-wikis.zh-CN.md | 4 +- scripts/fetch_wikis.py | 41 ++++++++----- tests/test_fetch_wikis.py | 115 ++++++++++++++++++++++-------------- 4 files changed, 102 insertions(+), 62 deletions(-) diff --git a/docs/sources-wikis.md b/docs/sources-wikis.md index 3cdf8f7..6641a30 100644 --- a/docs/sources-wikis.md +++ b/docs/sources-wikis.md @@ -28,7 +28,7 @@ The wiki song pages also include removed songs, port-exclusive songs, and images The script saves the raw bytes of the HTTP response; it does not re-encode, upscale, crop, color-correct, or remove backgrounds. Every image's actual format and dimensions are verified with Pillow, and its SHA-256, download time, real Content-Type, source page, request URL, final response URL, and wiki metadata are recorded. -Fandom CDN requests use the public `format=original` option to avoid default negotiation to WebP. The direct file URL is requested first; on a CDN backend error, the full revision URL provided by the API is tried next. If both original endpoints fail, the public full-size `format=png` is tried last; its checksum/size is still compared item by item against the original upload, and its `delivery_note` flags it as possibly re-encoded by the CDN only when they differ (otherwise the note says they match). The saved file's extension comes from the actual format verified by decoding; the URL extension is not blindly trusted. +Fandom CDN requests use the public `format=original` option to avoid default negotiation to WebP. The direct file URL is requested first; on a CDN backend error, the full revision URL provided by the API is tried next. If both original endpoints fail, the public full-size `format=png` is tried last; its checksum/size is still compared item by item against the original upload, and its `delivery_note` flags it as possibly re-encoded by the CDN only when they differ (otherwise the note says they match). Rate limiting (HTTP 429 or 567) instead fails the file at once, with no other URL tried: they are all on the same CDN, and the file is recorded in `failures` like any other failed download. The saved file's extension comes from the actual format verified by decoding; the URL extension is not blindly trusted. `wiki_original_size_matches` and `wiki_original_sha1_matches` indicate whether the downloaded file matches the original-upload metadata published by the wiki. A few CDN files may have the same dimensions but a different checksum/size; these records do not claim to be identical to the uploaded master file. Files of 1024×2048 or 2048×1024 may be atlases or contact sheets; they are kept as is and flagged, and are not cropped apart automatically. @@ -50,7 +50,7 @@ python -I scripts/fetch_wikis.py --resume --workers 4 `--resume` never drops a record from `wikis.json` and never deletes a file. A record whose candidate is no longer in the snapshot is kept with `"upstream_status": "removed"`. When a re-uploaded file is downloaded, the new version keeps the asset ID, so deep links stay stable, and the previous version keeps its file and record under the ID `:`, with `"upstream_status": "superseded"` and `"superseded_by"` naming the current ID. If a re-download fails, the previous record stays, marked `"upstream_status": "fetch_failed"`, next to its entry in `failures`; the next `--resume` retries it. -The script allows at most 4 concurrent requests, sets timeouts and limited retries (at most 3 attempts on connection errors, timeouts and HTTP 500, 502, 503 and 504, waiting 1 and 2 s between them; at most 6 attempts on the rate-limit statuses HTTP 429 and BWIKI's EdgeOne 567, waiting 5, 10, 20, 30 and 30 s; the server's `Retry-After` replaces any of these waits, capped at 30 s; any other HTTP error fails at once), and saves the manifest after every 25 results; every save replaces the file atomically and still lists every previous record, so an interrupted run loses nothing. An image download is refused, with nothing written and an entry in `failures` like any other failed download, unless every URL it requests, each redirect target included, is https under the wiki's own image path (`static.wikia.nocookie.net/deemo/images/` or `patchwiki.biligame.com/images/deemo/`, the list in `scripts/check_sources.py` that the source check also applies to a proposed snapshot) and the response is at most 64 MiB. The other requests, the MediaWiki API queries, the DeemoSongs mapping and the illustrator index, likewise follow a redirect only while it stays on https on the same host and read at most 64 MiB; otherwise they fail with an error, which for the illustrator index is recorded in `failures`. No request reads the body of a redirect. Discovery follows the continuation of each MediaWiki list for at most 1,000 continuation requests after its first request, 1,001 requests in all; page contents and `imageinfo` are requested for 30 and 50 titles at a time, and each such batch counts as one list. If a list is still continued after those requests, or a continuation is not an object or would repeat an earlier request, discovery fails with an error naming the list and writes no snapshot, so a list cut short never passes for a complete one. `--metadata-only` downloads no images and rewrites only `wiki-discovery.json` (candidates and statistics) and `wiki-song-index.json`; it keeps an existing `wikis.json` and leaves `song-mapping.json` alone, because the DeemoSongs mapping is fetched only by a full run (without `--resume` or `--metadata-only`). +The script allows at most 4 concurrent requests, sets timeouts and limited retries (at most 3 attempts on connection errors, timeouts and HTTP 500, 502, 503 and 504, waiting 1 and 2 s between them; at most 6 attempts on the rate-limit statuses HTTP 429 and BWIKI's EdgeOne 567, waiting 5, 10, 20, 30 and 30 s, counted apart from the other errors; the server's `Retry-After` replaces any of these waits, capped at 30 s; any other HTTP error fails at once), and saves the manifest after every 25 results; every save replaces the file atomically and still lists every previous record, so an interrupted run loses nothing. An image download is refused, with nothing written and an entry in `failures` like any other failed download, unless every URL it requests, each redirect target included, is https under the wiki's own image path (`static.wikia.nocookie.net/deemo/images/` or `patchwiki.biligame.com/images/deemo/`, the list in `scripts/check_sources.py` that the source check also applies to a proposed snapshot) and the response is at most 64 MiB. The other requests, the MediaWiki API queries, the DeemoSongs mapping and the illustrator index, likewise follow a redirect only while it stays on https on the same host and read at most 64 MiB; otherwise they fail with an error, which for the illustrator index is recorded in `failures`. No request reads the body of a redirect. Discovery follows the continuation of each MediaWiki list for at most 1,000 continuation requests after its first request, 1,001 requests in all; page contents and `imageinfo` are requested for 30 and 50 titles at a time, and each such batch counts as one list. If a list is still continued after those requests, or a continuation is not an object or would repeat an earlier request, discovery fails with an error naming the list and writes no snapshot, so a list cut short never passes for a complete one. `--metadata-only` downloads no images and rewrites only `wiki-discovery.json` (candidates and statistics) and `wiki-song-index.json`; it keeps an existing `wikis.json` and leaves `song-mapping.json` alone, because the DeemoSongs mapping is fetched only by a full run (without `--resume` or `--metadata-only`). ## Access limits in this run diff --git a/docs/sources-wikis.zh-CN.md b/docs/sources-wikis.zh-CN.md index f2b304f..aae0631 100644 --- a/docs/sources-wikis.zh-CN.md +++ b/docs/sources-wikis.zh-CN.md @@ -28,7 +28,7 @@ Wiki 歌页还包含已移除歌曲、端口独占曲与不同版本图片，因 脚本保存 HTTP 响应的原始 bytes，不重新编码、放大、裁切、调色或去背景。每张图片均用 Pillow 验证实际格式与尺寸，并记录 SHA-256、下载时间、真实 Content-Type、来源页、请求 URL、最终响应 URL 和 Wiki 元数据。 -Fandom CDN 请求使用公开的 `format=original`，避免默认协商成 WebP；先请求直接文件 URL，遇到 CDN 后端错误再尝试 API 提供的完整 revision URL。两个 original 端点都失败时，最后尝试公开的全尺寸 `format=png`，仍逐项比较原上传 checksum/size；只有两者不一致时，`delivery_note` 才标记可能经过 CDN 重编码（一致时注明相符）。保存文件扩展名来自解码验证的实际格式，不盲信 URL 扩展名。 +Fandom CDN 请求使用公开的 `format=original`，避免默认协商成 WebP；先请求直接文件 URL，遇到 CDN 后端错误再尝试 API 提供的完整 revision URL。两个 original 端点都失败时，最后尝试公开的全尺寸 `format=png`，仍逐项比较原上传 checksum/size；只有两者不一致时，`delivery_note` 才标记可能经过 CDN 重编码（一致时注明相符）。遇到限流（HTTP 429 或 567）则立即判该文件失败，不再尝试其他 URL：它们都在同一 CDN 上；该文件像其他下载失败一样记入 `failures`。保存文件扩展名来自解码验证的实际格式，不盲信 URL 扩展名。 `wiki_original_size_matches` 与 `wiki_original_sha1_matches` 表示所下载文件是否和 Wiki 发布的原上传元数据一致。少数 CDN 文件可能尺寸相同但 checksum/size 不同；这些记录不宣称与上传母文件相同。1024×2048 或 2048×1024 文件可能是 atlas 或拼图，原样保留并标记，不自动裁开。 @@ -50,7 +50,7 @@ python -I scripts/fetch_wikis.py --resume --workers 4 `--resume` 不会从 `wikis.json` 删去任何记录，也不会删除文件。候选已不在快照中的记录会保留，并标记 `"upstream_status": "removed"`。重新上传的文件下载成功后，新版本沿用原资产 ID，深链接保持不变；旧版本连同文件与记录改用 `<资产 ID>:<其 SHA-256 前 12 位十六进制>` 作为 ID，标记 `"upstream_status": "superseded"`，并以 `"superseded_by"` 指向当前 ID。重新下载失败时保留旧记录并标记 `"upstream_status": "fetch_failed"`，同时在 `failures` 中记录；下次 `--resume` 会重试。 -脚本限制最多 4 个并发请求，设置超时和有限重试（连接错误、超时与 HTTP 500、502、503、504 最多尝试 3 次，间隔 1、2 秒；限流状态 HTTP 429 与 BWIKI 的 EdgeOne 567 最多尝试 6 次，间隔 5、10、20、30、30 秒；服务器给出 `Retry-After` 时改按它等待，上限 30 秒；其他 HTTP 错误立即失败），逐 25 个结果保存 manifest；每次保存都以原子方式替换文件，并仍列出全部已有记录，中断的运行不会丢失记录。图片下载请求的每个 URL（包括每次重定向的目标）都必须是 https 且位于该 Wiki 自己的图片路径下（`static.wikia.nocookie.net/deemo/images/` 或 `patchwiki.biligame.com/images/deemo/`，即 `scripts/check_sources.py` 中的列表，来源检查也按它审查提议的快照），且响应不超过 64 MiB；否则拒绝下载，不写入任何文件，并像其他失败的下载一样记入 `failures`。其他请求（MediaWiki API 查询、DeemoSongs 映射与画师目录）同样只在重定向仍为 https 且在同一主机上时才跟随，响应最多读取 64 MiB；否则报错失败，画师目录的失败会记入 `failures`。任何请求都不读取重定向响应的正文。发现阶段跟随每个 MediaWiki 列表的续查（`continue`），在首次请求之后最多再发出 1000 次续查请求，即共 1001 次请求；页面内容与 `imageinfo` 分别按每批 30 个与 50 个标题请求，每一批各算作一个列表。若这些请求之后列表仍有续查，或续查参数不是对象、或会重复先前的某次请求，发现阶段会报错并指明是哪个列表，且不写入快照，因此被截断的列表不会被当作完整列表。`--metadata-only` 不下载图片，只重写 `wiki-discovery.json`（候选与统计）与 `wiki-song-index.json`；已有的 `wikis.json` 保持不变，`song-mapping.json` 也不受影响，因为 DeemoSongs 映射只在完整运行（不带 `--resume` 或 `--metadata-only`）时抓取。 +脚本限制最多 4 个并发请求，设置超时和有限重试（连接错误、超时与 HTTP 500、502、503、504 最多尝试 3 次，间隔 1、2 秒；限流状态 HTTP 429 与 BWIKI 的 EdgeOne 567 最多尝试 6 次，间隔 5、10、20、30、30 秒，与其他错误分开计数；服务器给出 `Retry-After` 时改按它等待，上限 30 秒；其他 HTTP 错误立即失败），逐 25 个结果保存 manifest；每次保存都以原子方式替换文件，并仍列出全部已有记录，中断的运行不会丢失记录。图片下载请求的每个 URL（包括每次重定向的目标）都必须是 https 且位于该 Wiki 自己的图片路径下（`static.wikia.nocookie.net/deemo/images/` 或 `patchwiki.biligame.com/images/deemo/`，即 `scripts/check_sources.py` 中的列表，来源检查也按它审查提议的快照），且响应不超过 64 MiB；否则拒绝下载，不写入任何文件，并像其他失败的下载一样记入 `failures`。其他请求（MediaWiki API 查询、DeemoSongs 映射与画师目录）同样只在重定向仍为 https 且在同一主机上时才跟随，响应最多读取 64 MiB；否则报错失败，画师目录的失败会记入 `failures`。任何请求都不读取重定向响应的正文。发现阶段跟随每个 MediaWiki 列表的续查（`continue`），在首次请求之后最多再发出 1000 次续查请求，即共 1001 次请求；页面内容与 `imageinfo` 分别按每批 30 个与 50 个标题请求，每一批各算作一个列表。若这些请求之后列表仍有续查，或续查参数不是对象、或会重复先前的某次请求，发现阶段会报错并指明是哪个列表，且不写入快照，因此被截断的列表不会被当作完整列表。`--metadata-only` 不下载图片，只重写 `wiki-discovery.json`（候选与统计）与 `wiki-song-index.json`；已有的 `wikis.json` 保持不变，`song-mapping.json` 也不受影响，因为 DeemoSongs 映射只在完整运行（不带 `--resume` 或 `--metadata-only`）时抓取。 ## 本次访问限制 diff --git a/scripts/fetch_wikis.py b/scripts/fetch_wikis.py index cba9885..6788e60 100644 --- a/scripts/fetch_wikis.py +++ b/scripts/fetch_wikis.py @@ -11,7 +11,6 @@ import argparse import hashlib import importlib.util -import itertools import json import math import os @@ -53,8 +52,9 @@ # Transient statuses worth another attempt; 567 is BWIKI's EdgeOne rate limiting. RETRY_STATUSES = {429, 500, 502, 503, 504, 567} ATTEMPTS = 3 -# Rate limiting outlasts a server hiccup, so these statuses get RATE_LIMIT_ATTEMPTS calls, and without a Retry-After -# the waits start at RATE_LIMIT_DELAY seconds and double up to MAX_RETRY_AFTER (5, 10, 20, 30 and 30 s). +# Rate limiting outlasts a server hiccup, so these statuses get RATE_LIMIT_ATTEMPTS attempts, counted apart from the +# other errors (see retried()), and without a Retry-After the waits start at RATE_LIMIT_DELAY seconds and double up to +# MAX_RETRY_AFTER (5, 10, 20, 30 and 30 s). RATE_LIMITED = {429, 567} RATE_LIMIT_ATTEMPTS = 6 RATE_LIMIT_DELAY = 5 @@ -122,10 +122,15 @@ def now(): return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") +def http_status(error): + """The status of an HTTPError's response, or None for any other requests error.""" + return error.response.status_code if isinstance(error, requests.HTTPError) else None + + def retry_delay(error, attempt): - """Seconds to wait before the next attempt: the server's Retry-After (seconds or an HTTP date), capped at - MAX_RETRY_AFTER, or else exponential backoff, from RATE_LIMIT_DELAY seconds for a RATE_LIMITED status and from - 1 second for any other error.""" + """Seconds to wait before the next attempt, where attempt counts (from 0) the earlier failures of error's kind (see + retried()): the server's Retry-After (seconds or an HTTP date), capped at MAX_RETRY_AFTER, or else exponential + backoff, from RATE_LIMIT_DELAY seconds for a RATE_LIMITED status and from 1 second for any other error.""" response = getattr(error, "response", None) value = response.headers.get("Retry-After") if response is not None else None if value: @@ -138,25 +143,30 @@ def retry_delay(error, attempt): seconds = math.nan if not math.isnan(seconds): return min(max(seconds, 0.0), MAX_RETRY_AFTER) - if getattr(response, "status_code", None) in RATE_LIMITED: + if http_status(error) in RATE_LIMITED: return min(RATE_LIMIT_DELAY * 2**attempt, MAX_RETRY_AFTER) return min(2**attempt, 8) def retried(call, *args, **kwargs): """call(*args, **kwargs), called again after a connection error, a timeout or an HTTP error whose status is in - RETRY_STATUSES, waiting retry_delay() in between: ATTEMPTS calls in all, or RATE_LIMIT_ATTEMPTS when the latest - error is a RATE_LIMITED status. Any other HTTP error is raised at once.""" - for attempt in itertools.count(): + RETRY_STATUSES, waiting retry_delay() in between; any other HTTP error is raised at once. Rate limiting (a + RATE_LIMITED status) and the other retried errors are counted apart: the error that is the RATE_LIMIT_ATTEMPTS-th + of the one kind or the ATTEMPTS-th of the other is raised, so call is made at most + ATTEMPTS + RATE_LIMIT_ATTEMPTS - 1 times.""" + failures = Counter() # the failed calls so far, keyed by whether they were rate limited + while True: try: return call(*args, **kwargs) except requests.RequestException as error: - status = error.response.status_code if isinstance(error, requests.HTTPError) else None + status = http_status(error) if status is not None and status not in RETRY_STATUSES: raise - if attempt + 1 >= (RATE_LIMIT_ATTEMPTS if status in RATE_LIMITED else ATTEMPTS): + limited = status in RATE_LIMITED + if failures[limited] + 1 >= (RATE_LIMIT_ATTEMPTS if limited else ATTEMPTS): raise - time.sleep(retry_delay(error, attempt)) + time.sleep(retry_delay(error, failures[limited])) + failures[limited] += 1 def on_host(url, host): @@ -798,13 +808,16 @@ def download(candidate, existing): record["delivery_note"] = note return record attempted_failures = [] - # A RefusedDownload is not a network error: it fails the candidate at once, with no other URL tried. + # A RefusedDownload is not a network error: it fails the candidate at once, with no other URL tried. Nor is another + # URL tried after rate limiting: every URL of a candidate is on one host, the one that limits the rate. for url in download_urls(candidate): try: response, payload = get_image(source, url, {"Referer": quote(SOURCES[source]["url"], safe=":/")}) break except requests.RequestException as error: attempted_failures.append({"url": url, "error": str(error)}) + if http_status(error) in RATE_LIMITED: + raise RuntimeError(json.dumps(attempted_failures)) from None else: raise RuntimeError(json.dumps(attempted_failures)) with Image.open(BytesIO(payload)) as image: diff --git a/tests/test_fetch_wikis.py b/tests/test_fetch_wikis.py index 4a1e5a2..b1c7f1f 100644 --- a/tests/test_fetch_wikis.py +++ b/tests/test_fetch_wikis.py @@ -836,6 +836,10 @@ def test_a_url_on_another_host_as_written_is_refused_though_requests_sends_it_to class RetryTests(FetcherRun): URL = "https://wiki.biligame.com/deemo/api.php" + def setUp(self): + super().setUp() + self.web.limit = 20 # retried() loops until it returns or raises; a broken bound fails here instead of hanging + def restart(self, route): self.web.calls.clear() self.sleeps.clear() @@ -883,17 +887,26 @@ def test_attempts_are_bounded_and_client_errors_fail_at_once(self): fetch.get(self.URL) self.assertEqual((len(self.web.calls), self.sleeps), (len(sleeps) + 1, sleeps)) - def test_the_latest_error_decides_how_many_attempts_a_request_gets(self): - """Attempts count across statuses: a server error after three rate-limited ones ends the request (a fourth - attempt is past the three a server error gets), and rate limiting after two server errors still gets six.""" - self.web.limit = 10 - self.restart([(567, b"", {})] * 3 + [(503, b"", {})]) - with self.assertRaises(requests.HTTPError): - fetch.get(self.URL) - self.assertEqual((len(self.web.calls), self.sleeps), (4, [5, 10, 20])) - self.restart([(503, b"", {})] * 2 + [(567, b"", {})] * 3 + [(200, b"{}", {})]) - self.assertEqual(fetch.get(self.URL).content, b"{}") - self.assertEqual((len(self.web.calls), self.sleeps), (6, [1, 2, 20, 30, 30])) + def test_rate_limiting_and_other_errors_are_counted_apart(self): + """Each kind of failure has its own attempts and its own waits, whatever came before it, so a request is made + at most ATTEMPTS + RATE_LIMIT_ATTEMPTS - 1 times.""" + limited, server = (567, b"", {}), (503, b"", {}) + for route, sleeps in ( + ([limited, limited, server, (200, b"{}", {})], [5, 10, 1]), + ([server, server, limited, limited, limited, (200, b"{}", {})], [1, 2, 5, 10, 20]), + ([limited] * 3 + [server], [5, 10, 20, 1, 2]), # the third 503 in a row ends it + ([server] * 2 + [limited] * 5 + [server], [1, 2, 5, 10, 20, 30, 30]), + ([server] * 2 + [limited], [1, 2, 5, 10, 20, 30, 30]), # the sixth 567 ends it + ): + with self.subTest(route=route): + self.restart(list(route)) # FakeWeb pops the responses it serves; the subtest keeps the whole route + if route[-1][0] == 200: + self.assertEqual(fetch.get(self.URL).content, b"{}") + else: + with self.assertRaises(requests.HTTPError): + fetch.get(self.URL) + self.assertEqual((len(self.web.calls), self.sleeps), (len(sleeps) + 1, sleeps)) + self.assertEqual(len(self.web.calls), fetch.ATTEMPTS + fetch.RATE_LIMIT_ATTEMPTS - 1) class DownloadIntegrityTests(FetcherRun): @@ -1219,6 +1232,25 @@ def test_a_refused_download_is_not_retried_and_tries_no_other_url(self): self.assertEqual(self.sleeps, []) self.assertTrue(failure["error"].startswith("Refusing"), failure["error"]) + def test_rate_limiting_fails_the_candidate_without_trying_its_other_urls(self): + # Every URL of a candidate is on the host that limits the rate. A server error moves on to the next URL. + payload = png("blue") + candidate, (first, second, _) = self.fandom(payload) + manifest = (self.root / MANIFEST).read_bytes() + for status in (429, 567, 503): + with self.subTest(status=status): + (self.root / MANIFEST).write_bytes(manifest) # no earlier failure, so the URLs keep their order + shutil.rmtree(self.root / "assets", ignore_errors=True) + self.serve({first: (status, b"", {}), second: payload}) + self.sleeps.clear() + if status == 503: + self.assertEqual(len(self.resume(candidate)["assets"]), 1) + self.assertEqual(self.web.calls, [first] * fetch.ATTEMPTS + [second]) + else: + self.refused(candidate, reason=f"{status} Error") + self.assertEqual(self.web.calls, [first] * fetch.RATE_LIMIT_ATTEMPTS) + self.assertEqual(self.sleeps, [5, 10, 20, 30, 30]) + def test_downloads_and_the_source_check_share_one_list_of_image_paths(self): self.assertEqual(Path(fetch.check_sources.__file__), REPO / "scripts/check_sources.py") url = BWIKI_IMAGES + "a/ab/Art.png" @@ -1888,6 +1920,14 @@ def test_discovery_writes_no_snapshot_from_an_incomplete_list(self): class DocumentationTests(unittest.TestCase): + def assertDocumented(self, documents): + """Each document (a path under the repository) says each of its phrases word for word.""" + for document, phrases in documents.items(): + text = (REPO / document).read_text(encoding="utf-8") + for phrase in phrases: + with self.subTest(document=document, phrase=phrase): + self.assertTrue(phrase in text, f"{document} does not say: {phrase}") + def test_the_continuation_limit_is_documented_as_it_is_applied(self): """docs/sources-wikis.md and its Chinese mirror state the limit continued() applies: MAX_CONTINUATIONS continuation requests after a list's first request, one more in all, where pages() and imageinfo() continue @@ -1904,31 +1944,24 @@ def api(source, **params): fetch.imageinfo("fandom", titles) self.assertEqual(batches, {"pages": [30, 30, 30, 11], "imageinfo": [50, 50, 1]}) limit = fetch.MAX_CONTINUATIONS - for document, phrases in ( - ( - "docs/sources-wikis.md", - ( + self.assertDocumented( + { + "docs/sources-wikis.md": ( f"for at most {limit:,} continuation requests after its first request, {limit + 1:,} requests in all", "page contents and `imageinfo` are requested for 30 and 50 titles at a time, and each such batch " "counts as one list", ), - ), - ( - "docs/sources-wikis.zh-CN.md", - ( + "docs/sources-wikis.zh-CN.md": ( f"在首次请求之后最多再发出 {limit} 次续查请求，即共 {limit + 1} 次请求", "页面内容与 `imageinfo` 分别按每批 30 个与 50 个标题请求，每一批各算作一个列表", ), - ), - ): - text = (REPO / document).read_text(encoding="utf-8") - for phrase in phrases: - with self.subTest(document=document, phrase=phrase): - self.assertTrue(phrase in text, f"{document} does not say: {phrase}") + } + ) def test_the_retries_are_documented_as_they_are_made(self): - """docs/sources-wikis.md and its Chinese mirror state which errors are retried, how many attempts each gets - and the waits retry_delay() puts between them when the server sends no Retry-After.""" + """docs/sources-wikis.md and its Chinese mirror state which errors are retried, how many attempts each kind + gets, the waits retry_delay() puts between them when the server sends no Retry-After, and that download() tries + no other URL after rate limiting.""" def waits(status, attempts): response = requests.Response() @@ -1937,38 +1970,32 @@ def waits(status, attempts): return [f"{fetch.retry_delay(error, attempt):g}" for attempt in range(attempts - 1)] def listed(items): - return ", ".join(items[:-1]) + " and " + items[-1] + return items[-1] if len(items) == 1 else ", ".join(items[:-1]) + " and " + items[-1] server = [str(status) for status in sorted(fetch.RETRY_STATUSES - fetch.RATE_LIMITED)] self.assertEqual(sorted(fetch.RATE_LIMITED), [429, 567]) server_waits, limit_waits = waits(503, fetch.ATTEMPTS), waits(567, fetch.RATE_LIMIT_ATTEMPTS) cap = fetch.MAX_RETRY_AFTER - for document, phrases in ( - ( - "docs/sources-wikis.md", - ( + self.assertDocumented( + { + "docs/sources-wikis.md": ( f"at most {fetch.ATTEMPTS} attempts on connection errors, timeouts and HTTP {listed(server)}, " f"waiting {listed(server_waits)} s between them", f"at most {fetch.RATE_LIMIT_ATTEMPTS} attempts on the rate-limit statuses HTTP 429 and BWIKI's " - f"EdgeOne 567, waiting {listed(limit_waits)} s", + f"EdgeOne 567, waiting {listed(limit_waits)} s, counted apart from the other errors", f"the server's `Retry-After` replaces any of these waits, capped at {cap} s", + "Rate limiting (HTTP 429 or 567) instead fails the file at once, with no other URL tried", ), - ), - ( - "docs/sources-wikis.zh-CN.md", - ( + "docs/sources-wikis.zh-CN.md": ( f"连接错误、超时与 HTTP {'、'.join(server)} 最多尝试 {fetch.ATTEMPTS} 次，" f"间隔 {'、'.join(server_waits)} 秒", f"限流状态 HTTP 429 与 BWIKI 的 EdgeOne 567 最多尝试 {fetch.RATE_LIMIT_ATTEMPTS} 次，" - f"间隔 {'、'.join(limit_waits)} 秒", + f"间隔 {'、'.join(limit_waits)} 秒，与其他错误分开计数", f"服务器给出 `Retry-After` 时改按它等待，上限 {cap} 秒", + "遇到限流（HTTP 429 或 567）则立即判该文件失败，不再尝试其他 URL", ), - ), - ): - text = (REPO / document).read_text(encoding="utf-8") - for phrase in phrases: - with self.subTest(document=document, phrase=phrase): - self.assertTrue(phrase in text, f"{document} does not say: {phrase}") + } + ) class EncodingTests(unittest.TestCase):