From 2cc465471c3eb747f6e5aa6b98946e7ce117206f Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 1 Oct 2026 17:29:04 -0500 Subject: [PATCH 1/4] feat: Add support for an SQLite cache as a low-memory option --- .gitignore | 1 + bun.lock | 3 ++ package.json | 1 + src/dependencies.ts | 10 ++++- src/services/cache.ts | 7 ++- src/services/chrome-web-store.ts | 2 +- src/services/edge-addon-store.ts | 2 +- src/services/extension-store.ts | 13 ++---- src/services/firefox-addon-store.ts | 2 +- src/services/in-memory-cache.ts | 32 ++++++++------ src/services/redis-cache.ts | 23 +++++----- src/services/sqlite-cache.ts | 67 +++++++++++++++++++++++++++++ src/utils/cache.ts | 31 +++++++++++++ 13 files changed, 153 insertions(+), 41 deletions(-) create mode 100644 src/services/sqlite-cache.ts create mode 100644 src/utils/cache.ts diff --git a/.gitignore b/.gitignore index 02f56f5..33ca9b6 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ tsconfig.tsbuildinfo # Other Files src/crawlers/__tests__/fixtures/chrome-web-store/.new/* !src/crawlers/__tests__/fixtures/chrome-web-store/.new/.keep +/data diff --git a/bun.lock b/bun.lock index 9c35f2f..b07bf08 100644 --- a/bun.lock +++ b/bun.lock @@ -7,6 +7,7 @@ "dependencies": { "@aklinker1/logger": "^1.0.6", "@aklinker1/zero-ioc": "^1.5.1", + "@aklinker1/zero-serialize-error": "^1.0.2", "@aklinker1/zeta": "^2.2.0", "dataloader": "^2.2.3", "dedent": "^1.7.2", @@ -34,6 +35,8 @@ "@aklinker1/zero-ioc": ["@aklinker1/zero-ioc@1.5.1", "", {}, "sha512-PIv6SlAJFIfBA7nrUxBswsMsUHSjHK0DYMqSxj9pFItst6lwJYTP9XfTo7lY1f/swwzauieRG7YopAjd1rvQfw=="], + "@aklinker1/zero-serialize-error": ["@aklinker1/zero-serialize-error@1.0.2", "", {}, "sha512-hGVkgFqb8Zs80JJ7MOSCGQXH2SyWfwnJwOl8Qpp1lUdkp5T7GTf8QrAMib42Hnzx6HtnNYdmNaSSg4MQy+MDRQ=="], + "@aklinker1/zeta": ["@aklinker1/zeta@2.2.0", "", { "dependencies": { "@standard-schema/spec": "^1.0.0", "openapi-types": "^12.1.3", "rou3": "^0.7.12", "scule": "^1.3.0" }, "peerDependencies": { "@types/bun": "*", "@types/deno": "*" }, "optionalPeers": ["@types/bun", "@types/deno"] }, "sha512-xTCBSNnP5ZhBTrB2wBl1tsX68zFczzucvTXZd4+rvFCWHNjPTvx1GzMP99eo6XcB3EjeijJxXsri1ungH8Q75A=="], "@cspell/cspell-bundled-dicts": ["@cspell/cspell-bundled-dicts@10.0.0", "", { "dependencies": { "@cspell/dict-ada": "^4.1.1", "@cspell/dict-al": "^1.1.1", "@cspell/dict-aws": "^4.0.17", "@cspell/dict-bash": "^4.2.2", "@cspell/dict-companies": "^3.2.11", "@cspell/dict-cpp": "^7.0.2", "@cspell/dict-cryptocurrencies": "^5.0.5", "@cspell/dict-csharp": "^4.0.8", "@cspell/dict-css": "^4.1.1", "@cspell/dict-dart": "^2.3.2", "@cspell/dict-data-science": "^2.0.13", "@cspell/dict-django": "^4.1.6", "@cspell/dict-docker": "^1.1.17", "@cspell/dict-dotnet": "^5.0.13", "@cspell/dict-elixir": "^4.0.8", "@cspell/dict-en-common-misspellings": "^2.1.12", "@cspell/dict-en-gb-mit": "^3.1.22", "@cspell/dict-en_us": "^4.4.33", "@cspell/dict-filetypes": "^3.0.18", "@cspell/dict-flutter": "^1.1.1", "@cspell/dict-fonts": "^4.0.6", "@cspell/dict-fsharp": "^1.1.1", "@cspell/dict-fullstack": "^3.2.9", "@cspell/dict-gaming-terms": "^1.1.2", "@cspell/dict-git": "^3.1.0", "@cspell/dict-golang": "^6.0.26", "@cspell/dict-google": "^1.0.9", "@cspell/dict-haskell": "^4.0.6", "@cspell/dict-html": "^4.0.15", "@cspell/dict-html-symbol-entities": "^4.0.5", "@cspell/dict-java": "^5.0.12", "@cspell/dict-julia": "^1.1.1", "@cspell/dict-k8s": "^1.0.12", "@cspell/dict-kotlin": "^1.1.1", "@cspell/dict-latex": "^5.1.0", "@cspell/dict-lorem-ipsum": "^4.0.5", "@cspell/dict-lua": "^4.0.8", "@cspell/dict-makefile": "^1.0.5", "@cspell/dict-markdown": "^2.0.16", "@cspell/dict-monkeyc": "^1.0.12", "@cspell/dict-node": "^5.0.9", "@cspell/dict-npm": "^5.2.38", "@cspell/dict-php": "^4.1.1", "@cspell/dict-powershell": "^5.0.15", "@cspell/dict-public-licenses": "^2.0.16", "@cspell/dict-python": "^4.2.26", "@cspell/dict-r": "^2.1.1", "@cspell/dict-ruby": "^5.1.1", "@cspell/dict-rust": "^4.1.2", "@cspell/dict-scala": "^5.0.9", "@cspell/dict-shell": "^1.1.2", "@cspell/dict-software-terms": "^5.2.2", "@cspell/dict-sql": "^2.2.1", "@cspell/dict-svelte": "^1.0.7", "@cspell/dict-swift": "^2.0.6", "@cspell/dict-terraform": "^1.1.3", "@cspell/dict-typescript": "^3.2.3", "@cspell/dict-vue": "^3.0.5", "@cspell/dict-zig": "^1.0.0" } }, "sha512-ci410HEkng2582oOjlRHQtlGXwh+rUC/mVcN9dObLHpKhvPgzn2S6vT56pARstxxZpcCUG/oLhn3dCqdJlVzmA=="], diff --git a/package.json b/package.json index e31a7f3..97abd06 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "dependencies": { "@aklinker1/logger": "^1.0.6", "@aklinker1/zero-ioc": "^1.5.1", + "@aklinker1/zero-serialize-error": "^1.0.2", "@aklinker1/zeta": "^2.2.0", "dataloader": "^2.2.3", "dedent": "^1.7.2", diff --git a/src/dependencies.ts b/src/dependencies.ts index daffaf3..d681ffc 100644 --- a/src/dependencies.ts +++ b/src/dependencies.ts @@ -9,9 +9,17 @@ import { createFirefoxAddonStore } from "./services/firefox-addon-store"; import { createFirefoxApi } from "./services/firefox-api"; import { createInMemoryCache } from "./services/in-memory-cache"; import { createRedisCache } from "./services/redis-cache"; +import { createSqliteCache } from "./services/sqlite-cache"; export const container = createIocContainer() - .register("cache", Bun.redis.connected ? createRedisCache : createInMemoryCache) + .register( + "cache", + process.env.SQLITE_CACHE === "true" + ? createSqliteCache + : Bun.redis.connected + ? createRedisCache + : createInMemoryCache, + ) .register("edgeApi", createEdgeApi) .register("firefoxApi", createFirefoxApi) .register("chromeWebStore", transient(createChromeWebStore)) diff --git a/src/services/cache.ts b/src/services/cache.ts index 1ead5aa..9dcf336 100644 --- a/src/services/cache.ts +++ b/src/services/cache.ts @@ -1,4 +1,7 @@ export interface Cache { - get(key: string): Promise; - set(key: string, value: T): Promise; + with(key: string, fn: () => Promise): Promise; } + +export type CacheSuccess = { success: true; data: T }; +export type CacheFailure = { success: false; error: unknown }; +export type CacheResult = CacheSuccess | CacheFailure; diff --git a/src/services/chrome-web-store.ts b/src/services/chrome-web-store.ts index 47045ab..7a6a03c 100644 --- a/src/services/chrome-web-store.ts +++ b/src/services/chrome-web-store.ts @@ -7,7 +7,7 @@ export type ChromeWebStore = ExtensionStore; export function createChromeWebStore({ cache }: { cache: Cache }): ChromeWebStore { return new ExtensionStore({ fetch: (id) => crawlExtension(String(id), "en"), - cacheKeyPrefix: "chrome-extension-", + cacheKeyPrefix: "chrome-extension-v2-", cache, }); } diff --git a/src/services/edge-addon-store.ts b/src/services/edge-addon-store.ts index 4cf985e..9440a52 100644 --- a/src/services/edge-addon-store.ts +++ b/src/services/edge-addon-store.ts @@ -13,7 +13,7 @@ export function createEdgeAddonStore({ }): EdgeAddonStore { return new ExtensionStore({ fetch: (id) => edgeApi.getAddon(String(id)), - cacheKeyPrefix: "edge-addon-", + cacheKeyPrefix: "edge-addon-v2-", cache, }); } diff --git a/src/services/extension-store.ts b/src/services/extension-store.ts index be033ce..f84a168 100644 --- a/src/services/extension-store.ts +++ b/src/services/extension-store.ts @@ -17,16 +17,9 @@ export class ExtensionStore { this.dataloader = new DataLoader( async (ids): Promise> => { const results = await Promise.allSettled( - ids.map(async (id) => { - const cacheKey = options.cacheKeyPrefix + id; - const cached = await options.cache.get(cacheKey); - if (cached) return cached; - - const result = await options.fetch(id); - if (result) await options.cache.set(cacheKey, result); - - return result; - }), + ids.map(async (id) => + options.cache.with(options.cacheKeyPrefix + id, () => options.fetch(id)), + ), ); return results.map((res) => (res.status === "fulfilled" ? res.value : res.reason)); }, diff --git a/src/services/firefox-addon-store.ts b/src/services/firefox-addon-store.ts index 4046c95..1df9be0 100644 --- a/src/services/firefox-addon-store.ts +++ b/src/services/firefox-addon-store.ts @@ -13,7 +13,7 @@ export function createFirefoxAddonStore({ }): FirefoxAddonStore { return new ExtensionStore({ fetch: (id) => firefoxApi.getAddon(String(id)), - cacheKeyPrefix: "firefox-addon-", + cacheKeyPrefix: "firefox-addon-v2-", cache, }); } diff --git a/src/services/in-memory-cache.ts b/src/services/in-memory-cache.ts index c69b323..dcdccac 100644 --- a/src/services/in-memory-cache.ts +++ b/src/services/in-memory-cache.ts @@ -1,29 +1,33 @@ import { createLogger } from "@aklinker1/logger"; +import { buildCacheWith } from "../utils/cache"; import { HOUR_MS } from "../utils/time"; -import type { Cache } from "./cache"; +import type { Cache, CacheResult } from "./cache"; -const logger = createLogger("redis"); +const logger = createLogger("in-memory-cache"); const TTL = HOUR_MS; export function createInMemoryCache(): Cache { logger.info("Using in-memory cache"); - let cache: Record = Object.create(null); + let cache: Record> = Object.create(null); let ttl: Record = Object.create(null); return { - get: async (key: string) => { - if (ttl[key] && Date.now() > ttl[key]) { - delete cache[key]; - delete ttl[key]; - } - return cache[key]; - }, - set: async (key: string, value: any) => { - cache[key] = value; - ttl[key] = Date.now() + TTL; - }, + with: buildCacheWith( + logger, + async (key) => { + if (ttl[key] && Date.now() > ttl[key]) { + delete cache[key]; + delete ttl[key]; + } + return cache[key]; + }, + async (key, res) => { + cache[key] = res; + ttl[key] = Date.now() + TTL; + }, + ), }; } diff --git a/src/services/redis-cache.ts b/src/services/redis-cache.ts index 7a719de..6046e62 100644 --- a/src/services/redis-cache.ts +++ b/src/services/redis-cache.ts @@ -1,9 +1,10 @@ import { createLogger } from "@aklinker1/logger"; +import { buildCacheWith } from "../utils/cache"; import { DAY_MS } from "../utils/time"; import type { Cache } from "./cache"; -const logger = createLogger("redis"); +const logger = createLogger("redis-cache"); const TTL = DAY_MS; const TTL_S = TTL / 1000; @@ -14,15 +15,15 @@ export function createRedisCache(): Cache { }); return { - async get(key: string): Promise { - const value = await Bun.redis.get(key); - if (value == null) return undefined; - - return JSON.parse(value) as T; - }, - async set(key: string, value: T): Promise { - await Bun.redis.set(key, JSON.stringify(value)); - await Bun.redis.expire(key, TTL_S); - }, + with: buildCacheWith( + logger, + async (key) => { + const cached = await Bun.redis.get(key); + return cached == null ? undefined : JSON.parse(cached); + }, + async (key, res) => { + await Bun.redis.set(key, JSON.stringify(res), "EX", TTL_S); + }, + ), }; } diff --git a/src/services/sqlite-cache.ts b/src/services/sqlite-cache.ts new file mode 100644 index 0000000..828ad21 --- /dev/null +++ b/src/services/sqlite-cache.ts @@ -0,0 +1,67 @@ +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; + +import { createLogger } from "@aklinker1/logger"; +import { sql, SQL } from "bun"; + +import { buildCacheWith } from "../utils/cache"; +import { DAY_MS } from "../utils/time"; +import type { Cache } from "./cache"; + +const logger = createLogger("sqlite-cache"); + +const TTL = DAY_MS; +const TABLE = "cache"; + +export function createSqliteCache(): Cache { + const dir = "data"; + mkdirSync(dir, { recursive: true }); + + const path = join(dir, "cache.db"); + logger.info("Using SQLite cache", { path }); + const sqlite = new SQL({ adapter: "sqlite", filename: path }); + + const ready = (async () => { + await sqlite` + CREATE TABLE IF NOT EXISTS ${sql(TABLE)} + ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + addedAt INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_cache_addedAt ON ${sql(TABLE)}(addedAt); + `; + await sqlite`PRAGMA journal_mode = WAL`; + await sqlite` + DELETE FROM ${sql(TABLE)} + WHERE addedAt < ${Date.now() - TTL} + `; + })(); + + const baseWith = buildCacheWith( + logger, + async (key) => { + const [result] = await sqlite` + SELECT value + FROM ${sql(TABLE)} + WHERE key = ${key} AND addedAt > ${Date.now() - TTL} + `; + return result == null ? undefined : JSON.parse(result.value); + }, + async (key, res) => { + await sqlite` + INSERT OR REPLACE INTO ${sql(TABLE)} + (key, value, addedAt) + VALUES (${key}, ${JSON.stringify(res)}, ${Date.now()}) + `; + }, + ); + + return { + with: async (key, fn) => { + await ready; + + return await baseWith(key, fn); + }, + }; +} diff --git a/src/utils/cache.ts b/src/utils/cache.ts new file mode 100644 index 0000000..6e02b73 --- /dev/null +++ b/src/utils/cache.ts @@ -0,0 +1,31 @@ +import type { Logger } from "@aklinker1/logger"; +import { deserializeError } from "@aklinker1/zero-serialize-error"; + +import type { Cache, CacheResult } from "../services/cache"; + +export function buildCacheWith( + logger: Logger, + get: (key: string) => Promise | undefined>, + save: (key: string, res: CacheResult) => Promise, +): Cache["with"] { + return (async (key, fn) => { + const cached = await get(key); + if (cached) { + if (cached.success) return cached.data; + else throw deserializeError(cached.error); + } + + try { + const data = await fn(); + await save(key, { success: true, data }).catch(() => { + logger.warn("Failed to save cache.with success result", { data }); + }); + return data; + } catch (error) { + await save(key, { success: false, error }).catch(() => { + logger.warn("Failed to save cache.with error result", { error }); + }); + throw error; + } + }) as Cache["with"]; +} From f99033b460a76ce443cf2aed69c0ba08e7967b49 Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 1 Oct 2026 17:41:26 -0500 Subject: [PATCH 2/4] Update failed test --- .../__tests__/chrome-crawler.e2e.test.ts | 36 ++++++++----------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/src/services/__tests__/chrome-crawler.e2e.test.ts b/src/services/__tests__/chrome-crawler.e2e.test.ts index 631751e..fcd03fe 100644 --- a/src/services/__tests__/chrome-crawler.e2e.test.ts +++ b/src/services/__tests__/chrome-crawler.e2e.test.ts @@ -2,42 +2,36 @@ import { describe, expect, it } from "bun:test"; import { crawlExtension } from "../chrome-crawler"; -const githubBetterLineCountsId = "ocfdgncpifmegplaglcnglhioflaimkd"; +const animeSkipPlayerId = "mgmdkjcljneegjfajchedjpdhbadklcf"; describe("Chrome Web Store Crawler E2E", () => { it("should load and crawl an extension ID correctly", async () => { - const res = await crawlExtension(githubBetterLineCountsId, "en", true); + const res = await crawlExtension(animeSkipPlayerId, "en", true); expect(res).toEqual({ iconUrl: - "https://lh3.googleusercontent.com/GcffNyCJaxT2G9dsQCJHhUEMlu_E0vEzph5cLPrQj7UHKat7QyCzGu69Dmp_DDUL8rY-bPMFJceQarS1wcqdwTalTg=s256", - id: githubBetterLineCountsId, + "https://lh3.googleusercontent.com/lTplXRdnpEB-7DGRa_1nCKao_3aJ3C_e-GNVs9tQV9hDUXgupc1SsW6OrruxgrkSdFBSOqeia56YWgJI2IgpV1MK49s=s256", + id: animeSkipPlayerId, lastUpdated: expect.any(String), - longDescription: expect.stringContaining("Lots of code is generated nowadays"), - name: "GitHub Better Line Counts", + longDescription: expect.stringContaining("Watch anime faster than ever!"), + name: "Anime Skip Player", rating: expect.any(Number), reviewCount: expect.any(Number), - shortDescription: "Remove generated files from GitHub line counts", + shortDescription: + "Custom video player for anime streaming websites. Skip intros, outros, and more.", storeUrl: expect.stringContaining( - "https://chromewebstore.google.com/detail/github-better-line-counts/ocfdgncpifmegplaglcnglhioflaimkd", + "https://chromewebstore.google.com/detail/anime-skip-player/mgmdkjcljneegjfajchedjpdhbadklcf", ), version: expect.any(String), users: expect.any(Number), weeklyActiveUsers: expect.any(Number), - screenshots: [ - { - index: 0, - indexUrl: - "http://localhost:3000/api/rest/chrome-web-store/ocfdgncpifmegplaglcnglhioflaimkd/screenshots/0", + screenshots: expect.arrayContaining( + [0, 1, 2, 3, 4].map((index) => ({ + index, + indexUrl: `http://localhost:3000/api/rest/chrome-web-store/${animeSkipPlayerId}/screenshots/${index}`, rawUrl: expect.any(String), - }, - { - index: 1, - indexUrl: - "http://localhost:3000/api/rest/chrome-web-store/ocfdgncpifmegplaglcnglhioflaimkd/screenshots/1", - rawUrl: expect.any(String), - }, - ], + })), + ), }); }); }); From 4ca814a0df76769326537aff6919c1d4fc455cc1 Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 1 Oct 2026 17:44:05 -0500 Subject: [PATCH 3/4] fix checks --- cspell.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cspell.yml b/cspell.yml index ae152c6..6c91729 100644 --- a/cspell.yml +++ b/cspell.yml @@ -13,8 +13,9 @@ words: - libgcc - libstdc - linkedom + - mgmdkjcljneegjfajchedjpdhbadklcf - microsoftedge - noindexed - - ocfdgncpifmegplaglcnglhioflaimkd + - outros - oxlint - valkey From 37d25c940e5862e43f309f2d65c3b024c501805d Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 1 Oct 2026 17:45:38 -0500 Subject: [PATCH 4/4] Fix checks --- src/apis/graphql-apis.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apis/graphql-apis.ts b/src/apis/graphql-apis.ts index 1d9d46b..d58eafd 100644 --- a/src/apis/graphql-apis.ts +++ b/src/apis/graphql-apis.ts @@ -37,7 +37,7 @@ export const graphqlApis = createApp({ } `, variables: { - id: "ocfdgncpifmegplaglcnglhioflaimkd", + id: "mgmdkjcljneegjfajchedjpdhbadklcf", }, operationName: "GetExtension", }, @@ -51,7 +51,7 @@ export const graphqlApis = createApp({ example: { data: { chromeExtension: { - id: "ocfdgncpifmegplaglcnglhioflaimkd", + id: "mgmdkjcljneegjfajchedjpdhbadklcf", screenshots: [ { rawUrl: