feat: Add support for an SQLite cache as a low-memory option

.gitignore

@@ -27,3 +27,4 @@ tsconfig.tsbuildinfo
 # Other Files
 src/crawlers/__tests__/fixtures/chrome-web-store/.new/*
 !src/crawlers/__tests__/fixtures/chrome-web-store/.new/.keep
+/data

cspell.yml

@@ -13,8 +13,9 @@ words:
   - libgcc
   - libstdc
   - linkedom
+  - mgmdkjcljneegjfajchedjpdhbadklcf
   - microsoftedge
   - noindexed
-  - ocfdgncpifmegplaglcnglhioflaimkd
+  - outros
   - oxlint
   - valkey

package.json

@@ -17,6 +17,7 @@
   "dependencies": {
     "@aklinker1/logger": "^1.0.6",
     "@aklinker1/zero-ioc": "^1.5.1",
+    "@aklinker1/zero-serialize-error": "^1.0.2",
     "@aklinker1/zeta": "^2.2.0",
     "dataloader": "^2.2.3",
     "dedent": "^1.7.2",

src/apis/graphql-apis.ts

@@ -37,7 +37,7 @@ export const graphqlApis = createApp({
               }
             `,
             variables: {
-              id: "ocfdgncpifmegplaglcnglhioflaimkd",
+              id: "mgmdkjcljneegjfajchedjpdhbadklcf",
             },
             operationName: "GetExtension",
           },
@@ -51,7 +51,7 @@ export const graphqlApis = createApp({
           example: {
             data: {
               chromeExtension: {
-                id: "ocfdgncpifmegplaglcnglhioflaimkd",
+                id: "mgmdkjcljneegjfajchedjpdhbadklcf",
                 screenshots: [
                   {
                     rawUrl:

src/dependencies.ts

@@ -9,9 +9,17 @@ import { createFirefoxAddonStore } from "./services/firefox-addon-store";
 import { createFirefoxApi } from "./services/firefox-api";
 import { createInMemoryCache } from "./services/in-memory-cache";
 import { createRedisCache } from "./services/redis-cache";
+import { createSqliteCache } from "./services/sqlite-cache";
 
 export const container = createIocContainer()
-  .register("cache", Bun.redis.connected ? createRedisCache : createInMemoryCache)
+  .register(
+    "cache",
+    process.env.SQLITE_CACHE === "true"
+      ? createSqliteCache
+      : Bun.redis.connected
+        ? createRedisCache
+        : createInMemoryCache,
+  )
   .register("edgeApi", createEdgeApi)
   .register("firefoxApi", createFirefoxApi)
   .register("chromeWebStore", transient(createChromeWebStore))

src/services/__tests__/chrome-crawler.e2e.test.ts

@@ -2,42 +2,36 @@ import { describe, expect, it } from "bun:test";
 
 import { crawlExtension } from "../chrome-crawler";
 
-const githubBetterLineCountsId = "ocfdgncpifmegplaglcnglhioflaimkd";
+const animeSkipPlayerId = "mgmdkjcljneegjfajchedjpdhbadklcf";
 
 describe("Chrome Web Store Crawler E2E", () => {
   it("should load and crawl an extension ID correctly", async () => {
-    const res = await crawlExtension(githubBetterLineCountsId, "en", true);
+    const res = await crawlExtension(animeSkipPlayerId, "en", true);
 
     expect(res).toEqual({
       iconUrl:
-        "https://lh3.googleusercontent.com/GcffNyCJaxT2G9dsQCJHhUEMlu_E0vEzph5cLPrQj7UHKat7QyCzGu69Dmp_DDUL8rY-bPMFJceQarS1wcqdwTalTg=s256",
-      id: githubBetterLineCountsId,
+        "https://lh3.googleusercontent.com/lTplXRdnpEB-7DGRa_1nCKao_3aJ3C_e-GNVs9tQV9hDUXgupc1SsW6OrruxgrkSdFBSOqeia56YWgJI2IgpV1MK49s=s256",
+      id: animeSkipPlayerId,
       lastUpdated: expect.any(String),
-      longDescription: expect.stringContaining("Lots of code is generated nowadays"),
-      name: "GitHub Better Line Counts",
+      longDescription: expect.stringContaining("Watch anime faster than ever!"),
+      name: "Anime Skip Player",
       rating: expect.any(Number),
       reviewCount: expect.any(Number),
-      shortDescription: "Remove generated files from GitHub line counts",
+      shortDescription:
+        "Custom video player for anime streaming websites. Skip intros, outros, and more.",
       storeUrl: expect.stringContaining(
-        "https://chromewebstore.google.com/detail/github-better-line-counts/ocfdgncpifmegplaglcnglhioflaimkd",
+        "https://chromewebstore.google.com/detail/anime-skip-player/mgmdkjcljneegjfajchedjpdhbadklcf",
       ),
       version: expect.any(String),
       users: expect.any(Number),
       weeklyActiveUsers: expect.any(Number),
-      screenshots: [
-        {
-          index: 0,
-          indexUrl:
-            "http://localhost:3000/api/rest/chrome-web-store/ocfdgncpifmegplaglcnglhioflaimkd/screenshots/0",
+      screenshots: expect.arrayContaining(
+        [0, 1, 2, 3, 4].map((index) => ({
+          index,
+          indexUrl: `http://localhost:3000/api/rest/chrome-web-store/${animeSkipPlayerId}/screenshots/${index}`,
           rawUrl: expect.any(String),
-        },
-        {
-          index: 1,
-          indexUrl:
-            "http://localhost:3000/api/rest/chrome-web-store/ocfdgncpifmegplaglcnglhioflaimkd/screenshots/1",
-          rawUrl: expect.any(String),
-        },
-      ],
+        })),
+      ),
     });
   });
 });

src/services/cache.ts

@@ -1,4 +1,7 @@
 export interface Cache {
-  get<T>(key: string): Promise<T | undefined>;
-  set<T>(key: string, value: T): Promise<void>;
+  with<T>(key: string, fn: () => Promise<T>): Promise<T>;
 }
+
+export type CacheSuccess<T> = { success: true; data: T };
+export type CacheFailure = { success: false; error: unknown };
+export type CacheResult<T> = CacheSuccess<T> | CacheFailure;

src/services/chrome-web-store.ts

@@ -7,7 +7,7 @@ export type ChromeWebStore = ExtensionStore<Gql.ChromeExtension>;
 export function createChromeWebStore({ cache }: { cache: Cache }): ChromeWebStore {
   return new ExtensionStore({
     fetch: (id) => crawlExtension(String(id), "en"),
-    cacheKeyPrefix: "chrome-extension-",
+    cacheKeyPrefix: "chrome-extension-v2-",
     cache,
   });
 }

src/services/edge-addon-store.ts

@@ -13,7 +13,7 @@ export function createEdgeAddonStore({
 }): EdgeAddonStore {
   return new ExtensionStore({
     fetch: (id) => edgeApi.getAddon(String(id)),
-    cacheKeyPrefix: "edge-addon-",
+    cacheKeyPrefix: "edge-addon-v2-",
     cache,
   });
 }

src/services/extension-store.ts

@@ -17,16 +17,9 @@ export class ExtensionStore<TGqlExtension extends Gql.Extension> {
     this.dataloader = new DataLoader<ExtensionId, TGqlExtension>(
       async (ids): Promise<Array<TGqlExtension | Error>> => {
         const results = await Promise.allSettled(
-          ids.map(async (id) => {
-            const cacheKey = options.cacheKeyPrefix + id;
-            const cached = await options.cache.get(cacheKey);
-            if (cached) return cached;
-
-            const result = await options.fetch(id);
-            if (result) await options.cache.set(cacheKey, result);
-
-            return result;
-          }),
+          ids.map(async (id) =>
+            options.cache.with(options.cacheKeyPrefix + id, () => options.fetch(id)),
+          ),
         );
         return results.map((res) => (res.status === "fulfilled" ? res.value : res.reason));
       },

src/services/firefox-addon-store.ts

@@ -13,7 +13,7 @@ export function createFirefoxAddonStore({
 }): FirefoxAddonStore {
   return new ExtensionStore({
     fetch: (id) => firefoxApi.getAddon(String(id)),
-    cacheKeyPrefix: "firefox-addon-",
+    cacheKeyPrefix: "firefox-addon-v2-",
     cache,
   });
 }

src/services/in-memory-cache.ts

@@ -1,29 +1,33 @@
 import { createLogger } from "@aklinker1/logger";
 
+import { buildCacheWith } from "../utils/cache";
 import { HOUR_MS } from "../utils/time";
-import type { Cache } from "./cache";
+import type { Cache, CacheResult } from "./cache";
 
-const logger = createLogger("redis");
+const logger = createLogger("in-memory-cache");
 
 const TTL = HOUR_MS;
 
 export function createInMemoryCache(): Cache {
   logger.info("Using in-memory cache");
 
-  let cache: Record<string, any> = Object.create(null);
+  let cache: Record<string, CacheResult<unknown>> = Object.create(null);
   let ttl: Record<string, number> = Object.create(null);
 
   return {
-    get: async (key: string) => {
-      if (ttl[key] && Date.now() > ttl[key]) {
-        delete cache[key];
-        delete ttl[key];
-      }
-      return cache[key];
-    },
-    set: async (key: string, value: any) => {
-      cache[key] = value;
-      ttl[key] = Date.now() + TTL;
-    },
+    with: buildCacheWith(
+      logger,
+      async (key) => {
+        if (ttl[key] && Date.now() > ttl[key]) {
+          delete cache[key];
+          delete ttl[key];
+        }
+        return cache[key];
+      },
+      async (key, res) => {
+        cache[key] = res;
+        ttl[key] = Date.now() + TTL;
+      },
+    ),
   };
 }

src/services/redis-cache.ts

@@ -1,9 +1,10 @@
 import { createLogger } from "@aklinker1/logger";
 
+import { buildCacheWith } from "../utils/cache";
 import { DAY_MS } from "../utils/time";
 import type { Cache } from "./cache";
 
-const logger = createLogger("redis");
+const logger = createLogger("redis-cache");
 
 const TTL = DAY_MS;
 const TTL_S = TTL / 1000;
@@ -14,15 +15,15 @@ export function createRedisCache(): Cache {
   });
 
   return {
-    async get<T>(key: string): Promise<T | undefined> {
-      const value = await Bun.redis.get(key);
-      if (value == null) return undefined;
-
-      return JSON.parse(value) as T;
-    },
-    async set<T>(key: string, value: T): Promise<void> {
-      await Bun.redis.set(key, JSON.stringify(value));
-      await Bun.redis.expire(key, TTL_S);
-    },
+    with: buildCacheWith(
+      logger,
+      async (key) => {
+        const cached = await Bun.redis.get(key);
+        return cached == null ? undefined : JSON.parse(cached);
+      },
+      async (key, res) => {
+        await Bun.redis.set(key, JSON.stringify(res), "EX", TTL_S);
+      },
+    ),
   };
 }

src/services/sqlite-cache.ts

@@ -0,0 +1,67 @@
+import { mkdirSync } from "node:fs";
+import { join } from "node:path";
+
+import { createLogger } from "@aklinker1/logger";
+import { sql, SQL } from "bun";
+
+import { buildCacheWith } from "../utils/cache";
+import { DAY_MS } from "../utils/time";
+import type { Cache } from "./cache";
+
+const logger = createLogger("sqlite-cache");
+
+const TTL = DAY_MS;
+const TABLE = "cache";
+
+export function createSqliteCache(): Cache {
+  const dir = "data";
+  mkdirSync(dir, { recursive: true });
+
+  const path = join(dir, "cache.db");
+  logger.info("Using SQLite cache", { path });
+  const sqlite = new SQL({ adapter: "sqlite", filename: path });
+
+  const ready = (async () => {
+    await sqlite`
+      CREATE TABLE IF NOT EXISTS ${sql(TABLE)}
+      (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL,
+        addedAt INTEGER NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_cache_addedAt ON ${sql(TABLE)}(addedAt);
+    `;
+    await sqlite`PRAGMA journal_mode = WAL`;
+    await sqlite`
+      DELETE FROM ${sql(TABLE)}
+      WHERE addedAt < ${Date.now() - TTL}
+    `;
+  })();
+
+  const baseWith = buildCacheWith(
+    logger,
+    async (key) => {
+      const [result] = await sqlite`
+        SELECT value
+        FROM ${sql(TABLE)}
+        WHERE key = ${key} AND addedAt > ${Date.now() - TTL}
+      `;
+      return result == null ? undefined : JSON.parse(result.value);
+    },
+    async (key, res) => {
+      await sqlite`
+        INSERT OR REPLACE INTO ${sql(TABLE)}
+        (key, value, addedAt)
+        VALUES (${key}, ${JSON.stringify(res)}, ${Date.now()})
+      `;
+    },
+  );
+
+  return {
+    with: async (key, fn) => {
+      await ready;
+
+      return await baseWith(key, fn);
+    },
+  };
+}

src/utils/cache.ts

@@ -0,0 +1,31 @@
+import type { Logger } from "@aklinker1/logger";
+import { deserializeError } from "@aklinker1/zero-serialize-error";
+
+import type { Cache, CacheResult } from "../services/cache";
+
+export function buildCacheWith(
+  logger: Logger,
+  get: (key: string) => Promise<CacheResult<unknown> | undefined>,
+  save: (key: string, res: CacheResult<unknown>) => Promise<void>,
+): Cache["with"] {
+  return (async (key, fn) => {
+    const cached = await get(key);
+    if (cached) {
+      if (cached.success) return cached.data;
+      else throw deserializeError(cached.error);
+    }
+
+    try {
+      const data = await fn();
+      await save(key, { success: true, data }).catch(() => {
+        logger.warn("Failed to save cache.with success result", { data });
+      });
+      return data;
+    } catch (error) {
+      await save(key, { success: false, error }).catch(() => {
+        logger.warn("Failed to save cache.with error result", { error });
+      });
+      throw error;
+    }
+  }) as Cache["with"];
+}