diff --git a/bun.lock b/bun.lock index fd8501a..78ff047 100644 --- a/bun.lock +++ b/bun.lock @@ -9,6 +9,7 @@ "@aklinker1/zero-ioc": "^1.6.0", "@aklinker1/zero-serialize-error": "^1.0.2", "@aklinker1/zeta": "^2.2.0", + "@prometheus-io/client": "^0.16.1", "dataloader": "^2.2.3", "dedent": "^1.7.2", "graphql": "^16.13.2", @@ -184,6 +185,8 @@ "@cspell/url": ["@cspell/url@10.0.0", "", {}, "sha512-q+0pHQ8DbqjemyaOn/mTtBRbCuKDqhnsVbZ6J9zkTsxPgMpccjy0s5oLXwomfrrxMRBH+UcbERwtUmE+SbnoIQ=="], + "@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="], + "@oxfmt/binding-android-arm-eabi": ["@oxfmt/binding-android-arm-eabi@0.61.0", "", { "os": "android", "cpu": "arm" }, "sha512-BaS+1OVvg9sr+Xav0+KdWedQRcAzrdoEcwMZeqoc2F6ieC1s/t5eM35YQoRPQ7vAqkZ+p3tbQb1r9I9mrV5oGA=="], "@oxfmt/binding-android-arm64": ["@oxfmt/binding-android-arm64@0.61.0", "", { "os": "android", "cpu": "arm64" }, "sha512-of8atAV0M1egGcVOMbgZCvc10sFOP3ayQBNQV5h5G3fNq8gACdEswfFk9bzGrdbM23rtg0Coxi7np7oPLcueNw=="], @@ -260,6 +263,8 @@ "@oxlint/binding-win32-x64-msvc": ["@oxlint/binding-win32-x64-msvc@1.59.0", "", { "os": "win32", "cpu": "x64" }, "sha512-xkE7puteDS/vUyRngLXW0t8WgdWoS/tfxXjhP/P7SMqPDx+hs44SpssO3h3qmTqECYEuXBUPzcAw5257Ka+ofA=="], + "@prometheus-io/client": ["@prometheus-io/client@0.16.1", "", { "dependencies": { "@opentelemetry/api": "^1.4.0", "tdigest": "^0.1.1" } }, "sha512-XkFPsGFGoSs/UMg/vh0QLDpanl/tsrOV5PLHdpGe8Ho+ir/Llhytr15j6y8ZwLrpIGzK9gTVVpIalA6C4UjUZw=="], + "@standard-schema/spec": ["@standard-schema/spec@1.0.0", "", {}, "sha512-m2bOd0f2RT9k8QJx1JN85cZYyH1RqFBdlwtkSlf4tBDYLCiiZnv1fIIwacK6cqwXavOydf0NPToMQgpKq+dVlA=="], "@types/bun": ["@types/bun@1.4.2", "", { "dependencies": { "bun-types": "1.4.2" } }, "sha512-GimotNn7+ZV0uVArItBbriZsR1oNf0+WTzPkdcFrzShI7k2norL0uzEaJT8T33dWr7O/c9ZDuAFQrctKCi72oQ=="], @@ -310,6 +315,8 @@ "array-timsort": ["array-timsort@1.0.3", "", {}, "sha512-/+3GRL7dDAGEfM6TseQk/U+mi18TU2Ms9I3UlLdUMhz2hbvGNTKdj9xniwXfUqgYhHxRx0+8UnKkvlNwVU+cWQ=="], + "bintrees": ["bintrees@1.0.2", "", {}, "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw=="], + "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], "bun-types": ["bun-types@1.4.2", "", { "dependencies": { "@types/node": "*" } }, "sha512-bxV1FgK7yBIzjRe5zBozIM4Bem11ZJcCXSrjWRG3YWLt8yFDePu4cLjpebO8OvPeIE9trbyPF4fuj3Cia4Fj3w=="], @@ -420,6 +427,8 @@ "superlock": ["superlock@1.3.5", "", {}, "sha512-XpWNthvezZnWp0u7/UL8rBbBOnq2Qx39fw+0RNMC/+eotOd81glzsmIWVH0ejarhTeDQihxOKSbjm2XXFo5a8w=="], + "tdigest": ["tdigest@0.1.3", "", { "dependencies": { "bintrees": "1.0.2" } }, "sha512-zbRt+lT+/H4fRItHshczHErVCQnitJk8MfMT24MqFJf3YL7SJJPqGIGeuOdvxXxM/AHFzKBl7WoyaYwqO9s3Kw=="], + "tinyglobby": ["tinyglobby@0.2.16", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg=="], "tinypool": ["tinypool@2.1.0", "", {}, "sha512-Pugqs6M0m7Lv1I7FtxN4aoyToKg1C4tu+/381vH35y8oENM/Ai7f7C4StcoK4/+BSw9ebcS8jRiVrORFKCALLw=="], diff --git a/package.json b/package.json index bbd5c63..6aaa33c 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "@aklinker1/zero-ioc": "^1.6.0", "@aklinker1/zero-serialize-error": "^1.0.2", "@aklinker1/zeta": "^2.2.0", + "@prometheus-io/client": "^0.16.1", "dataloader": "^2.2.3", "dedent": "^1.7.2", "graphql": "^16.13.2", diff --git a/scripts/dev.ts b/scripts/dev.ts index ca510a7..81fb09e 100644 --- a/scripts/dev.ts +++ b/scripts/dev.ts @@ -2,9 +2,12 @@ import { createLogger } from "@aklinker1/logger"; import { version } from "../package.json"; +import { startMetricsServer } from "../src/metrics-server"; import app from "../src/server"; import { generateGqlTypes } from "./generate-gql-types"; +startMetricsServer(); + const logger = createLogger("http"); const fetch = app.build(); diff --git a/src/main.ts b/src/main.ts index ce559f3..87a8aef 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,8 +2,11 @@ import { createLogger } from "@aklinker1/logger"; import { version } from "../package.json"; +import { startMetricsServer } from "./metrics-server"; import app from "./server"; +startMetricsServer(); + const logger = createLogger("http"); const port = Number(process.env.PORT ?? "3000"); diff --git a/src/metrics-server.ts b/src/metrics-server.ts new file mode 100644 index 0000000..d1adae5 --- /dev/null +++ b/src/metrics-server.ts @@ -0,0 +1,20 @@ +import { register } from "@prometheus-io/client"; + +/** + * Serve internal metrics on another port so infra can control access separate from the main API + * server. + */ +export function startMetricsServer() { + return Bun.serve({ + hostname: process.env.METRICS_HOSTNAME, + port: Number(process.env.METRICS_PORT) || 3333, + routes: { + "/metrics": { + GET: async () => + new Response(await register.metrics(), { + headers: { "Content-Type": register.contentType }, + }), + }, + }, + }); +} diff --git a/src/services/chrome-web-store.ts b/src/services/chrome-web-store.ts index 608dc67..39fee60 100644 --- a/src/services/chrome-web-store.ts +++ b/src/services/chrome-web-store.ts @@ -1,9 +1,13 @@ import type { Cache } from "./cache"; import { crawlExtension } from "./chrome-crawler"; -import { ExtensionStore } from "./extension-store"; +import { ExtensionStore, type ExtensionId } from "./extension-store"; export class ChromeWebStore extends ExtensionStore { constructor(deps: { cache: Cache }) { - super(deps.cache, "chrome-extension-v2-", (id) => crawlExtension(String(id), "en")); + super(deps.cache, "chrome-web-store", "chrome-extension-v2-"); + } + + protected fetchExtension(id: ExtensionId): Promise { + return crawlExtension(String(id), "en"); } } diff --git a/src/services/edge-addon-store.ts b/src/services/edge-addon-store.ts index 13333f6..bb525fe 100644 --- a/src/services/edge-addon-store.ts +++ b/src/services/edge-addon-store.ts @@ -1,9 +1,13 @@ import type { Cache } from "./cache"; import type { EdgeApi } from "./edge-api"; -import { ExtensionStore } from "./extension-store"; +import { ExtensionStore, type ExtensionId } from "./extension-store"; export class EdgeAddonStore extends ExtensionStore { - constructor(deps: { cache: Cache; edgeApi: EdgeApi }) { - super(deps.cache, "edge-addon-v2-", (id) => deps.edgeApi.getAddon(String(id))); + constructor(private deps: { cache: Cache; edgeApi: EdgeApi }) { + super(deps.cache, "edge-addon-store", "edge-addon-v2-"); + } + + protected fetchExtension(id: ExtensionId): Promise { + return this.deps.edgeApi.getAddon(String(id)); } } diff --git a/src/services/extension-store.ts b/src/services/extension-store.ts index 3f32880..9aaaea7 100644 --- a/src/services/extension-store.ts +++ b/src/services/extension-store.ts @@ -1,21 +1,33 @@ import DataLoader from "dataloader"; +import { fetchExtensionCounter } from "../utils/metrics"; import type { Cache } from "./cache"; export type ExtensionId = string | number; -export class ExtensionStore { +export abstract class ExtensionStore { private dataloader: DataLoader; constructor( readonly cache: Cache, + readonly storeName: string, readonly cacheKeyPrefix: string, - readonly fetch: (id: ExtensionId) => Promise, ) { this.dataloader = new DataLoader( async (ids): Promise> => { const results = await Promise.allSettled( - ids.map(async (id) => cache.with(cacheKeyPrefix + id, () => fetch(id))), + ids.map((id) => + cache.with(cacheKeyPrefix + id, async () => { + try { + const res = await this.fetchExtension(id); + fetchExtensionCounter.inc({ store_name: this.storeName, result: "success" }); + return res; + } catch (err) { + fetchExtensionCounter.inc({ store_name: this.storeName, result: "error" }); + throw err; + } + }), + ), ); return results.map((res) => (res.status === "fulfilled" ? res.value : res.reason)); }, @@ -43,4 +55,6 @@ export class ExtensionStore { ); return screenshot?.rawUrl; } + + protected abstract fetchExtension(id: ExtensionId): Promise; } diff --git a/src/services/firefox-addon-store.ts b/src/services/firefox-addon-store.ts index 0f93e0f..d607b96 100644 --- a/src/services/firefox-addon-store.ts +++ b/src/services/firefox-addon-store.ts @@ -1,9 +1,13 @@ import type { Cache } from "./cache"; -import { ExtensionStore } from "./extension-store"; +import { ExtensionStore, type ExtensionId } from "./extension-store"; import type { FirefoxApi } from "./firefox-api"; export class FirefoxAddonStore extends ExtensionStore { - constructor(deps: { cache: Cache; firefoxApi: FirefoxApi }) { - super(deps.cache, "firefox-addon-v2-", (id) => deps.firefoxApi.getAddon(String(id))); + constructor(private deps: { cache: Cache; firefoxApi: FirefoxApi }) { + super(deps.cache, "firefox-addon-store", "firefox-addon-v2-"); + } + + protected fetchExtension(id: ExtensionId): Promise { + return this.deps.firefoxApi.getAddon(id); } } diff --git a/src/utils/metrics.ts b/src/utils/metrics.ts new file mode 100644 index 0000000..9773033 --- /dev/null +++ b/src/utils/metrics.ts @@ -0,0 +1,11 @@ +import { collectDefaultMetrics, Counter } from "@prometheus-io/client"; + +const prefix = "wxt_queue_"; + +collectDefaultMetrics({ prefix }); + +export const fetchExtensionCounter = new Counter({ + name: prefix + "fetch_extension_total", + help: "The number of times the server fetched real, upstream extension details without caching", + labelNames: ["store_name", "result"] as const, +});