feat: Add prometheus metrics for fetching extension without a cache

package.json

@@ -19,6 +19,7 @@
     "@aklinker1/zero-ioc": "^1.6.0",
     "@aklinker1/zero-serialize-error": "^1.0.2",
     "@aklinker1/zeta": "^2.2.0",
+    "@prometheus-io/client": "^0.16.1",
     "dataloader": "^2.2.3",
     "dedent": "^1.7.2",
     "graphql": "^16.13.2",

scripts/dev.ts

@@ -2,9 +2,12 @@
 import { createLogger } from "@aklinker1/logger";
 
 import { version } from "../package.json";
+import { startMetricsServer } from "../src/metrics-server";
 import app from "../src/server";
 import { generateGqlTypes } from "./generate-gql-types";
 
+startMetricsServer();
+
 const logger = createLogger("http");
 
 const fetch = app.build();

src/main.ts

@@ -2,8 +2,11 @@
 import { createLogger } from "@aklinker1/logger";
 
 import { version } from "../package.json";
+import { startMetricsServer } from "./metrics-server";
 import app from "./server";
 
+startMetricsServer();
+
 const logger = createLogger("http");
 
 const port = Number(process.env.PORT ?? "3000");

src/metrics-server.ts

@@ -0,0 +1,20 @@
+import { register } from "@prometheus-io/client";
+
+/**
+ * Serve internal metrics on another port so infra can control access separate from the main API
+ * server.
+ */
+export function startMetricsServer() {
+  return Bun.serve({
+    hostname: process.env.METRICS_HOSTNAME,
+    port: Number(process.env.METRICS_PORT) || 3333,
+    routes: {
+      "/metrics": {
+        GET: async () =>
+          new Response(await register.metrics(), {
+            headers: { "Content-Type": register.contentType },
+          }),
+      },
+    },
+  });
+}

src/services/chrome-web-store.ts

@@ -1,9 +1,13 @@
 import type { Cache } from "./cache";
 import { crawlExtension } from "./chrome-crawler";
-import { ExtensionStore } from "./extension-store";
+import { ExtensionStore, type ExtensionId } from "./extension-store";
 
 export class ChromeWebStore extends ExtensionStore<Gql.ChromeExtension> {
   constructor(deps: { cache: Cache }) {
-    super(deps.cache, "chrome-extension-v2-", (id) => crawlExtension(String(id), "en"));
+    super(deps.cache, "chrome-web-store", "chrome-extension-v2-");
+  }
+
+  protected fetchExtension(id: ExtensionId): Promise<Gql.ChromeExtension | undefined> {
+    return crawlExtension(String(id), "en");
   }
 }

src/services/edge-addon-store.ts

@@ -1,9 +1,13 @@
 import type { Cache } from "./cache";
 import type { EdgeApi } from "./edge-api";
-import { ExtensionStore } from "./extension-store";
+import { ExtensionStore, type ExtensionId } from "./extension-store";
 
 export class EdgeAddonStore extends ExtensionStore<Gql.EdgeAddon> {
-  constructor(deps: { cache: Cache; edgeApi: EdgeApi }) {
-    super(deps.cache, "edge-addon-v2-", (id) => deps.edgeApi.getAddon(String(id)));
+  constructor(private deps: { cache: Cache; edgeApi: EdgeApi }) {
+    super(deps.cache, "edge-addon-store", "edge-addon-v2-");
+  }
+
+  protected fetchExtension(id: ExtensionId): Promise<Gql.EdgeAddon | undefined> {
+    return this.deps.edgeApi.getAddon(String(id));
   }
 }

src/services/extension-store.ts

@@ -1,21 +1,33 @@
 import DataLoader from "dataloader";
 
+import { fetchExtensionCounter } from "../utils/metrics";
 import type { Cache } from "./cache";
 
 export type ExtensionId = string | number;
 
-export class ExtensionStore<TGqlExtension extends Gql.Extension> {
+export abstract class ExtensionStore<TGqlExtension extends Gql.Extension> {
   private dataloader: DataLoader<ExtensionId, TGqlExtension>;
 
   constructor(
     readonly cache: Cache,
+    readonly storeName: string,
     readonly cacheKeyPrefix: string,
-    readonly fetch: (id: ExtensionId) => Promise<TGqlExtension | undefined>,
   ) {
     this.dataloader = new DataLoader<ExtensionId, TGqlExtension>(
       async (ids): Promise<Array<TGqlExtension | Error>> => {
         const results = await Promise.allSettled(
-          ids.map(async (id) => cache.with(cacheKeyPrefix + id, () => fetch(id))),
+          ids.map((id) =>
+            cache.with(cacheKeyPrefix + id, async () => {
+              try {
+                const res = await this.fetchExtension(id);
+                fetchExtensionCounter.inc({ store_name: this.storeName, result: "success" });
+                return res;
+              } catch (err) {
+                fetchExtensionCounter.inc({ store_name: this.storeName, result: "error" });
+                throw err;
+              }
+            }),
+          ),
         );
         return results.map((res) => (res.status === "fulfilled" ? res.value : res.reason));
       },
@@ -43,4 +55,6 @@ export class ExtensionStore<TGqlExtension extends Gql.Extension> {
     );
     return screenshot?.rawUrl;
   }
+
+  protected abstract fetchExtension(id: ExtensionId): Promise<TGqlExtension | undefined>;
 }

src/services/firefox-addon-store.ts

@@ -1,9 +1,13 @@
 import type { Cache } from "./cache";
-import { ExtensionStore } from "./extension-store";
+import { ExtensionStore, type ExtensionId } from "./extension-store";
 import type { FirefoxApi } from "./firefox-api";
 
 export class FirefoxAddonStore extends ExtensionStore<Gql.FirefoxAddon> {
-  constructor(deps: { cache: Cache; firefoxApi: FirefoxApi }) {
-    super(deps.cache, "firefox-addon-v2-", (id) => deps.firefoxApi.getAddon(String(id)));
+  constructor(private deps: { cache: Cache; firefoxApi: FirefoxApi }) {
+    super(deps.cache, "firefox-addon-store", "firefox-addon-v2-");
+  }
+
+  protected fetchExtension(id: ExtensionId): Promise<Gql.FirefoxAddon | undefined> {
+    return this.deps.firefoxApi.getAddon(id);
   }
 }

src/utils/metrics.ts

@@ -0,0 +1,11 @@
+import { collectDefaultMetrics, Counter } from "@prometheus-io/client";
+
+const prefix = "wxt_queue_";
+
+collectDefaultMetrics({ prefix });
+
+export const fetchExtensionCounter = new Counter({
+  name: prefix + "fetch_extension_total",
+  help: "The number of times the server fetched real, upstream extension details without caching",
+  labelNames: ["store_name", "result"] as const,
+});