diff --git a/src/services/__tests__/__snapshots__/chrome-crawler.test.ts.snap b/src/services/__tests__/__snapshots__/chrome-crawler.test.ts.snap index 7cf5a43..ea70195 100644 --- a/src/services/__tests__/__snapshots__/chrome-crawler.test.ts.snap +++ b/src/services/__tests__/__snapshots__/chrome-crawler.test.ts.snap @@ -20,36 +20,6 @@ AI Integration: Extract and convert relevant website information into markdown f "rating": 0, "reviewCount": 0, "screenshots": [ - { - "index": 5, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/kofbbilhmnkcmibjbioafflgmpkbnmme/screenshots/5", - "rawUrl": "https://lh3.googleusercontent.com/SwOe0XmN3ZkCkJQ4IF1a1i2qqCsw0b7I-Ez6VSQflAvafwiJhGbdRW6YAKNX8fAwdv159XogYSFc5Uykgt1fsYSv=s1280", - }, - { - "index": 1, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/kofbbilhmnkcmibjbioafflgmpkbnmme/screenshots/1", - "rawUrl": "https://lh3.googleusercontent.com/vDnkawI_rgAd5ou0gasjR5CQbm-HmbTj9SBTdlpxU_-ri1jeE9dz0qNnSOijaboiUd4tuwQBGg-ujIO-znDN4-InVZA=s1280", - }, - { - "index": 2, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/kofbbilhmnkcmibjbioafflgmpkbnmme/screenshots/2", - "rawUrl": "https://lh3.googleusercontent.com/F9H2EBxfqxrriMb0zdjoFIiCufav1oV8QI-enLqT9AEuL0nPFJLaj0286R2UH_ekxLoVa-yO76f5sSDOAjEuORrRLg=s1280", - }, - { - "index": 3, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/kofbbilhmnkcmibjbioafflgmpkbnmme/screenshots/3", - "rawUrl": "https://lh3.googleusercontent.com/kkK_WlsYeZNGK-sU-4nvH-sEXtx7xUMuqpsK-IHq29IUkmyC3C40n1DDsDdt0f6-LgbKGL_obH_Cse-zJWJPHk-OLQ=s1280", - }, - { - "index": 4, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/kofbbilhmnkcmibjbioafflgmpkbnmme/screenshots/4", - "rawUrl": "https://lh3.googleusercontent.com/VJy9hYOMvlmwp2FhtrpIm8hNwwJVZWl-V4dZVl2cCl4aD8xNyuHaEgdm5bfjMBiZJXI0ysxul8P1GAyFaJrRvjDSuMY=s1280", - }, - { - "index": 5, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/kofbbilhmnkcmibjbioafflgmpkbnmme/screenshots/5", - "rawUrl": "https://lh3.googleusercontent.com/SwOe0XmN3ZkCkJQ4IF1a1i2qqCsw0b7I-Ez6VSQflAvafwiJhGbdRW6YAKNX8fAwdv159XogYSFc5Uykgt1fsYSv=s1280", - }, { "index": 1, "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/kofbbilhmnkcmibjbioafflgmpkbnmme/screenshots/1", @@ -160,36 +130,6 @@ Absolutely! Blync is 100% secure. "rating": 5, "reviewCount": 2, "screenshots": [ - { - "index": 5, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/odffpjnpocjfcaclnenaaaddghkgijdb/screenshots/5", - "rawUrl": "https://lh3.googleusercontent.com/MXf271aHeP9deq2VjO0-_QDU-VSKgu81hcc9iKNMbAg74sI1RnvOb7Esgy95P-NTrQ7vVJtW2fMOPnTYv8pnbnMEng=s1280", - }, - { - "index": 1, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/odffpjnpocjfcaclnenaaaddghkgijdb/screenshots/1", - "rawUrl": "https://lh3.googleusercontent.com/WVvtpr2Wzfr4zT34Z_Gy39ZL-I8UOZ5uReOAz-vmtC8lT4hebXlnur7y0bAWzXd6xihMbTP_PfZgQN_C4MfhixI2=s1280", - }, - { - "index": 2, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/odffpjnpocjfcaclnenaaaddghkgijdb/screenshots/2", - "rawUrl": "https://lh3.googleusercontent.com/e3JEIuiGu_dI70zGBPBXzL9YNZuHy8nItCoEKJSql56CcUVyIQC9Lor8HkeScWE2-qAFw5DLZQijXEfNxlInSoLl0DE=s1280", - }, - { - "index": 3, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/odffpjnpocjfcaclnenaaaddghkgijdb/screenshots/3", - "rawUrl": "https://lh3.googleusercontent.com/O9OuxL28A3GOp5-R8u2kyV9wkOchMTx0ZNWraAz93gqThZRwVpCbaEW0VL1bZ0dKaKr0ReN0qJYJfSFAPOkCuOGA=s1280", - }, - { - "index": 4, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/odffpjnpocjfcaclnenaaaddghkgijdb/screenshots/4", - "rawUrl": "https://lh3.googleusercontent.com/vfXSijB8FQ4hkARrKdJJfXe3b_ahd8yXsSuAbbKNwU9yondg0SIj1fenPJRosimBMo8bi1hj99lypD1CS1y1KhJE=s1280", - }, - { - "index": 5, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/odffpjnpocjfcaclnenaaaddghkgijdb/screenshots/5", - "rawUrl": "https://lh3.googleusercontent.com/MXf271aHeP9deq2VjO0-_QDU-VSKgu81hcc9iKNMbAg74sI1RnvOb7Esgy95P-NTrQ7vVJtW2fMOPnTYv8pnbnMEng=s1280", - }, { "index": 1, "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/odffpjnpocjfcaclnenaaaddghkgijdb/screenshots/1", @@ -251,51 +191,6 @@ With just two clicks, Eye Dropper allows you to pick colors from any website wit "rating": 4.3, "reviewCount": 3, "screenshots": [ - { - "index": 3, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/3", - "rawUrl": "https://lh3.googleusercontent.com/0YMjrWbn7_iSWpjvF45C2xdswzPDZZ8Qf2cPeM2-VrQF8PNX9F0sWWjr-CBbCOH0kogD0CEW2uewbx5TyvLAf1Ia=s1280", - }, - { - "index": 4, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/4", - "rawUrl": "https://lh3.googleusercontent.com/YaXuIdUkKlr7g5tq2mpY8uWvckzknb6xyv9TD6iam0-6ri-7_a1jjntmTv3dz0x0kz38lEUEHW-r9q6zp0eMwPov=s1280", - }, - { - "index": 0, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/0", - "rawUrl": "https://lh3.googleusercontent.com/cLm5vTn9bPNn6QvxuJdcqgP3JF47XB4hKm9rP3XibkyT2arxWo_HwCa6c8RwcIV4Et0c1TXfnDie4Xgj24svzbye=s1280", - }, - { - "index": 1, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/1", - "rawUrl": "https://lh3.googleusercontent.com/AeVZqwDPLzUMZb3JAV2qsD8KbxRnioA176cE3DYBANpYY3TyCQljLcfTKmlGvV5Xdt52AJkjT65ok8t4d0cYnobvIQ=s1280", - }, - { - "index": 2, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/2", - "rawUrl": "https://lh3.googleusercontent.com/Hqn2xlSKEuxQfBe-YHPlhUMjox2G0ZpYd1ouFx9ATfb_YbwtqfVkp9v8AjBI9RzBghbH9yci-cxdoJnwL8VCskRZMQ=s1280", - }, - { - "index": 3, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/3", - "rawUrl": "https://lh3.googleusercontent.com/0YMjrWbn7_iSWpjvF45C2xdswzPDZZ8Qf2cPeM2-VrQF8PNX9F0sWWjr-CBbCOH0kogD0CEW2uewbx5TyvLAf1Ia=s1280", - }, - { - "index": 4, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/4", - "rawUrl": "https://lh3.googleusercontent.com/YaXuIdUkKlr7g5tq2mpY8uWvckzknb6xyv9TD6iam0-6ri-7_a1jjntmTv3dz0x0kz38lEUEHW-r9q6zp0eMwPov=s1280", - }, - { - "index": 0, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/0", - "rawUrl": "https://lh3.googleusercontent.com/cLm5vTn9bPNn6QvxuJdcqgP3JF47XB4hKm9rP3XibkyT2arxWo_HwCa6c8RwcIV4Et0c1TXfnDie4Xgj24svzbye=s1280", - }, - { - "index": 1, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/1", - "rawUrl": "https://lh3.googleusercontent.com/AeVZqwDPLzUMZb3JAV2qsD8KbxRnioA176cE3DYBANpYY3TyCQljLcfTKmlGvV5Xdt52AJkjT65ok8t4d0cYnobvIQ=s1280", - }, { "index": 0, "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/0", @@ -359,31 +254,6 @@ Upgrade your browsing experience—make every new tab a beautiful, functional wo "rating": 4.6, "reviewCount": 10, "screenshots": [ - { - "index": 2, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/dhiekgdaipindoapjmcnpompdknjeijf/screenshots/2", - "rawUrl": "https://lh3.googleusercontent.com/gj2e-tiTc3QYptxlIEcVfRogxZcXaJ_uQu09lK4wwI9ih1f765cnB4Vx-Z2m3ib7YxxeWWonL5ZF2CAHZCAcVck6=s1280", - }, - { - "index": 0, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/dhiekgdaipindoapjmcnpompdknjeijf/screenshots/0", - "rawUrl": "https://lh3.googleusercontent.com/5cf2eynPB11FLePhVBJmft3XXcJJ9Y6ZPL_jcPB2T-eX1k2YW5JORQN5kPvPzvQfQH2GytQWi7iTB1sEfSkafoLY7Q=s1280", - }, - { - "index": 1, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/dhiekgdaipindoapjmcnpompdknjeijf/screenshots/1", - "rawUrl": "https://lh3.googleusercontent.com/fZfIjaptorXfgT-JAsfUsqpHsxweFuMmqfDafphvbDsGdaKtLROsUPeE8_6kmI0QIHOotPMoKiEWbIpcB1wajTaGx_o=s1280", - }, - { - "index": 2, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/dhiekgdaipindoapjmcnpompdknjeijf/screenshots/2", - "rawUrl": "https://lh3.googleusercontent.com/gj2e-tiTc3QYptxlIEcVfRogxZcXaJ_uQu09lK4wwI9ih1f765cnB4Vx-Z2m3ib7YxxeWWonL5ZF2CAHZCAcVck6=s1280", - }, - { - "index": 0, - "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/dhiekgdaipindoapjmcnpompdknjeijf/screenshots/0", - "rawUrl": "https://lh3.googleusercontent.com/5cf2eynPB11FLePhVBJmft3XXcJJ9Y6ZPL_jcPB2T-eX1k2YW5JORQN5kPvPzvQfQH2GytQWi7iTB1sEfSkafoLY7Q=s1280", - }, { "index": 0, "indexUrl": "http://localhost:3000/api/rest/chrome-web-store/dhiekgdaipindoapjmcnpompdknjeijf/screenshots/0", diff --git a/src/services/__tests__/chrome-crawler.e2e.test.ts b/src/services/__tests__/chrome-crawler.e2e.test.ts index fcd03fe..a99b1cb 100644 --- a/src/services/__tests__/chrome-crawler.e2e.test.ts +++ b/src/services/__tests__/chrome-crawler.e2e.test.ts @@ -25,13 +25,11 @@ describe("Chrome Web Store Crawler E2E", () => { version: expect.any(String), users: expect.any(Number), weeklyActiveUsers: expect.any(Number), - screenshots: expect.arrayContaining( - [0, 1, 2, 3, 4].map((index) => ({ - index, - indexUrl: `http://localhost:3000/api/rest/chrome-web-store/${animeSkipPlayerId}/screenshots/${index}`, - rawUrl: expect.any(String), - })), - ), + screenshots: [0, 1, 2, 3, 4].map((index) => ({ + index, + indexUrl: `http://localhost:3000/api/rest/chrome-web-store/${animeSkipPlayerId}/screenshots/${index}`, + rawUrl: expect.any(String), + })), }); }); }); diff --git a/src/services/chrome-crawler.ts b/src/services/chrome-crawler.ts index 65f6495..144a459 100644 --- a/src/services/chrome-crawler.ts +++ b/src/services/chrome-crawler.ts @@ -126,18 +126,30 @@ export async function crawlExtension( ?.textContent?.replaceAll("\n\n", "\n"), ]); + const extractScreenshots = (includeCarousel: boolean): Gql.Screenshot[] => { + const screenshots = [...document.querySelectorAll("div[data-media-url]")] + .filter((div) => div.getAttribute("data-is-video") === "false") + // The main carousel contains cloned slides at both ends for infinite scrolling, so by + // default only use the preview thumbnails, which are listed exactly once. + .filter((div) => includeCarousel || !div.closest('[role="region"]')) + .map((div) => { + const index = parseInt(div.getAttribute("data-slide-index")); + return { + index, + rawUrl: div.getAttribute("data-media-url") + "=s1280", // "s1280" gets the full resolution + indexUrl: buildScreenshotUrl(ExtensionStoreName.ChromeWebStore, id, index), + }; + }); + if (!includeCarousel) return screenshots; + + // Remove duplicates from the carousel's cloned slides + return [...new Map(screenshots.map((s) => [s.index, s])).values()].sort( + (a, b) => a.index - b.index, + ); + }; const screenshots = tryExtract("screenshots", validateGqlScreenshots, [ - () => - [...document.querySelectorAll("div[data-media-url]")] - .filter((div) => div.getAttribute("data-is-video") === "false") - .map((div) => { - const index = parseInt(div.getAttribute("data-slide-index")); - return { - index, - rawUrl: div.getAttribute("data-media-url") + "=s1280", // "s1280" gets the full resolution - indexUrl: buildScreenshotUrl(ExtensionStoreName.ChromeWebStore, id, index), - }; - }), + () => extractScreenshots(false), + () => extractScreenshots(true), ]); const result: Gql.ChromeExtension = {