From 32fdd7ad980106c016f5ef7f4c7a412c73b9bd2c Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Sun, 11 Oct 2026 00:20:37 +1100 Subject: [PATCH] feat(identity): count anonymous sign-in and setup outcomes --- app/pages/cli/authorize.vue | 7 +++ checks/_helpers/collectors.mjs | 6 ++- checks/_helpers/observability.mjs | 4 +- checks/_helpers/signup-analytics.ts | 18 +++++++ .../app/composables/useSignupEvents.ts | 13 +++++ layers/identity/app/pages/me/index.vue | 14 ++++- .../app/pages/onboarding/discover.vue | 10 +++- .../identity/app/pages/onboarding/email.vue | 13 ++++- .../server/api/cli/device/poll.post.ts | 5 +- .../server/api/cli/oauth/token.post.ts | 5 +- .../identity/server/api/events/signup.post.ts | 13 +++++ .../identity/server/routes/auth/github.get.ts | 9 +++- .../identity/server/utils/signup-analytics.ts | 25 +++++++++ layers/marketing/content/pages/privacy.md | 8 +-- shared/checkin-external.ts | 2 +- shared/signup-analytics.ts | 38 +++++++++++++ test/unit/email-form-submit.nuxt.test.ts | 4 +- test/unit/external-checkin-collectors.test.ts | 6 ++- test/unit/github-login-intent.nuxt.test.ts | 7 ++- test/unit/signup-analytics.test.ts | 54 +++++++++++++++++++ test/unit/signup-event-endpoint.test.ts | 35 ++++++++++++ 21 files changed, 276 insertions(+), 20 deletions(-) create mode 100644 checks/_helpers/signup-analytics.ts create mode 100644 layers/identity/app/composables/useSignupEvents.ts create mode 100644 layers/identity/server/api/events/signup.post.ts create mode 100644 layers/identity/server/utils/signup-analytics.ts create mode 100644 shared/signup-analytics.ts create mode 100644 test/unit/signup-analytics.test.ts create mode 100644 test/unit/signup-event-endpoint.test.ts diff --git a/app/pages/cli/authorize.vue b/app/pages/cli/authorize.vue index 53b7c283..66528f54 100644 --- a/app/pages/cli/authorize.vue +++ b/app/pages/cli/authorize.vue @@ -3,6 +3,8 @@ definePageMeta({ layout: 'auth', middleware: ['session'] }) const route = useRoute() const { loggedIn } = useUserSession() +const recordSignup = useSignupEvents() +const signupEntry = computed(() => userCode.value ? 'device' as const : 'loopback' as const) const status = ref<'loading' | 'ready' | 'done' | 'error'>('loading') const error = ref('') @@ -44,6 +46,7 @@ async function initAuthorize() { return try { + recordSignup({ stage: 'cli', outcome: 'viewed', entry: signupEntry.value }) if (userCode.value) { device.value = await apiFetch(lookupUrl, { query: { user_code: userCode.value }, @@ -65,9 +68,11 @@ async function initAuthorize() { v: version.value, }, }) + recordSignup({ stage: 'cli', outcome: 'authorized', entry: 'loopback' }) window.location.replace(response.redirect) } catch (err) { + recordSignup({ stage: 'cli', outcome: 'failed', entry: signupEntry.value }) status.value = 'error' error.value = err instanceof Error ? err.message : 'Authorization failed' } @@ -84,8 +89,10 @@ async function authorizeDevice() { body: { user_code: device.value.user_code }, }) status.value = 'done' + recordSignup({ stage: 'cli', outcome: 'authorized', entry: 'device' }) } catch (err) { + recordSignup({ stage: 'cli', outcome: 'failed', entry: 'device' }) status.value = 'error' error.value = err instanceof Error ? err.message : 'Device authorization failed' } diff --git a/checks/_helpers/collectors.mjs b/checks/_helpers/collectors.mjs index b6702243..38a8daa9 100644 --- a/checks/_helpers/collectors.mjs +++ b/checks/_helpers/collectors.mjs @@ -3,6 +3,7 @@ import { readdir, readFile } from 'node:fs/promises' import { basename, join } from 'node:path' import { readBoundedResponseText, runCheckCommand } from '@harlan-zw/nuxt-checkin/external' import { ANALYTICS_ACCOUNT_TAG, buildCopyQuery, buildCopyTotalsQuery, buildWorkersQuery, collectWorkflowRuns, parseWorkflowName, runListArgs, summarizeCopies, summarizeWorkflowRuns } from './observability.mjs' +import { buildSignupQuery, summarizeSignupEvents } from './signup-analytics.ts' const resources = Object.fromEntries(['github-auth', 'production-ref', 'git', 'deploy', 'ci', 'd1', 'workers', 'analytics'].map(key => [key, {}])) @@ -343,10 +344,11 @@ export function collectAnalytics(context) { throw new Error('Analytics Engine copy evidence is unavailable.') return rows } - const [totalsRows, topRows] = await Promise.all([ + const [totalsRows, topRows, signupRows] = await Promise.all([ readRows(buildCopyTotalsQuery(context.since.toISOString(), context.now.toISOString())), readRows(buildCopyQuery(context.since.toISOString(), context.now.toISOString())), + readRows(buildSignupQuery(context.since.toISOString(), context.now.toISOString())), ]) - return { commandCopies: summarizeCopies(totalsRows, topRows) } + return { commandCopies: summarizeCopies(totalsRows, topRows), signup: summarizeSignupEvents(signupRows) } }) } diff --git a/checks/_helpers/observability.mjs b/checks/_helpers/observability.mjs index 63656801..ccf99d6f 100644 --- a/checks/_helpers/observability.mjs +++ b/checks/_helpers/observability.mjs @@ -259,7 +259,7 @@ export function analyticsTimestamp(iso) { * so a busy day does not read as a quiet one. */ export function buildCopyQuery(sinceIso, nowIso) { - return `SELECT blob2 AS mode, blob3 AS kind, blob4 AS slug, sum(_sample_interval * double1) AS copies FROM ${COPY_DATASET} WHERE timestamp >= toDateTime('${analyticsTimestamp(sinceIso)}') AND timestamp < toDateTime('${analyticsTimestamp(nowIso)}') GROUP BY mode, kind, slug ORDER BY copies DESC LIMIT 50` + return `SELECT blob2 AS mode, blob3 AS kind, blob4 AS slug, sum(_sample_interval * double1) AS copies FROM ${COPY_DATASET} WHERE double1 > 0 AND timestamp >= toDateTime('${analyticsTimestamp(sinceIso)}') AND timestamp < toDateTime('${analyticsTimestamp(nowIso)}') GROUP BY mode, kind, slug ORDER BY copies DESC LIMIT 50` } /** @@ -268,7 +268,7 @@ export function buildCopyQuery(sinceIso, nowIso) { * still reports its full count. */ export function buildCopyTotalsQuery(sinceIso, nowIso) { - return `SELECT blob2 AS mode, sum(_sample_interval * double1) AS copies FROM ${COPY_DATASET} WHERE timestamp >= toDateTime('${analyticsTimestamp(sinceIso)}') AND timestamp < toDateTime('${analyticsTimestamp(nowIso)}') GROUP BY mode` + return `SELECT blob2 AS mode, sum(_sample_interval * double1) AS copies FROM ${COPY_DATASET} WHERE double1 > 0 AND timestamp >= toDateTime('${analyticsTimestamp(sinceIso)}') AND timestamp < toDateTime('${analyticsTimestamp(nowIso)}') GROUP BY mode` } /** diff --git a/checks/_helpers/signup-analytics.ts b/checks/_helpers/signup-analytics.ts new file mode 100644 index 00000000..50dcc7eb --- /dev/null +++ b/checks/_helpers/signup-analytics.ts @@ -0,0 +1,18 @@ +import { z } from 'zod' + +const rowSchema = z.object({ + stage: z.enum(['oauth', 'discover', 'email', 'cli']), + outcome: z.string(), + entry: z.string(), + choice: z.string(), + events: z.coerce.number().finite().nonnegative(), +}) + +export function buildSignupQuery(since: string, until: string): string { + const timestamp = (value: string) => new Date(value).toISOString().slice(0, 19).replace('T', ' ') + return `SELECT blob1 AS stage, blob6 AS outcome, blob4 AS entry, blob7 AS choice, sum(_sample_interval * double3) AS events FROM skilld_web_v1 WHERE blob3 = 'signup' AND timestamp >= toDateTime('${timestamp(since)}') AND timestamp < toDateTime('${timestamp(until)}') GROUP BY stage, outcome, entry, choice ORDER BY stage, entry, outcome, choice` +} + +export function summarizeSignupEvents(rows: unknown) { + return { unit: 'events' as const, stages: z.array(rowSchema).parse(rows) } +} diff --git a/layers/identity/app/composables/useSignupEvents.ts b/layers/identity/app/composables/useSignupEvents.ts new file mode 100644 index 00000000..92613b0c --- /dev/null +++ b/layers/identity/app/composables/useSignupEvents.ts @@ -0,0 +1,13 @@ +import type { SignupBrowserEvent } from '#shared/signup-analytics' + +export function useSignupEvents() { + const { loggedIn } = useUserSession() + return (body: SignupBrowserEvent): void => { + if (!loggedIn.value) + return + // Keep the action independent of analytics and survive same-tab redirects. + void $fetch('/api/events/signup', { method: 'POST', body, retry: false, keepalive: true }).catch(() => { + console.warn('[signup-analytics] Could not record signup event') + }) + } +} diff --git a/layers/identity/app/pages/me/index.vue b/layers/identity/app/pages/me/index.vue index 5f5040e9..78b3a4b9 100644 --- a/layers/identity/app/pages/me/index.vue +++ b/layers/identity/app/pages/me/index.vue @@ -8,6 +8,7 @@ import type { IdentitySubscriptionRef, } from '../../../shared/contracts/account' import type { StarsSyncResponse } from '../../utils/sync-starred-repos' +import { signupEmailChoice } from '#shared/signup-analytics' import { SKILL_VALIDATION_COPY } from '#shared/skill-validation-copy' import { accountDeletionConfirmed } from '../../../shared/contracts/account' import SkillgenRepositories from '../../components/_SkillgenRepositories.vue' @@ -15,6 +16,7 @@ import { identityAccountQueries, identityAccountQueryOptions } from '../../queri import { syncStarredRepos } from '../../utils/sync-starred-repos' definePageMeta({ layout: 'account', middleware: ['auth'] }) +const recordSignup = useSignupEvents() interface LikedSkill { owner: string @@ -147,7 +149,11 @@ const emailMissingAddress = computed(() => async function saveEmail() { if (emailMissingAddress.value || saveEmailMutation.pending.value) return - const saved = await saveEmailMutation.mutateSafe({ ...emailForm }) + const submitted = { ...emailForm } + const saved = await saveEmailMutation.mutateSafe(submitted) + recordSignup(saved._tag === 'ok' + ? { stage: 'email', outcome: 'saved', entry: 'dashboard', choice: signupEmailChoice(submitted.weekly_opt_in, submitted.email_opt_in) } + : { stage: 'email', outcome: 'failed', entry: 'dashboard' }) if (saved._tag === 'ok') showEmail.value = false } @@ -181,6 +187,12 @@ async function clearWelcomeQuery() { await navigateTo({ path: route.path, query: { ...route.query, welcome: undefined } }, { replace: true }) } onMounted(() => { + if (view.value === 'email') + recordSignup({ stage: 'email', outcome: 'viewed', entry: 'dashboard' }) + watch(view, (next) => { + if (next === 'email') + recordSignup({ stage: 'email', outcome: 'viewed', entry: 'dashboard' }) + }) if (route.query.welcome === '1') { toast.add({ title: 'You\'re all set', diff --git a/layers/identity/app/pages/onboarding/discover.vue b/layers/identity/app/pages/onboarding/discover.vue index 2f983969..25410001 100644 --- a/layers/identity/app/pages/onboarding/discover.vue +++ b/layers/identity/app/pages/onboarding/discover.vue @@ -5,6 +5,7 @@ import { syncStarredRepos } from '../../utils/sync-starred-repos' definePageMeta({ layout: 'auth', middleware: ['auth'] }) const { user } = useUserSession() +const recordSignup = useSignupEvents() interface Skill { name: string @@ -61,8 +62,10 @@ async function syncStars() { ) await refresh() preselectAll() + recordSignup({ stage: 'discover', outcome: 'imported', entry: 'onboarding' }) } catch (e) { + recordSignup({ stage: 'discover', outcome: 'import-failed', entry: 'onboarding' }) syncError.value = (e as { statusMessage?: string, message?: string }).statusMessage ?? (e as Error).message ?? 'Sync failed' @@ -73,6 +76,7 @@ async function syncStars() { } onMounted(() => { + recordSignup({ stage: 'discover', outcome: 'viewed', entry: 'onboarding' }) if (data.value && !data.value.syncedAt) { void syncStars() return @@ -88,6 +92,7 @@ async function watchSelected() { return actionFailed.clear('start watching those repos') if (!selected.value.size) { + recordSignup({ stage: 'discover', outcome: 'continued', entry: 'onboarding' }) await navigateTo('/onboarding/email') return } @@ -101,6 +106,7 @@ async function watchSelected() { body: { source: 'star-import', repos }, }).catch(actionFailed('start watching those repos')) submitting.value = false + recordSignup({ stage: 'discover', outcome: saved ? 'continued' : 'watch-failed', entry: 'onboarding' }) if (saved) await navigateTo('/onboarding/email') } @@ -163,7 +169,7 @@ useSeoMeta({ title: 'Discover skills', robots: 'noindex' }) - + skip ahead . @@ -222,7 +228,7 @@ useSeoMeta({ title: 'Discover skills', robots: 'noindex' })
- + Skip import type { IdentityEmailPatchBody, IdentityMutationResponse } from '../../../shared/contracts/account' +import { signupEmailChoice } from '#shared/signup-analytics' import { identityAccountQueries, identityAccountQueryOptions } from '../../queries/account' definePageMeta({ layout: 'auth', middleware: ['auth'] }) const { data: me, error: accountError, status: accountStatus, refresh: retryAccount } = await useNuxtRpcQuery(identityAccountQueries.me(), identityAccountQueryOptions) const { fetchSession } = useAuth() +const recordSignup = useSignupEvents() +onMounted(() => recordSignup({ stage: 'email', outcome: 'viewed', entry: 'onboarding' })) const email = ref(me.value?.digest_email || me.value?.email || '') // A stored address records a deliberate choice, including a previous opt-out. @@ -51,12 +54,18 @@ async function finish() { email_opt_in: optIn.value, weekly_opt_in: weeklyOptIn.value, }) - if (saved._tag === 'err') + if (saved._tag === 'err') { + recordSignup({ stage: 'email', outcome: 'failed', entry: 'onboarding' }) return + } const onboarded = await finishOnboardingMutation.mutateSafe() - if (onboarded._tag === 'err') + if (onboarded._tag === 'err') { + recordSignup({ stage: 'email', outcome: 'completion-failed', entry: 'onboarding' }) return + } + + recordSignup({ stage: 'email', outcome: 'saved', entry: 'onboarding', choice: signupEmailChoice(weeklyOptIn.value, optIn.value) }) await fetchSession() await navigateTo('/me?welcome=1') diff --git a/layers/identity/server/api/cli/device/poll.post.ts b/layers/identity/server/api/cli/device/poll.post.ts index 6c97808a..76b7cd2d 100644 --- a/layers/identity/server/api/cli/device/poll.post.ts +++ b/layers/identity/server/api/cli/device/poll.post.ts @@ -1,6 +1,7 @@ import { DevicePollInputSchema, DevicePollResponseSchema } from 'skilld-protocol/wire' import { defineApiHandler } from '#shared/server/handler' import { issueSession, presentTokenResponse } from '../../../utils/cli-tokens' +import { emitSignupEvent } from '../../../utils/signup-analytics' import { getUserById } from '../../../utils/users' interface DeviceRow { @@ -49,6 +50,8 @@ export default defineApiHandler({ `UPDATE cli_device_sessions SET status = 'expired' WHERE device_code = ?1`, ).bind(row.device_code).run() - return { status: 'authorized' as const, tokens: presentTokenResponse(tokens, user.login) } + const response = { status: 'authorized' as const, tokens: presentTokenResponse(tokens, user.login) } + emitSignupEvent(event, { stage: 'cli', outcome: 'connected', entry: 'device' }) + return response }, }) diff --git a/layers/identity/server/api/cli/oauth/token.post.ts b/layers/identity/server/api/cli/oauth/token.post.ts index fff31e97..7b1e76b6 100644 --- a/layers/identity/server/api/cli/oauth/token.post.ts +++ b/layers/identity/server/api/cli/oauth/token.post.ts @@ -2,6 +2,7 @@ import { OauthTokenInputSchema, TokenResponseSchema } from 'skilld-protocol/wire import { z } from 'zod' import { defineApiHandler } from '#shared/server/handler' import { issueSession, presentTokenResponse, sha256Base64Url } from '../../../utils/cli-tokens' +import { emitSignupEvent } from '../../../utils/signup-analytics' import { getUserById } from '../../../utils/users' interface AuthCodeRow { @@ -55,6 +56,8 @@ export default defineApiHandler({ deviceLabel: body.device_label, }) - return presentTokenResponse(session, user.login) + const response = presentTokenResponse(session, user.login) + emitSignupEvent(event, { stage: 'cli', outcome: 'connected', entry: 'loopback' }) + return response }, }) diff --git a/layers/identity/server/api/events/signup.post.ts b/layers/identity/server/api/events/signup.post.ts new file mode 100644 index 00000000..dd17c4da --- /dev/null +++ b/layers/identity/server/api/events/signup.post.ts @@ -0,0 +1,13 @@ +import { defineApiHandler } from '#shared/server/handler' +import { SignupBrowserEvent } from '#shared/signup-analytics' +import { identityMutationResponseSchema } from '../../../shared/contracts/account' +import { authenticated } from '../../policies/authenticated' +import { emitSignupEvent } from '../../utils/signup-analytics' + +export default defineApiHandler({ + schema: SignupBrowserEvent, + policy: [authenticated], + response: identityMutationResponseSchema, + handler: ({ event, body }) => emitSignupEvent(event, body), + presenter: () => ({ ok: true as const }), +}) diff --git a/layers/identity/server/routes/auth/github.get.ts b/layers/identity/server/routes/auth/github.get.ts index 6847f6ca..debaa872 100644 --- a/layers/identity/server/routes/auth/github.get.ts +++ b/layers/identity/server/routes/auth/github.get.ts @@ -2,9 +2,11 @@ import type { H3Event } from 'h3' import { useSession } from 'h3' import { z } from 'zod' import { parseReturnTo } from '#shared/return-to' +import { signupEntry } from '#shared/signup-analytics' import { sendEmailWithEnv, signUnsubToken } from '../../utils/email' import { fetchVerifiedPrimaryEmail } from '../../utils/github-emails' import { ownedRepoScanWarning, scanOwnedRepos } from '../../utils/scan-owned-repos' +import { emitSignupEvent } from '../../utils/signup-analytics' import { sendSkillValidationSummary } from '../../utils/skill-validation-email' import { upsertUserFromGithub } from '../../utils/users' import { handleWatchAction } from '../../utils/watch-actions' @@ -132,6 +134,8 @@ const githubHandler = defineOAuthGitHubEventHandler({ await intent.clear() + emitSignupEvent(event, { stage: 'oauth', outcome: 'succeeded', entry: signupEntry(action, returnTo) }) + if (returnTo) return sendRedirect(event, returnTo) @@ -148,6 +152,7 @@ async function loginFailed(event: H3Event) { const intent = await loginIntentSession(event) const params = new URLSearchParams({ error: 'oauth' }) const returnTo = parseReturnTo(intent.data.returnTo, '') + emitSignupEvent(event, { stage: 'oauth', outcome: 'failed', entry: signupEntry(intent.data.action ?? '', returnTo) }) if (returnTo) params.set('return_to', returnTo) if (typeof intent.data.action === 'string' && intent.data.action) @@ -174,7 +179,9 @@ export default defineEventHandler(async (event) => { const action = ['like-skill', 'watch-skill', 'watch-collection'].includes(String(query.action)) ? String(query.action) : '' - await intent.update({ returnTo: parseReturnTo(query.return_to, ''), action }) + const returnTo = parseReturnTo(query.return_to, '') + await intent.update({ returnTo, action }) + emitSignupEvent(event, { stage: 'oauth', outcome: 'started', entry: signupEntry(action, returnTo) }) } // Provider network failures can throw before the OAuth error callback runs. return githubHandler(event).catch(() => loginFailed(event)) diff --git a/layers/identity/server/utils/signup-analytics.ts b/layers/identity/server/utils/signup-analytics.ts new file mode 100644 index 00000000..c5ad848f --- /dev/null +++ b/layers/identity/server/utils/signup-analytics.ts @@ -0,0 +1,25 @@ +import type { H3Event } from 'h3' +import type { SignupEvent } from '#shared/signup-analytics' +import { signupDataPoint } from '#shared/signup-analytics' + +export function recordSignupEvent( + dataset: Pick | undefined, + point: SignupEvent, + onFailure: (cause: unknown) => void, +): void { + // Analytics failure must never change sign-in or an account mutation. + try { + if (!dataset) + throw new Error('Signup analytics binding missing') + dataset.writeDataPoint(signupDataPoint(point)) + } + catch (cause) { + onFailure(cause) + } +} + +export function emitSignupEvent(event: H3Event, point: SignupEvent): void { + recordSignupEvent(event.context.platform?.env?.SKILLD_WEB_ANALYTICS, point, () => { + console.warn('[signup-analytics] Could not record signup event') + }) +} diff --git a/layers/marketing/content/pages/privacy.md b/layers/marketing/content/pages/privacy.md index 271c5d42..78c9f423 100644 --- a/layers/marketing/content/pages/privacy.md +++ b/layers/marketing/content/pages/privacy.md @@ -1,14 +1,14 @@ --- title: Privacy description: What skilld.dev stores, why it needs the data, how long it keeps it, and how you delete your account. -label: Last updated 8 October 2026 -updatedAt: 2026-10-08 +label: Last updated 11 October 2026 +updatedAt: 2026-10-11 --- ## In short - You can browse skilld.dev and run Skills without an account. -- skilld.dev counts page views, demo views, command copies, and demo link copies. The counts hold no name, IP address, or cookie. It loads no advertising scripts. +- skilld.dev counts page views, demo views, command copies, demo link copies, and sign-in steps. The counts hold no name, IP address, or cookie. It loads no advertising scripts. - An account is optional. You sign in with [GitHub](https://github.com) to like Skills, watch Repositories, get email, or connect the skilld CLI. - You can delete your account from your dashboard at any time. - The skilld CLI sends no telemetry. @@ -50,6 +50,8 @@ Demo counts also record the displayed demo, the surface, and whether you copied The skilld CLI sends no telemetry. skilld.dev accepts an anonymous run count from a CLI at the same address, under the same rules, and drops any account ID the request carries. +Sign-in counts record a fixed entry category, a setup step, and its outcome. Steps cover GitHub sign-in, star import, email choices, and browser authorization of the CLI. Email saves count which email types you chose, without storing your address. Counts contain no account ID, GitHub login, authorization code, return page, or query string. They use the same 90-day Analytics Engine storage. Repeat visits count again, so these counts measure attempts, not individual users. A blocked request or a closed tab can leave a count missing. + ### Cookies skilld.dev sets these cookies and no others: diff --git a/shared/checkin-external.ts b/shared/checkin-external.ts index 4d116bf0..efedf2fd 100644 --- a/shared/checkin-external.ts +++ b/shared/checkin-external.ts @@ -17,7 +17,7 @@ Interpret this site evidence: - \`skilld.baseline\` result evidence records the window against the daily cadence. \`stale\` means the saved baseline is old; report the \`gapHours\` figure and compare rates per day, never as overnight step changes. The shared CLI rejects invalid or future state before collecting evidence. Repair the state or supply an explicit \`--since\` boundary. - Use failed-job \`first_failed_at\` and \`last_failed_at\`, plus \`d1.recentJobBatches\` and \`d1.registryMaintenance\`, to distinguish an active incident from a recovered burst. Do not call a window clean because the latest batch passed, or active because an older batch failed. - \`d1.inventory.broken_repos\` is known cumulative inventory. Review \`d1.pipeline.newly_broken_repos_total\`, but gate health on \`d1.pipeline.newly_broken_repos_impacted\`. A source removal is impacting when it still backs a skill or appears in a star, subscription, or collection. - - \`skilld.analytics\` holds command copies from Analytics Engine, which is the activation signal. Report \`commandCopies.run\` and \`commandCopies.install\` apart: a run copy reads a Skill once, an install copy keeps it. Analytics Engine samples, so treat a small day as a range, not an exact count. + - \`skilld.analytics\` holds command copies from Analytics Engine, which is the activation signal. Report \`commandCopies.run\` and \`commandCopies.install\` apart: a run copy reads a Skill once, an install copy keeps it. Analytics Engine samples, so treat a small day as a range, not an exact count. \`signup.stages\` counts sign-in and setup events by entry and outcome. These are attempts, not unique users or a joined conversion funnel. Report failures and email opt-outs separately. Missing events cannot prove abandonment. - \`skilld.signing-key\` Warn or Fail means Artifact signing ends soon. Quote \`signingEndsAt\` and \`daysLeft\`, and point to \`docs/runbooks/signing-key-rotation.md\`. After that date every \`skilld run\` fails. - AI cost is only the recorded batch estimate. Unmeasured services are unknown, not $0.`, }, diff --git a/shared/signup-analytics.ts b/shared/signup-analytics.ts new file mode 100644 index 00000000..bfd1b3c1 --- /dev/null +++ b/shared/signup-analytics.ts @@ -0,0 +1,38 @@ +import type { AnalyticsDataPoint } from './analytics' +import { z } from 'zod' + +const browserStages = [ + z.object({ stage: z.literal('discover'), entry: z.literal('onboarding'), outcome: z.enum(['viewed', 'imported', 'import-failed', 'continued', 'skipped', 'watch-failed']) }).strict(), + z.object({ stage: z.literal('email'), entry: z.enum(['onboarding', 'dashboard']), outcome: z.enum(['viewed', 'failed', 'completion-failed']) }).strict(), + z.object({ stage: z.literal('email'), entry: z.enum(['onboarding', 'dashboard']), outcome: z.literal('saved'), choice: z.enum(['none', 'weekly', 'monthly', 'both']) }).strict(), + z.object({ stage: z.literal('cli'), entry: z.enum(['device', 'loopback']), outcome: z.enum(['viewed', 'authorized', 'failed']) }).strict(), +] as const + +// Fixed enums reject paths, codes, addresses, account IDs, and free text. +export const SignupBrowserEvent = z.union(browserStages) +export type SignupBrowserEvent = z.infer +export type SignupEntry = 'direct' | 'return-page' | 'cli' | 'like-skill' | 'watch-skill' | 'watch-collection' +export type SignupEvent = SignupBrowserEvent + | { stage: 'oauth', entry: SignupEntry, outcome: 'started' | 'succeeded' | 'failed' } + | { stage: 'cli', entry: 'device' | 'loopback', outcome: 'connected' } + +export function signupEntry(action: string, returnTo: string): SignupEntry { + if (returnTo.split('?')[0] === '/cli/authorize') + return 'cli' + if (action === 'like-skill' || action === 'watch-skill' || action === 'watch-collection') + return action + return returnTo ? 'return-page' : 'direct' +} + +export function signupEmailChoice(weekly: boolean, monthly: boolean): 'none' | 'weekly' | 'monthly' | 'both' { + return weekly ? (monthly ? 'both' : 'weekly') : (monthly ? 'monthly' : 'none') +} + +/** Double 1 remains command copies; double 2 remains demo events. */ +export function signupDataPoint(event: SignupEvent): AnalyticsDataPoint { + return { + blobs: [event.stage, '', 'signup', event.entry, '', event.outcome, 'choice' in event ? event.choice : ''], + doubles: [0, 0, 1], + indexes: [`signup:${event.stage}`], + } +} diff --git a/test/unit/email-form-submit.nuxt.test.ts b/test/unit/email-form-submit.nuxt.test.ts index 35b04589..df5c10e8 100644 --- a/test/unit/email-form-submit.nuxt.test.ts +++ b/test/unit/email-form-submit.nuxt.test.ts @@ -4,7 +4,7 @@ import { ref } from 'vue' import Account from '../../layers/identity/app/pages/me/index.vue' import OnboardingEmail from '../../layers/identity/app/pages/onboarding/email.vue' -const mocks = vi.hoisted(() => ({ execute: vi.fn(), query: vi.fn(), navigate: vi.fn(), session: vi.fn() })) +const mocks = vi.hoisted(() => ({ execute: vi.fn(), query: vi.fn(), navigate: vi.fn(), session: vi.fn(), signup: vi.fn() })) const account = ref({ id: 1, login: 'email-review', @@ -30,6 +30,7 @@ mockNuxtImport('useFetch', () => () => ({ data: ref({ items: [] }), status: ref( mockNuxtImport('useNuxtRpc', () => () => ({ execute: mocks.execute, query: mocks.query })) mockNuxtImport('useAuth', () => () => ({ fetchSession: mocks.session })) mockNuxtImport('navigateTo', () => mocks.navigate) +mockNuxtImport('useSignupEvents', () => () => mocks.signup) mockNuxtImport('useActionFailure', () => () => Object.assign(() => vi.fn(), { clear: vi.fn() })) mockNuxtImport('useNuxtMutation', () => (options: { mutation: (body?: unknown) => Promise }) => ({ pending: ref(false), @@ -95,6 +96,7 @@ describe.each(screens)('$name email submission', ({ component, form: selector }) }) if (component === OnboardingEmail) expect(mocks.navigate).toHaveBeenCalledWith('/me?welcome=1') + expect(mocks.signup).toHaveBeenCalledWith({ stage: 'email', outcome: 'saved', entry: component === Account ? 'dashboard' : 'onboarding', choice: 'none' }) } finally { wrapper.unmount() diff --git a/test/unit/external-checkin-collectors.test.ts b/test/unit/external-checkin-collectors.test.ts index 9edca527..0d68aebb 100644 --- a/test/unit/external-checkin-collectors.test.ts +++ b/test/unit/external-checkin-collectors.test.ts @@ -132,14 +132,16 @@ it('totals command copies from an untruncated rollup, not the capped top page', const topRows = Array.from({ length: 50 }, (_, index) => ({ mode: 'run', kind: 'skill', slug: `owner/skill-${index}`, copies: 1 })) const totalsRows = [{ mode: 'run', copies: 50 }, { mode: 'install', copies: 10 }] vi.stubGlobal('fetch', vi.fn(async (_url: string | URL, init?: { body?: string }) => { - return Response.json({ data: init?.body?.includes('LIMIT 50') ? topRows : totalsRows }) + return Response.json({ data: init?.body?.includes('blob3 = \'signup\'') + ? [{ stage: 'email', outcome: 'failed', entry: 'onboarding', choice: '', events: '2' }] + : init?.body?.includes('LIMIT 50') ? topRows : totalsRows }) })) const { report } = await runExternalChecks([analyticsCheck], { required: [analyticsCheck.id] }, { env: { CLOUDFLARE_USAGE_TOKEN: 'test-token' } }) expect(report.results.find(check => check.id === 'skilld.analytics')?.result).toMatchObject({ _tag: 'Pass', - evidence: { commandCopies: { total: 60, run: 50, install: 10 } }, + evidence: { commandCopies: { total: 60, run: 50, install: 10 }, signup: { unit: 'events', stages: [{ stage: 'email', outcome: 'failed', entry: 'onboarding', choice: '', events: 2 }] } }, }) }) diff --git a/test/unit/github-login-intent.nuxt.test.ts b/test/unit/github-login-intent.nuxt.test.ts index 6560b32d..5a7ff2f4 100644 --- a/test/unit/github-login-intent.nuxt.test.ts +++ b/test/unit/github-login-intent.nuxt.test.ts @@ -8,6 +8,7 @@ const mocks = vi.hoisted(() => ({ scan: vi.fn(), email: vi.fn(), establish: vi.fn(), + analytics: vi.fn(), })) vi.mock('h3', async importOriginal => ({ ...await importOriginal(), useSession: mocks.session })) vi.mock('../../layers/identity/server/utils/users', () => ({ upsertUserFromGithub: mocks.upsert })) @@ -25,7 +26,7 @@ describe('gitHub browser login round trip', () => { const clearIntent = vi.fn(async () => { intent = {} }) - const event = { context: { platform: { env: {} } } } as unknown as H3Event + const event = { context: { platform: { env: { SKILLD_WEB_ANALYTICS: { writeDataPoint: mocks.analytics } } } } } as unknown as H3Event beforeEach(async () => { vi.resetModules() @@ -88,6 +89,10 @@ describe('gitHub browser login round trip', () => { expect(target.searchParams.get('action')).toBe('watch-collection') expect(target.searchParams.get('error')).toBe('oauth') expect(mocks.replay).not.toHaveBeenCalled() + expect(mocks.analytics.mock.calls.map(([point]) => point)).toEqual([ + { blobs: ['oauth', '', 'signup', 'watch-collection', '', 'started', ''], doubles: [0, 0, 1], indexes: ['signup:oauth'] }, + { blobs: ['oauth', '', 'signup', 'watch-collection', '', 'failed', ''], doubles: [0, 0, 1], indexes: ['signup:oauth'] }, + ]) }) it('returns a retry link when the provider request throws', async () => { diff --git a/test/unit/signup-analytics.test.ts b/test/unit/signup-analytics.test.ts new file mode 100644 index 00000000..a2a38965 --- /dev/null +++ b/test/unit/signup-analytics.test.ts @@ -0,0 +1,54 @@ +import { summarizeSignupEvents } from '../../checks/_helpers/signup-analytics' +import { recordSignupEvent } from '../../layers/identity/server/utils/signup-analytics' +import { SignupBrowserEvent, signupDataPoint, signupEmailChoice, signupEntry } from '../../shared/signup-analytics' + +it('classifies CLI sign-in without storing its authorization parameters', () => { + expect(signupEntry('', '/cli/authorize?user_code=SECRET')).toBe('cli') + expect(signupEntry('like-skill', '/gh/owner/repo/skill')).toBe('like-skill') + expect(signupEntry('', '/me')).toBe('return-page') + expect(signupEntry('', '')).toBe('direct') +}) + +it('keeps signup events out of command and demo totals', () => { + expect(signupDataPoint({ stage: 'email', outcome: 'saved', entry: 'onboarding', choice: 'none' })).toEqual({ + blobs: ['email', '', 'signup', 'onboarding', '', 'saved', 'none'], + doubles: [0, 0, 1], + indexes: ['signup:email'], + }) +}) + +it.each([ + [false, false, 'none'], + [true, false, 'weekly'], + [false, true, 'monthly'], + [true, true, 'both'], +] as const)('names chosen email types without the address', (weekly, monthly, choice) => { + expect(signupEmailChoice(weekly, monthly)).toBe(choice) +}) + +it('rejects identifying fields and browser claims about OAuth', () => { + expect(SignupBrowserEvent.safeParse({ stage: 'discover', outcome: 'viewed', entry: 'onboarding', login: 'alice' }).success).toBe(false) + expect(SignupBrowserEvent.safeParse({ stage: 'oauth', outcome: 'succeeded', entry: 'direct' }).success).toBe(false) + expect(SignupBrowserEvent.safeParse({ stage: 'email', outcome: 'saved', entry: 'onboarding', choice: 'none' }).success).toBe(true) +}) + +it('reports a missing analytics binding without failing the user action', () => { + const failed = vi.fn() + recordSignupEvent(undefined, { stage: 'oauth', outcome: 'started', entry: 'direct' }, failed) + expect(failed).toHaveBeenCalledWith(expect.any(Error)) +}) + +it('reports a rejected analytics write without failing the user action', () => { + const failed = vi.fn() + recordSignupEvent({ writeDataPoint: () => { + throw new Error('unavailable') + } }, { stage: 'oauth', outcome: 'started', entry: 'direct' }, failed) + expect(failed).toHaveBeenCalledWith(expect.objectContaining({ message: 'unavailable' })) +}) + +it('reports sampled attempt counts without treating them as unique users', () => { + expect(summarizeSignupEvents([{ stage: 'email', outcome: 'failed', entry: 'onboarding', choice: '', events: '4' }])).toEqual({ + unit: 'events', + stages: [{ stage: 'email', outcome: 'failed', entry: 'onboarding', choice: '', events: 4 }], + }) +}) diff --git a/test/unit/signup-event-endpoint.test.ts b/test/unit/signup-event-endpoint.test.ts new file mode 100644 index 00000000..bdc4e0b4 --- /dev/null +++ b/test/unit/signup-event-endpoint.test.ts @@ -0,0 +1,35 @@ +import type { EventHandler } from 'h3' + +const mocks = vi.hoisted(() => ({ session: vi.fn(), write: vi.fn() })) +vi.mock('../../shared/server/session-access', () => ({ readUserSession: mocks.session })) + +async function request(body: unknown, signedIn: boolean) { + mocks.session.mockResolvedValue(signedIn ? { user: { id: 42, login: 'private-login' } } : null) + vi.stubGlobal('defineEventHandler', (handler: EventHandler) => handler) + vi.stubGlobal('createError', (input: { statusCode: number }) => Object.assign(new Error('rejected'), input)) + vi.stubGlobal('readBody', async () => body) + const handler = (await import('../../layers/identity/server/api/events/signup.post')).default as unknown as (event: unknown) => Promise + return handler({ + method: 'POST', + node: { req: { headers: {} } }, + context: { platform: { env: { SKILLD_WEB_ANALYTICS: { writeDataPoint: mocks.write } } } }, + }) +} + +beforeEach(() => vi.clearAllMocks()) +afterEach(() => vi.unstubAllGlobals()) + +it('writes only fixed event labels from an authenticated request', async () => { + await expect(request({ stage: 'discover', entry: 'onboarding', outcome: 'viewed' }, true)).resolves.toEqual({ ok: true }) + expect(mocks.write).toHaveBeenCalledWith({ blobs: ['discover', '', 'signup', 'onboarding', '', 'viewed', ''], doubles: [0, 0, 1], indexes: ['signup:discover'] }) +}) + +it('refuses counts from a signed-out request', async () => { + await expect(request({ stage: 'discover', entry: 'onboarding', outcome: 'viewed' }, false)).rejects.toMatchObject({ statusCode: 403 }) + expect(mocks.write).not.toHaveBeenCalled() +}) + +it('refuses an event carrying an account identifier', async () => { + await expect(request({ stage: 'email', entry: 'onboarding', outcome: 'saved', choice: 'none', userId: 42 }, true)).rejects.toMatchObject({ statusCode: 400 }) + expect(mocks.write).not.toHaveBeenCalled() +})