From 2efe4435039635819033cc989514e58f4f13b6c6 Mon Sep 17 00:00:00 2001 From: yumc-afk Date: Sat, 21 Nov 2026 21:05:59 +0800 Subject: [PATCH] feat(export): add chunked, resumable R2 multipart database dumps with DO alarm breathing intervals (#59) --- src/do.test.ts | 452 ++++++++++++ src/do.ts | 291 ++++++-- src/export/dump.test.ts | 1250 +++++++++++++++++++++++++++++++- src/export/dump.ts | 1409 +++++++++++++++++++++++++++++++++++-- src/handler.ts | 40 +- src/index.ts | 2 + src/types.ts | 1 + worker-configuration.d.ts | 1 + wrangler.toml | 6 + 9 files changed, 3360 insertions(+), 92 deletions(-) diff --git a/src/do.test.ts b/src/do.test.ts index 272c4e9..ae620f7 100644 --- a/src/do.test.ts +++ b/src/do.test.ts @@ -145,4 +145,456 @@ describe('StarbaseDBDurableObject Tests', () => { instance.executeQuery({ sql: 'INVALID QUERY' }) ).rejects.toThrow('Query failed') }) + + it('should fail missing-bucket dump job via shared path (sending webhook) and still schedule next queued job', async () => { + const setAlarmMock = vi.fn() + const stateWithAlarm = { + storage: { + ...mockStorage, + setAlarm: setAlarmMock, + getAlarm: vi.fn().mockResolvedValue(null), + deleteAlarm: vi.fn(), + }, + getTags: vi.fn().mockReturnValue([]), + } as any + + const doInstance = new StarbaseDBDurableObject( + stateWithAlarm, + {} as any + ) + const jobs = new Map([ + [ + 'job-no-r2-1', + { + id: 'job-no-r2-1', + status: 'in_progress', + r2_key: 'dump_1.sql', + upload_id: null, + tables_json: '[]', + table_index: 0, + row_offset: 0, + last_rowid: null, + parts_json: '[]', + pending_staging_key: null, + chunk_size: 500, + work_budget_ms: 5000, + breathing_interval_ms: 5000, + rows_exported: 0, + bytes_written: 0, + callback_url: 'https://hooks.example.com/fail', + callback_status: null, + error: null, + created_at: '2024-01-01T17:00:00.000Z', + updated_at: '2024-01-01T17:00:00.000Z', + completed_at: null, + }, + ], + [ + 'job-no-r2-2', + { + id: 'job-no-r2-2', + status: 'in_progress', + r2_key: 'dump_2.sql', + upload_id: null, + tables_json: '[]', + table_index: 0, + row_offset: 0, + last_rowid: null, + parts_json: '[]', + pending_staging_key: null, + chunk_size: 500, + work_budget_ms: 5000, + breathing_interval_ms: 5000, + rows_exported: 0, + bytes_written: 0, + callback_url: null, + callback_status: null, + error: null, + created_at: '2024-01-01T17:00:01.000Z', + updated_at: '2024-01-01T17:00:01.000Z', + completed_at: null, + }, + ], + ]) + const slots = new Map() + + vi.spyOn(doInstance, 'executeQuery').mockImplementation( + async (opts: { sql: string; params?: unknown[] }) => { + const sql = opts.sql.replace(/\s+/g, ' ').trim() + if (sql.includes('FROM tmp_alarm_slots')) { + return Array.from(slots.entries()).map( + ([slot, scheduled_at]) => ({ slot, scheduled_at }) + ) as any + } + if (sql.startsWith('INSERT OR REPLACE INTO tmp_alarm_slots')) { + slots.set( + String(opts.params?.[0]), + Number(opts.params?.[1]) + ) + return [] as any + } + if (sql.startsWith('DELETE FROM tmp_alarm_slots')) { + slots.delete(String(opts.params?.[0])) + return [] as any + } + if (sql.includes('SELECT * FROM tmp_dump_jobs')) { + const active = Array.from(jobs.values()).filter( + (j) => j.status === 'in_progress' + ) + return active.slice(0, 1) as any + } + if (sql.includes('SELECT id FROM tmp_dump_jobs')) { + const active = Array.from(jobs.values()).filter( + (j) => j.status === 'in_progress' + ) + return active.slice(0, 1).map((j) => ({ id: j.id })) as any + } + if (sql.startsWith('INSERT OR REPLACE INTO tmp_dump_jobs')) { + const cols = sql + .match(/\(([^)]+)\)/)![1] + .split(',') + .map((c) => c.trim()) + const rec: Record = {} + cols.forEach((c, i) => (rec[c] = opts.params?.[i] ?? null)) + jobs.set(String(rec.id), rec) + return [] as any + } + if (sql.includes('FROM tmp_cron_tasks')) { + throw new Error('no such table: tmp_cron_tasks') + } + return [] as any + } + ) + + const fetchSpy = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue(new Response('ok', { status: 200 })) + + await doInstance.alarm() + + expect(jobs.get('job-no-r2-1').status).toBe('failed') + expect(jobs.get('job-no-r2-1').callback_status).toBe('delivered') + expect(fetchSpy).toHaveBeenCalledTimes(1) + // Second queued job MUST get a follow-up wake-up alarm + expect(slots.has('dump')).toBe(true) + expect(setAlarmMock).toHaveBeenCalled() + fetchSpy.mockRestore() + }) + + it('should continue an active dump job in alarm() when DUMP_BUCKET is bound', async () => { + const setAlarmMock = vi.fn() + const stateWithAlarm = { + storage: { + ...mockStorage, + setAlarm: setAlarmMock, + getAlarm: vi.fn().mockResolvedValue(null), + deleteAlarm: vi.fn(), + }, + getTags: vi.fn().mockReturnValue([]), + } as any + + const completedParts: R2UploadedPart[] = [] + const mockBucket = { + resumeMultipartUpload: vi.fn(() => ({ + uploadPart: vi.fn(async (partNumber: number) => { + const part = { partNumber, etag: `e-${partNumber}` } + completedParts.push(part) + return part + }), + complete: vi.fn(async () => ({ + key: 'dump_20240101-170000.sql', + })), + abort: vi.fn(), + })), + get: vi.fn(async () => ({ + arrayBuffer: async () => + new TextEncoder().encode('SQLite format 3\0').buffer, + })), + put: vi.fn(), + delete: vi.fn(async () => {}), + } as unknown as R2Bucket + + const doInstance = new StarbaseDBDurableObject(stateWithAlarm, { + DUMP_BUCKET: mockBucket, + } as any) + + let updatedStatus = '' + vi.spyOn(doInstance, 'executeQuery').mockImplementation( + async (opts: { sql: string; params?: unknown[] }) => { + if (opts.sql.includes('SELECT * FROM tmp_dump_jobs')) { + return [ + { + id: 'job-alarm-1', + status: 'in_progress', + r2_key: 'dump_20240101-170000.sql', + upload_id: 'u-1', + tables_json: JSON.stringify(['users']), + table_index: 0, + row_offset: 0, + last_rowid: null, + parts_json: '[]', + pending_staging_key: + 'dump_20240101-170000.sql.pending.16', + chunk_size: 500, + work_budget_ms: 5000, + breathing_interval_ms: 5000, + rows_exported: 0, + bytes_written: 16, + callback_url: null, + callback_status: null, + error: null, + created_at: '2024-01-01T17:00:00.000Z', + updated_at: '2024-01-01T17:00:00.000Z', + completed_at: null, + }, + ] as any + } + if (opts.sql.includes('SELECT id FROM tmp_dump_jobs')) { + return [] as any + } + if (opts.sql.includes('SELECT sql FROM sqlite_master')) { + return [{ sql: 'CREATE TABLE users (id INTEGER)' }] as any + } + if (opts.sql.includes('FROM "users"')) { + return [ + { __sb_rid: 1, id: 1 }, + { __sb_rid: 2, id: 2 }, + ] as any + } + if ( + opts.sql.startsWith('INSERT OR REPLACE INTO tmp_dump_jobs') + ) { + updatedStatus = String(opts.params?.[1] ?? '') + return [] as any + } + if (opts.sql.includes('FROM tmp_cron_tasks')) { + return [] as any + } + return [] as any + } + ) + + await doInstance.alarm() + + expect(updatedStatus).toBe('completed') + expect(completedParts.length).toBe(1) + }) + + it('should preserve bidirectional alarm coexistence without starvation, early cron fire, or redundant SQL writes', async () => { + let physicalAlarm: number | null = null + const setAlarmSpy = vi.fn(async (ts: number) => { + physicalAlarm = ts + }) + const deleteAlarmSpy = vi.fn(async () => { + physicalAlarm = null + }) + const stateWithAlarm = { + storage: { + ...mockStorage, + setAlarm: setAlarmSpy, + getAlarm: vi.fn(async () => physicalAlarm), + deleteAlarm: deleteAlarmSpy, + }, + getTags: vi.fn().mockReturnValue([]), + } as any + + const mockBucket = { + resumeMultipartUpload: vi.fn(() => ({ + uploadPart: vi.fn(async (partNumber: number) => ({ + partNumber, + etag: `e-${partNumber}`, + })), + complete: vi.fn(async () => ({ + key: 'dump_20240101-170000.sql', + })), + abort: vi.fn(), + })), + get: vi.fn(async () => ({ + arrayBuffer: async () => + new TextEncoder().encode('SQLite format 3\0').buffer, + })), + put: vi.fn(), + delete: vi.fn(async () => {}), + } as unknown as R2Bucket + + const doInstance = new StarbaseDBDurableObject(stateWithAlarm, { + CLIENT_AUTHORIZATION_TOKEN: 'client-token', + DUMP_BUCKET: mockBucket, + } as any) + + const alarmSlots = new Map() + let slotWriteCount = 0 + let dumpJob: any = { + id: 'job-coexist-1', + status: 'in_progress', + r2_key: 'dump_20240101-170000.sql', + upload_id: 'u-coexist', + tables_json: JSON.stringify(['users']), + table_index: 0, + row_offset: 0, + last_rowid: null, + parts_json: '[]', + pending_staging_key: 'dump_20240101-170000.sql.pending.16', + chunk_size: 500, + work_budget_ms: 5000, + breathing_interval_ms: 5000, + rows_exported: 0, + bytes_written: 16, + callback_url: null, + callback_status: null, + error: null, + created_at: '2024-01-01T17:00:00.000Z', + updated_at: '2024-01-01T17:00:00.000Z', + completed_at: null, + } + const cronTasks = [ + { + name: 'nightly_sync', + cron_tab: '0 * * * *', + payload: '{}', + callback_host: 'https://starbase.workers.dev', + is_active: 1, + }, + ] + + vi.spyOn(doInstance, 'executeQuery').mockImplementation( + async (opts: { sql: string; params?: unknown[] }) => { + const sql = opts.sql.replace(/\s+/g, ' ').trim() + if ( + sql.includes( + 'SELECT slot, scheduled_at FROM tmp_alarm_slots' + ) + ) { + return Array.from(alarmSlots.entries()).map( + ([slot, scheduled_at]) => ({ slot, scheduled_at }) + ) as any + } + if (sql.startsWith('INSERT OR REPLACE INTO tmp_alarm_slots')) { + slotWriteCount++ + alarmSlots.set( + String(opts.params?.[0]), + Number(opts.params?.[1]) + ) + return [] as any + } + if (sql.startsWith('DELETE FROM tmp_alarm_slots')) { + alarmSlots.delete(String(opts.params?.[0])) + return [] as any + } + if (sql.includes('SELECT * FROM tmp_dump_jobs')) { + return dumpJob.status === 'in_progress' + ? [{ ...dumpJob }] + : ([] as any) + } + if (sql.includes('SELECT id FROM tmp_dump_jobs')) { + return dumpJob.status === 'in_progress' + ? [{ id: dumpJob.id }] + : ([] as any) + } + if (sql.includes('SELECT sql FROM sqlite_master')) { + return [{ sql: 'CREATE TABLE users (id INTEGER)' }] as any + } + if (sql.includes('FROM "users"')) { + return [{ __sb_rid: 1, id: 1 }] as any + } + if (sql.startsWith('INSERT OR REPLACE INTO tmp_dump_jobs')) { + dumpJob = { + ...dumpJob, + status: String(opts.params?.[1]), + } + return [] as any + } + if (sql.includes('FROM tmp_cron_tasks')) { + return cronTasks as any + } + return [] as any + } + ) + + const fetchSpy = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue(new Response('ok', { status: 200 })) + + const realNow = Date.now + let fakeNow = 100_000 + Date.now = () => fakeNow + + try { + // 1. Earliest dump slot is preserved even if a slower dump schedules +30s later + await doInstance.scheduleDumpAlarm(fakeNow + 5_000) // 105_000 + await doInstance.scheduleDumpAlarm(fakeNow + 30_000) + expect(alarmSlots.get('dump')).toBe(105_000) + expect(physicalAlarm).toBe(105_000) + + // 2. Repeated CronPlugin middleware setAlarm(105_400) does not clobber 105_000 AND skips no-op SQL write + const writesBefore = slotWriteCount + await doInstance.setAlarm(fakeNow + 5_400) // 105_400 + await doInstance.setAlarm(fakeNow + 5_400) // identical -> no-op write! + expect(slotWriteCount - writesBefore).toBe(1) + expect(physicalAlarm).toBe(105_000) + + // 3. Sustained <1s traffic past due time MUST NOT slide physicalAlarm forward (no starvation!) + fakeNow = 104_400 + await doInstance.setAlarm(105_400) + fakeNow = 105_000 + await doInstance.setAlarm(165_000) + expect(physicalAlarm).toBe(105_000) + + // Reset cron to 105_400 (only 400ms after dump at 105_000) and fire dump alarm at 105_000 + alarmSlots.set('cron', 105_400) + physicalAlarm = null + await doInstance.alarm() + + // Dump completed; cron at 105_400 MUST NOT fire early at 105_000; physical alarm re-armed to 105_400! + expect(dumpJob.status).toBe('completed') + expect(fetchSpy).not.toHaveBeenCalled() + expect(alarmSlots.has('dump')).toBe(false) + expect(alarmSlots.get('cron')).toBe(105_400) + expect(physicalAlarm).toBe(105_400) + + // 4. Advance clock to 105_400 -> cron now fires normally + fakeNow = 105_400 + physicalAlarm = null + await doInstance.alarm() + expect(fetchSpy).toHaveBeenCalledTimes(1) + expect(String(fetchSpy.mock.calls[0][0])).toBe( + 'https://starbase.workers.dev/cron/callback' + ) + + // 4b. Final-second CronPlugin middleware call (now=109_500, target=110_000) MUST NOT shift + // logical slots.cron forward to 110_500 (otherwise firing at 110_000 would miss cron!). + fakeNow = 106_000 + await doInstance.setAlarm(110_000) + expect(alarmSlots.get('cron')).toBe(110_000) + expect(physicalAlarm).toBe(110_000) + fakeNow = 109_500 + await doInstance.setAlarm(110_000) + expect(alarmSlots.get('cron')).toBe(110_000) + expect(physicalAlarm).toBe(110_000) + fakeNow = 110_000 + physicalAlarm = null + await doInstance.alarm() + expect(fetchSpy).toHaveBeenCalledTimes(2) + + // 5. deleteAlarm clears cron slot while preserving an active dump slot, and deletes physical alarm if none left + await doInstance.scheduleDumpAlarm(fakeNow + 5_000) + await doInstance.setAlarm(fakeNow + 60_000) + await doInstance.deleteAlarm() + expect(alarmSlots.has('cron')).toBe(false) + expect(alarmSlots.get('dump')).toBe(fakeNow + 5_000) + expect(physicalAlarm).toBe(fakeNow + 5_000) + + // Stale dump slot with no active job is cleared cleanly + fakeNow += 5_000 + physicalAlarm = null + await doInstance.alarm() + expect(alarmSlots.has('dump')).toBe(false) + + await doInstance.deleteAlarm() + expect(deleteAlarmSpy).toHaveBeenCalledTimes(1) + } finally { + Date.now = realNow + fetchSpy.mockRestore() + } + }) }) diff --git a/src/do.ts b/src/do.ts index b6bb2b6..3d6bb2e 100644 --- a/src/do.ts +++ b/src/do.ts @@ -1,4 +1,19 @@ import { DurableObject } from 'cloudflare:workers' +import { + ALARM_SLOTS_TABLE, + DUMP_JOBS_SCHEMA_SQL, + DUMP_JOBS_TABLE, + DumpJobRecord, + processDumpJobSlice, +} from './export/dump' + +const ALARM_SLOTS_SCHEMA_SQL = ` +CREATE TABLE IF NOT EXISTS ${ALARM_SLOTS_TABLE} ( + slot TEXT PRIMARY KEY, + scheduled_at INTEGER NOT NULL +);` + +type AlarmSlot = 'cron' | 'dump' export class StarbaseDBDurableObject extends DurableObject { // Durable storage for the SQL database @@ -9,6 +24,10 @@ export class StarbaseDBDurableObject extends DurableObject { public connections = new Map() // Store the client auth token for requests back to our Worker private clientAuthToken: string + // Optional R2 bucket binding for large resumable database dumps + private dumpBucket?: R2Bucket + // Cached alarm slots to skip no-op SQL writes on repeated CronPlugin middleware calls + private cachedAlarmSlots?: Record /** * The constructor is invoked once upon creation of the Durable Object, i.e. the first call to @@ -20,6 +39,7 @@ export class StarbaseDBDurableObject extends DurableObject { constructor(ctx: DurableObjectState, env: Env) { super(ctx, env) this.clientAuthToken = env.CLIENT_AUTHORIZATION_TOKEN + this.dumpBucket = env.DUMP_BUCKET this.sql = ctx.storage.sql this.storage = ctx.storage @@ -63,12 +83,15 @@ export class StarbaseDBDurableObject extends DurableObject { this.executeQuery({ sql: allowlistStatement }) this.executeQuery({ sql: allowlistRejectedStatement }) this.executeQuery({ sql: rlsStatement }) + this.sql.exec(DUMP_JOBS_SCHEMA_SQL) + this.sql.exec(ALARM_SLOTS_SCHEMA_SQL) } init() { return { getAlarm: this.getAlarm.bind(this), setAlarm: this.setAlarm.bind(this), + scheduleDumpAlarm: this.scheduleDumpAlarm.bind(this), deleteAlarm: this.deleteAlarm.bind(this), getStatistics: this.getStatistics.bind(this), executeQuery: this.executeQuery.bind(this), @@ -79,73 +102,251 @@ export class StarbaseDBDurableObject extends DurableObject { return await this.storage.getAlarm() } + private async readAlarmSlots(): Promise> { + if (this.cachedAlarmSlots) { + return { ...this.cachedAlarmSlots } + } + const slots: Record = { + cron: null, + dump: null, + } + try { + const rows = (await this.executeQuery({ + sql: `SELECT slot, scheduled_at FROM ${ALARM_SLOTS_TABLE};`, + isRaw: false, + })) as Record[] + for (const row of rows) { + const slot = String(row.slot ?? '') + const ts = Number(row.scheduled_at) + if ( + (slot === 'cron' || slot === 'dump') && + Number.isFinite(ts) + ) { + slots[slot] = ts + } + } + } catch (err) { + console.error('Error reading alarm slots:', err) + } + this.cachedAlarmSlots = { ...slots } + return slots + } + + private async upsertAlarmSlot( + slot: AlarmSlot, + scheduledAt: number + ): Promise { + await this.executeQuery({ + sql: `INSERT OR REPLACE INTO ${ALARM_SLOTS_TABLE} (slot, scheduled_at) VALUES (?, ?);`, + params: [slot, scheduledAt], + }) + if (this.cachedAlarmSlots) { + this.cachedAlarmSlots[slot] = scheduledAt + } + } + + private async clearAlarmSlot(slot: AlarmSlot): Promise { + await this.executeQuery({ + sql: `DELETE FROM ${ALARM_SLOTS_TABLE} WHERE slot = ?;`, + params: [slot], + }) + if (this.cachedAlarmSlots) { + this.cachedAlarmSlots[slot] = null + } + } + + private async syncPhysicalAlarm( + options?: DurableObjectSetAlarmOptions, + forceRearm = false + ): Promise { + const slots = await this.readAlarmSlots() + const candidates = [slots.cron, slots.dump].filter( + (v): v is number => v !== null && Number.isFinite(v) + ) + if (candidates.length === 0) return + + const earliest = Math.min(...candidates) + if (!forceRearm) { + // Never push back an already-scheduled earlier/due physical alarm (prevents starvation under traffic). + const currentPhysical = await this.storage + .getAlarm() + .catch(() => null) + if (currentPhysical !== null && currentPhysical <= earliest) { + return + } + } + await this.storage.setAlarm(earliest, options) + } + + private async scheduleSlotAlarm( + slot: AlarmSlot, + scheduledTime: number | Date, + options?: DurableObjectSetAlarmOptions + ): Promise { + const now = Date.now() + const inputTime = + scheduledTime instanceof Date + ? scheduledTime.getTime() + : scheduledTime + const finalTime = Math.max(inputTime, now + 1000) + + const slots = await this.readAlarmSlots() + if ( + slot === 'cron' && + slots.cron !== null && + inputTime <= slots.cron && + finalTime >= slots.cron && + slots.cron >= now + ) { + // Preserve unchanged or final-second cron deadlines without clamping them past + // the already-armed physical alarm, and skip redundant SQLite writes. + await this.syncPhysicalAlarm(options) + return + } + + // Preserve an earlier pending dump wake-up so a slower job never delays a faster one + const nextSlotTime = + slot === 'dump' && slots.dump !== null + ? Math.min(slots.dump, finalTime) + : finalTime + + if (slots[slot] !== nextSlotTime) { + await this.upsertAlarmSlot(slot, nextSlotTime) + } + await this.syncPhysicalAlarm(options) + } + public async setAlarm( scheduledTime: number | Date, options?: DurableObjectSetAlarmOptions ): Promise { try { - const now = Date.now() - const inputTime = - scheduledTime instanceof Date - ? scheduledTime.getTime() - : scheduledTime - - // Ensure the time is in the future and at least 1 second from now - const minimumTime = now + 1000 - const finalTime = Math.max(inputTime, minimumTime) - await this.storage.setAlarm(finalTime, options) + await this.scheduleSlotAlarm('cron', scheduledTime, options) } catch (e) { console.error('Error setting alarm: ', e) throw e } } - public deleteAlarm(options?: DurableObjectSetAlarmOptions): Promise { + public async scheduleDumpAlarm( + scheduledTime: number | Date, + options?: DurableObjectSetAlarmOptions + ): Promise { + try { + await this.scheduleSlotAlarm('dump', scheduledTime, options) + } catch (e) { + console.error('Error setting dump alarm: ', e) + throw e + } + } + + public async deleteAlarm( + options?: DurableObjectSetAlarmOptions + ): Promise { + await this.clearAlarmSlot('cron') + const slots = await this.readAlarmSlots() + if (slots.dump !== null) { + await this.syncPhysicalAlarm(options, true) + return + } return this.storage.deleteAlarm(options) } async alarm() { - try { - // Fetch all the tasks that are marked to emit an event for this cycle. - const task = (await this.executeQuery({ - sql: 'SELECT * FROM tmp_cron_tasks WHERE is_active = 1;', - isRaw: false, - })) as Record[] - - if (!task.length) { - return - } - + const now = Date.now() + // Refresh cached slots on alarm entry to reflect durable SQLite state + this.cachedAlarmSlots = undefined + const initialSlots = await this.readAlarmSlots() + const dumpDue = initialSlots.dump === null || initialSlots.dump <= now + let ranDumpJob = false + + // 1. Continue any active resumable database dump jobs with breathing intervals + if (dumpDue) { try { - const firstTask = task[0] - await fetch(`${firstTask.callback_host}/cron/callback`, { - method: 'POST', - headers: { - Authorization: `Bearer ${this.clientAuthToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(task ?? []), - }) - } catch (error) { - console.error('Failed to call the alarm/cron callback:', error) - - // If the callback fails, we should try to reschedule to prevent the chain from breaking - try { - await this.setAlarm(Date.now() + 60000) - } catch (retryError) { - console.error('Failed to set recovery alarm:', retryError) + const activeDumpJobs = (await this.executeQuery({ + sql: `SELECT * FROM ${DUMP_JOBS_TABLE} WHERE status = 'in_progress' ORDER BY created_at ASC LIMIT 1;`, + isRaw: false, + })) as Record[] + + if (activeDumpJobs.length > 0) { + ranDumpJob = true + await this.clearAlarmSlot('dump') + const job = activeDumpJobs[0] as unknown as DumpJobRecord + const updated = await processDumpJobSlice({ + job, + executeSql: async (sql, params) => + (await this.executeQuery({ + sql, + params, + isRaw: false, + })) as Record[], + bucket: this.dumpBucket, + scheduleAlarm: async (scheduledTime) => { + await this.scheduleDumpAlarm(scheduledTime) + }, + }) + + if (updated.status !== 'in_progress') { + const remaining = (await this.executeQuery({ + sql: `SELECT id FROM ${DUMP_JOBS_TABLE} WHERE status = 'in_progress' LIMIT 1;`, + isRaw: false, + })) as Record[] + if (remaining.length > 0) { + await this.scheduleDumpAlarm(Date.now() + 1000) + } + } + } else if (initialSlots.dump !== null) { + await this.clearAlarmSlot('dump') } + } catch (dumpErr) { + console.error('Error processing dump job alarm:', dumpErr) } - } catch (e) { - console.error('There was an error processing an alarm: ', e) + } - // Try to recover by scheduling a retry in 1 minute + // 2. Process active cron plugin tasks ONLY when cron slot is actually due (strict <= now) + const cronDue = + initialSlots.cron !== null ? initialSlots.cron <= now : !ranDumpJob + + if (cronDue) { + if (initialSlots.cron !== null) { + await this.clearAlarmSlot('cron') + } try { - await this.setAlarm(Date.now() + 60000) - } catch (retryError) { - console.error('Failed to set recovery alarm:', retryError) + const task = (await this.executeQuery({ + sql: 'SELECT * FROM tmp_cron_tasks WHERE is_active = 1;', + isRaw: false, + })) as Record[] + + if (task.length) { + const firstTask = task[0] + await fetch(`${firstTask.callback_host}/cron/callback`, { + method: 'POST', + headers: { + Authorization: `Bearer ${this.clientAuthToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(task ?? []), + }) + } + } catch (e: any) { + if ( + !( + typeof e?.message === 'string' && + e.message.includes('no such table') + ) + ) { + console.error('There was an error processing an alarm: ', e) + await this.setAlarm(Date.now() + 60000).catch((retryErr) => + console.error('Failed to set recovery alarm:', retryErr) + ) + } } } + + // 3. Re-arm the single physical DO alarm to the earliest remaining slot + await this.syncPhysicalAlarm(undefined, true).catch((syncErr) => + console.error('Failed to synchronize remaining alarm:', syncErr) + ) } public async getStatistics(): Promise<{ diff --git a/src/export/dump.test.ts b/src/export/dump.test.ts index ca65b43..ca2bcc8 100644 --- a/src/export/dump.test.ts +++ b/src/export/dump.test.ts @@ -1,7 +1,21 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' -import { dumpDatabaseRoute } from './dump' +import { + dumpDatabaseRoute, + getDumpJobStatusRoute, + downloadDumpJobRoute, + processDumpJobSlice, + R2MultipartDumpWriter, + formatDumpFilename, + formatSqlValue, + formatInsertStatement, + shouldSkipTable, + validateCallbackUrl, + DUMP_JOBS_TABLE, + ALARM_SLOTS_TABLE, + DumpJobRecord, + SQLITE_DUMP_HEADER, +} from './dump' import { executeOperation } from '.' -import { createResponse } from '../utils' import type { DataSource } from '../types' import type { StarbaseDBConfiguration } from '../handler' @@ -19,6 +33,289 @@ vi.mock('../utils', () => ({ ), })) +/** + * In-memory R2Bucket test double that enforces Cloudflare R2's exact equal + * non-final multipart part-size invariant, staging put/get/delete, and abort. + */ +class MockR2Bucket { + public objects = new Map() + public activeUploads = new Map< + string, + { + key: string + parts: Map + aborted: boolean + completed: boolean + } + >() + public completedPartSizes: number[] = [] + public abortCalls = 0 + private seq = 0 + + private toUint8Array(value: any): Uint8Array { + if (value instanceof Uint8Array) return new Uint8Array(value) + if (value instanceof ArrayBuffer) return new Uint8Array(value) + if (typeof value === 'string') return new TextEncoder().encode(value) + throw new Error('Unsupported mock R2 payload type') + } + + private makeUploadHandle(key: string, uploadId: string): R2MultipartUpload { + return { + key, + uploadId, + uploadPart: async (partNumber: number, value: any) => { + const state = this.activeUploads.get(uploadId) + if (!state || state.aborted || state.completed) { + throw new Error(`Upload ${uploadId} is not active`) + } + const bytes = this.toUint8Array(value) + const etag = `etag-${uploadId}-${partNumber}` + state.parts.set(partNumber, { etag, bytes }) + return { partNumber, etag } + }, + complete: async (uploadedParts: R2UploadedPart[]) => { + const state = this.activeUploads.get(uploadId) + if (!state || state.aborted || state.completed) { + throw new Error(`Upload ${uploadId} cannot be completed`) + } + const ordered = [...uploadedParts].sort( + (a, b) => a.partNumber - b.partNumber + ) + const chunks: Uint8Array[] = [] + let totalLen = 0 + this.completedPartSizes = [] + for (const p of ordered) { + const stored = state.parts.get(p.partNumber) + if (!stored || stored.etag !== p.etag) { + throw new Error( + `Missing or mismatched part ${p.partNumber}` + ) + } + chunks.push(stored.bytes) + this.completedPartSizes.push(stored.bytes.byteLength) + totalLen += stored.bytes.byteLength + } + if (this.completedPartSizes.length > 1) { + const partSize = this.completedPartSizes[0] + for ( + let i = 0; + i < this.completedPartSizes.length - 1; + i++ + ) { + if (this.completedPartSizes[i] !== partSize) { + throw new Error( + `R2 multipart violation: non-final part ${i + 1} (${this.completedPartSizes[i]}) !== part 1 (${partSize})` + ) + } + } + const last = + this.completedPartSizes[ + this.completedPartSizes.length - 1 + ] + if (last > partSize) { + throw new Error( + `R2 multipart violation: final part (${last}) > part 1 (${partSize})` + ) + } + } + const full = new Uint8Array(totalLen) + let offset = 0 + for (const c of chunks) { + full.set(c, offset) + offset += c.byteLength + } + this.objects.set(key, full) + state.completed = true + return { key, size: totalLen } as R2Object + }, + abort: async () => { + this.abortCalls++ + const state = this.activeUploads.get(uploadId) + if (state) state.aborted = true + }, + } as unknown as R2MultipartUpload + } + + async createMultipartUpload(key: string): Promise { + const uploadId = `upload-${++this.seq}` + this.activeUploads.set(uploadId, { + key, + parts: new Map(), + aborted: false, + completed: false, + }) + return this.makeUploadHandle(key, uploadId) + } + + resumeMultipartUpload(key: string, uploadId: string): R2MultipartUpload { + return this.makeUploadHandle(key, uploadId) + } + + async put(key: string, value: any): Promise { + const bytes = this.toUint8Array(value) + this.objects.set(key, bytes) + return { key, size: bytes.byteLength } as R2Object + } + + async get(key: string): Promise { + const bytes = this.objects.get(key) + if (!bytes) return null + const copy = new Uint8Array(bytes) + return { + key, + size: copy.byteLength, + body: new ReadableStream({ + start(controller) { + controller.enqueue(copy) + controller.close() + }, + }), + arrayBuffer: async () => copy.buffer, + text: async () => new TextDecoder().decode(copy), + } as unknown as R2ObjectBody + } + + async delete(key: string): Promise { + this.objects.delete(key) + } +} + +/** + * In-memory SQL dispatcher supporting rowid keyset pagination, WITHOUT ROWID + * OFFSET fallback, UNIQUE(r2_key) enforcement, and dynamic `saveJob` writes. + */ +function createInMemorySqlHarness(fixture: { + tables: Record< + string, + { + schema: string + rows: { rowid: number | bigint; data: Record }[] + } + > +}) { + const jobs = new Map() + + const executeSql = async ( + sql: string, + params: unknown[] = [] + ): Promise[]> => { + const norm = sql.replace(/\s+/g, ' ').trim() + + if (norm === "SELECT name FROM sqlite_master WHERE type='table';") { + return [ + ...Object.keys(fixture.tables).map((name) => ({ name })), + { name: 'sqlite_sequence' }, + { name: '_cf_KV' }, + { name: 'tmp_cache' }, + { name: 'tmp_query_log' }, + { name: 'tmp_cron_tasks' }, + { name: DUMP_JOBS_TABLE }, + { name: ALARM_SLOTS_TABLE }, + ] + } + + if ( + norm === + "SELECT sql FROM sqlite_master WHERE type='table' AND name=?;" + ) { + const tbl = fixture.tables[String(params[0])] + return tbl ? [{ sql: tbl.schema }] : [] + } + + const rawTrim = sql.trim() + + // Keyset pagination: SELECT CAST(_rowid_ AS TEXT) AS __sb_rid[_*], * FROM "t" [WHERE _rowid_ > N] ORDER BY _rowid_ ASC LIMIT M; + const keysetMatch = rawTrim.match( + /^SELECT CAST\(_rowid_ AS TEXT\) AS (__sb_rid_*), \* FROM "([\s\S]+)"(?: WHERE _rowid_ > (-?\d+))? ORDER BY _rowid_ ASC LIMIT (\d+);$/ + ) + if (keysetMatch) { + const alias = keysetMatch[1] + const tableName = keysetMatch[2].replace(/""/g, '"') + const afterRowId = + keysetMatch[3] !== undefined ? BigInt(keysetMatch[3]) : null + const limit = Number(keysetMatch[4]) + const tbl = fixture.tables[tableName] + if (!tbl) throw new Error(`No such table: ${tableName}`) + if (/\bWITHOUT\s+ROWID\b/i.test(tbl.schema)) { + throw new Error('no such column: _rowid_') + } + const filtered = tbl.rows + .filter( + (r) => afterRowId === null || BigInt(r.rowid) > afterRowId + ) + .sort((a, b) => + BigInt(a.rowid) < BigInt(b.rowid) + ? -1 + : BigInt(a.rowid) > BigInt(b.rowid) + ? 1 + : 0 + ) + .slice(0, limit) + return filtered.map((r) => ({ + [alias]: String(r.rowid), + ...r.data, + })) + } + + // OFFSET fallback: SELECT * FROM "t" LIMIT M OFFSET O; + const offsetMatch = rawTrim.match( + /^SELECT \* FROM "([\s\S]+)" LIMIT (\d+) OFFSET (\d+);$/ + ) + if (offsetMatch) { + const tableName = offsetMatch[1].replace(/""/g, '"') + const limit = Number(offsetMatch[2]) + const offset = Number(offsetMatch[3]) + const tbl = fixture.tables[tableName] + if (!tbl) throw new Error(`No such table: ${tableName}`) + return tbl.rows + .slice(offset, offset + limit) + .map((r) => ({ ...r.data })) + } + + // Generic INSERT / INSERT OR REPLACE INTO tmp_dump_jobs (cols) VALUES (...) + const insertMatch = norm.match( + /^INSERT (OR REPLACE )?INTO tmp_dump_jobs \(([^)]+)\) VALUES \(([^)]+)\);$/ + ) + if (insertMatch) { + const isReplace = Boolean(insertMatch[1]) + const cols = insertMatch[2].split(',').map((c) => c.trim()) + const rec: Record = {} + cols.forEach((col, i) => { + rec[col] = params[i] ?? null + }) + if (!isReplace) { + for (const existing of jobs.values()) { + if (existing.r2_key === rec.r2_key) { + throw new Error( + 'UNIQUE constraint failed: tmp_dump_jobs.r2_key' + ) + } + } + } + jobs.set(String(rec.id), rec as DumpJobRecord) + return [] + } + + if (norm === `DELETE FROM ${DUMP_JOBS_TABLE} WHERE id = ?;`) { + jobs.delete(String(params[0])) + return [] + } + + if (norm === `SELECT * FROM ${DUMP_JOBS_TABLE} WHERE id = ? LIMIT 1;`) { + const j = jobs.get(String(params[0])) + return j ? [{ ...j }] : [] + } + + throw new Error(`Unhandled SQL in harness: ${norm}`) + } + + return { jobs, executeSql } +} + +function makeRows(rows: Record[]) { + return rows.map((data, idx) => ({ rowid: idx + 1, data })) +} + let mockDataSource: DataSource let mockConfig: StarbaseDBConfiguration @@ -28,7 +325,11 @@ beforeEach(() => { mockDataSource = { source: 'external', external: { dialect: 'sqlite' }, - rpc: { executeQuery: vi.fn() }, + rpc: { + executeQuery: vi.fn(), + setAlarm: vi.fn(), + scheduleDumpAlarm: vi.fn(), + }, } as any mockConfig = { @@ -71,6 +372,7 @@ describe('Database Dump Module', () => { expect(dumpText).toContain( 'CREATE TABLE users (id INTEGER, name TEXT);' ) + expect(dumpText).not.toContain(';;') expect(dumpText).toContain("INSERT INTO users VALUES (1, 'Alice');") expect(dumpText).toContain("INSERT INTO users VALUES (2, 'Bob');") expect(dumpText).toContain( @@ -141,5 +443,947 @@ describe('Database Dump Module', () => { expect(response.status).toBe(500) const jsonResponse: { error: string } = await response.json() expect(jsonResponse.error).toBe('Failed to create database dump') + consoleErrorMock.mockRestore() + }) +}) + +describe('Database Dump Helpers & SQL Edge Cases', () => { + it('formats UTC timestamped R2 dump filenames cleanly', () => { + expect(formatDumpFilename(new Date('2024-01-01T17:00:00.000Z'))).toBe( + 'dump_20240101-170000.sql' + ) + }) + + it('formats SQL values accurately for NULL, booleans, numbers, BLOBs, and objects', () => { + expect(formatSqlValue(null)).toBe('NULL') + expect(formatSqlValue(undefined)).toBe('NULL') + expect(formatSqlValue(true)).toBe('1') + expect(formatSqlValue(false)).toBe('0') + expect(formatSqlValue(Number.NaN)).toBe('NULL') + expect(formatSqlValue(Number.POSITIVE_INFINITY)).toBe('NULL') + expect(formatSqlValue(42.5)).toBe('42.5') + expect(formatSqlValue(9007199254740993n)).toBe('9007199254740993') + expect(formatSqlValue(new Uint8Array([0xde, 0xad, 0xbe, 0xef]))).toBe( + "X'DEADBEEF'" + ) + expect(formatSqlValue(new Uint8Array([0x01, 0x0a]).buffer)).toBe( + "X'010A'" + ) + expect(formatSqlValue({ k: "v'1" })).toBe('\'{"k":"v\'\'1"}\'') + }) + + it('quotes reserved SQLite keywords, special table names, and numeric column names cleanly', () => { + expect(formatInsertStatement('users', { id: 1, name: null })).toBe( + 'INSERT INTO users VALUES (1, NULL);\n' + ) + expect(formatInsertStatement('order', { v: 'safe' })).toBe( + 'INSERT INTO "order" VALUES (\'safe\');\n' + ) + expect(formatInsertStatement('group', { id: 1 })).toBe( + 'INSERT INTO "group" VALUES (1);\n' + ) + expect( + formatInsertStatement('user events', { id: 2, active: true }) + ).toBe('INSERT INTO "user events" VALUES (2, 1);\n') + // JS puts integer-index key "1" before "a" -> explicit column list prevents swapped values + expect( + formatInsertStatement('numbered', { a: 'first', '1': 'second' }) + ).toBe( + 'INSERT INTO numbered ("1", "a") VALUES (\'second\', \'first\');\n' + ) + }) + + it('skips sqlite_*, _cf_*, and StarbaseDB internal tables while preserving user tmp_* tables', () => { + expect(shouldSkipTable('sqlite_sequence')).toBe(true) + expect(shouldSkipTable('_cf_KV')).toBe(true) + expect(shouldSkipTable('tmp_query_log')).toBe(true) + expect(shouldSkipTable('tmp_cache')).toBe(true) + expect(shouldSkipTable('tmp_rls_policies')).toBe(true) + expect(shouldSkipTable('tmp_cron_tasks')).toBe(true) + expect(shouldSkipTable(DUMP_JOBS_TABLE)).toBe(true) + expect(shouldSkipTable(ALARM_SLOTS_TABLE)).toBe(true) + expect(shouldSkipTable('tmp_user_staging')).toBe(false) + expect(shouldSkipTable('users')).toBe(false) + }) + + it('sanitizes newlines in table names inside SQL header comments', async () => { + const evilName = 'x\nDROP TABLE users;' + const harness = createInMemorySqlHarness({ + tables: { + [evilName]: { + schema: `CREATE TABLE "${evilName}" (id INTEGER)`, + rows: makeRows([{ id: 1 }]), + }, + }, + }) + vi.mocked(executeOperation).mockImplementation(async (q) => + harness.executeSql(q[0].sql, q[0].params) + ) + const res = await dumpDatabaseRoute(mockDataSource, mockConfig) + const sql = await res.text() + expect(sql).toContain('-- Table: x DROP TABLE users;\n') + expect(sql).not.toContain('\nDROP TABLE users;\n') + }) + + it('validates callback URLs and rejects unsafe schemes with 400', async () => { + expect(validateCallbackUrl('https://hooks.example.com/dump')).toBe( + 'https://hooks.example.com/dump' + ) + expect(validateCallbackUrl('')).toBeUndefined() + expect(() => validateCallbackUrl('javascript:alert(1)')).toThrow( + /only http:\/\/ and https:\/\// + ) + const req = new Request( + 'https://starbase.workers.dev/export/dump?callback_url=ftp://evil.example/cb' + ) + const res = await dumpDatabaseRoute(mockDataSource, mockConfig, req) + expect(res.status).toBe(400) + }) +}) + +describe('Security, Pagination & Streaming Fallback (#59)', () => { + it('enforces admin-only access (403) on ?async=true, status, and download routes while keeping client sync streaming', async () => { + const clientConfig: StarbaseDBConfiguration = { + ...mockConfig, + role: 'client', + } + const dsWithR2: DataSource = { + ...mockDataSource, + dumpBucket: new MockR2Bucket() as unknown as R2Bucket, + } + + const asyncRes = await dumpDatabaseRoute( + dsWithR2, + clientConfig, + new Request('https://starbase.workers.dev/export/dump?async=true') + ) + expect(asyncRes.status).toBe(403) + + const statusRes = await getDumpJobStatusRoute( + 'job-1', + dsWithR2, + clientConfig + ) + expect(statusRes.status).toBe(403) + + const dlRes = await downloadDumpJobRoute( + 'job-1', + dsWithR2, + clientConfig + ) + expect(dlRes.status).toBe(403) + + // Client-role sync dump stays on ReadableStream path (never bypasses RLS via DO alarm) + const harness = createInMemorySqlHarness({ + tables: { + notes: { + schema: 'CREATE TABLE notes (id INTEGER, body TEXT)', + rows: makeRows([{ id: 1, body: 'owned' }]), + }, + }, + }) + vi.mocked(executeOperation).mockImplementation(async (q) => + harness.executeSql(q[0].sql, q[0].params) + ) + const clientSync = await dumpDatabaseRoute( + dsWithR2, + clientConfig, + undefined, + { maxSyncBytes: 1 } // Even with tiny byte cap, non-admin stays on streaming sync (no R2 bypass!) + ) + expect(clientSync.status).toBe(200) + expect(await clientSync.text()).toContain( + "INSERT INTO notes VALUES (1, 'owned');" + ) + }) + + it('streams paginated dumps via ReadableStream without R2 and uses keyset rowid immune to concurrent deletes', async () => { + const tableRows = makeRows([{ id: 1 }, { id: 2 }, { id: 3 }, { id: 4 }]) + const harness = createInMemorySqlHarness({ + tables: { + t: { + schema: 'CREATE TABLE t (id INTEGER PRIMARY KEY)', + rows: tableRows, + }, + kv: { + schema: 'CREATE TABLE kv (k TEXT PRIMARY KEY, v TEXT) WITHOUT ROWID', + rows: makeRows([ + { k: 'a', v: '1' }, + { k: 'b', v: '2' }, + { k: 'c', v: '3' }, + ]), + }, + }, + }) + let deletedAfterFirstPage = false + vi.mocked(executeOperation).mockImplementation(async (q) => { + const res = await harness.executeSql(q[0].sql, q[0].params) + if ( + !deletedAfterFirstPage && + q[0].sql.includes('FROM "t"') && + !q[0].sql.includes('WHERE _rowid_ >') + ) { + deletedAfterFirstPage = true + // Concurrent delete of rowid 1 while export is mid-table: keyset MUST still export id 3 and 4! + tableRows.shift() + } + return res + }) + + // No dumpBucket bound -> streams cleanly via ReadableStream without 503 + const res = await dumpDatabaseRoute( + mockDataSource, + mockConfig, + undefined, + { chunkSize: 2 } + ) + expect(res.status).toBe(200) + const sql = await res.text() + expect(sql.startsWith(SQLITE_DUMP_HEADER)).toBe(true) + expect(sql).toContain('INSERT INTO t VALUES (1);') + expect(sql).toContain('INSERT INTO t VALUES (2);') + expect(sql).toContain('INSERT INTO t VALUES (3);') + expect(sql).toContain('INSERT INTO t VALUES (4);') + // WITHOUT ROWID table fell back cleanly to OFFSET pagination + expect(sql).toContain("INSERT INTO kv VALUES ('a', '1');") + expect(sql).toContain("INSERT INTO kv VALUES ('c', '3');") + // Stock internal tables excluded + expect(sql).not.toContain('tmp_query_log') + expect(sql).not.toContain('_cf_KV') + expect(sql).not.toContain('sqlite_sequence') + }) + + it('enforces maxSyncBytes after appending the final short page (no oversized 200 bypass)', async () => { + const harness = createInMemorySqlHarness({ + tables: { + wide: { + schema: 'CREATE TABLE wide (x TEXT)', + rows: makeRows([{ x: 'a'.repeat(500) }]), + }, + }, + }) + vi.mocked(executeOperation).mockImplementation(async (q) => + harness.executeSql(q[0].sql, q[0].params) + ) + const mockBucket = new MockR2Bucket() + const dsWithR2: DataSource = { + ...mockDataSource, + dumpBucket: mockBucket as unknown as R2Bucket, + } + const res = await dumpDatabaseRoute(dsWithR2, mockConfig, undefined, { + maxSyncBytes: 120, + minPartSizeBytes: 64, + }) + expect(res.status).toBe(202) + const { result } = (await res.json()) as { result: any } + expect(result.status).toBe('in_progress') + expect(mockDataSource.rpc.scheduleDumpAlarm).toHaveBeenCalledTimes(1) + }) +}) + +describe('Resumable R2 Multipart Dumps, Concurrency & Crash Safety (#59)', () => { + it('promotes slow dump to R2, resumes across DO breathing alarms, enforces exact part sizes, hides callbackUrl, and matches sync SQL byte-for-byte', async () => { + const usersRows = makeRows( + Array.from({ length: 7 }, (_, i) => ({ + id: i + 1, + name: `User_${i + 1}`, + payload: 'x'.repeat(200), + })) + ) + const ordersRows = makeRows( + Array.from({ length: 6 }, (_, i) => ({ + id: i + 100, + user_id: (i % 7) + 1, + amount: (i + 1) * 12.5, + raw: new Uint8Array([0xca, 0xfe, i]), + })) + ) + + const harness = createInMemorySqlHarness({ + tables: { + users: { + schema: 'CREATE TABLE users (id INTEGER, name TEXT, payload TEXT)', + rows: usersRows, + }, + orders: { + schema: 'CREATE TABLE orders (id INTEGER, user_id INTEGER, amount REAL, raw BLOB)', + rows: ordersRows, + }, + }, + }) + + // Ground-truth single-pass streaming dump for byte-exact comparison + vi.mocked(executeOperation).mockImplementation(async (q) => + harness.executeSql(q[0].sql, q[0].params) + ) + const expectedSql = await ( + await dumpDatabaseRoute(mockDataSource, mockConfig, undefined, { + chunkSize: 100, + }) + ).text() + + const mockBucket = new MockR2Bucket() + const scheduledAlarms: number[] = [] + const webhookCalls: { url: string; body: any; event: string | null }[] = + [] + let webhookSawCompleted = false + let webhookHadSignal = false + + const dsWithR2: DataSource = { + ...mockDataSource, + dumpBucket: mockBucket as unknown as R2Bucket, + rpc: { + executeQuery: vi.fn(), + setAlarm: vi.fn(), + scheduleDumpAlarm: vi.fn(async (ts: number | Date) => { + scheduledAlarms.push(ts instanceof Date ? ts.getTime() : ts) + }), + } as any, + } + + let fakeClock = Date.parse('2024-01-01T17:00:00.000Z') + vi.mocked(executeOperation).mockImplementation(async (q) => { + if (q[0].sql.startsWith('SELECT CAST(_rowid_')) fakeClock += 4_000 + return harness.executeSql(q[0].sql, q[0].params) + }) + + const mockFetch = vi.fn( + async (input: RequestInfo | URL, init?: RequestInit) => { + const payload = JSON.parse(String(init?.body || '{}')) + webhookCalls.push({ + url: String(input), + body: payload, + event: + (init?.headers as Record)?.[ + 'X-Starbase-Event' + ] ?? null, + }) + webhookHadSignal = init?.signal instanceof AbortSignal + webhookSawCompleted = + harness.jobs.get(payload.jobId)?.status === 'completed' + return new Response('ok', { status: 200 }) + } + ) as unknown as typeof fetch + + const req = new Request( + 'https://starbase.workers.dev/export/dump?callback_url=https://hooks.example.com/secret-token&chunk_size=2&breathing_ms=5000&work_budget_ms=5000' + ) + const initRes = await dumpDatabaseRoute(dsWithR2, mockConfig, req, { + syncTimeoutMs: 6_000, + workBudgetMs: 5_000, + breathingIntervalMs: 5_000, + minPartSizeBytes: 512, + now: () => fakeClock, + fetchFn: mockFetch, + }) + + expect(initRes.status).toBe(202) + const initJson = (await initRes.json()) as { result: any } + expect(initJson.result.status).toBe('in_progress') + expect(initJson.result.key).toMatch(/^dump_20240101-1700\d\d\.sql$/) + // Secret webhook URL MUST NOT be exposed in public summary + expect(initJson.result.callbackUrl).toBeUndefined() + expect(scheduledAlarms.length).toBe(1) + + // Status returns 200 without leaking callbackUrl; download returns 409 while in_progress + const statusRes = await getDumpJobStatusRoute( + initJson.result.jobId, + dsWithR2, + mockConfig + ) + expect(statusRes.status).toBe(200) + expect( + ((await statusRes.json()) as { result: any }).result.callbackUrl + ).toBeUndefined() + + const earlyDl = await downloadDumpJobRoute( + initJson.result.jobId, + dsWithR2, + mockConfig + ) + expect(earlyDl.status).toBe(409) + + // Drive DO alarm slices with breathing pauses until completed + let alarmTicks = 0 + let latestJob = harness.jobs.get(initJson.result.jobId)! + while (latestJob.status === 'in_progress') { + alarmTicks++ + expect(alarmTicks).toBeLessThan(20) + fakeClock += latestJob.breathing_interval_ms + latestJob = await processDumpJobSlice({ + job: latestJob, + executeSql: async (sql, params) => { + if (sql.startsWith('SELECT CAST(_rowid_')) + fakeClock += 3_000 + return harness.executeSql(sql, params) + }, + bucket: mockBucket as unknown as R2Bucket, + scheduleAlarm: async (ts) => { + scheduledAlarms.push(ts) + }, + minPartSizeBytes: 512, + now: () => fakeClock, + fetchFn: mockFetch, + }) + } + + expect(alarmTicks).toBeGreaterThanOrEqual(2) + expect(latestJob.status).toBe('completed') + expect(latestJob.rows_exported).toBe( + usersRows.length + ordersRows.length + ) + expect(latestJob.callback_status).toBe('delivered') + expect(webhookCalls.length).toBe(1) + expect(webhookCalls[0].event).toBe('export.dump.completed') + expect(webhookSawCompleted).toBe(true) + expect(webhookHadSignal).toBe(true) + + // Strict Cloudflare R2 equal non-final part size invariant + expect(mockBucket.completedPartSizes.length).toBeGreaterThanOrEqual(3) + for (let i = 0; i < mockBucket.completedPartSizes.length - 1; i++) { + expect(mockBucket.completedPartSizes[i]).toBe(512) + } + expect( + mockBucket.completedPartSizes[ + mockBucket.completedPartSizes.length - 1 + ] + ).toBeLessThanOrEqual(512) + + // All versioned staging keys must be retired/cleaned up + expect( + Array.from(mockBucket.objects.keys()).filter((k) => + k.includes('.pending.') + ) + ).toEqual([]) + + const dlDone = await downloadDumpJobRoute( + initJson.result.jobId, + dsWithR2, + mockConfig + ) + expect(dlDone.status).toBe(200) + expect(await dlDone.text()).toBe(expectedSql) + }) + + it('guarantees unique R2 keys across concurrent same-second ?async=true requests and delegates execution solely to DO alarm', async () => { + const harness = createInMemorySqlHarness({ + tables: { + items: { + schema: 'CREATE TABLE items (id INTEGER)', + rows: makeRows([{ id: 1 }]), + }, + }, + }) + vi.mocked(executeOperation).mockImplementation(async (q) => + harness.executeSql(q[0].sql, q[0].params) + ) + const mockBucket = new MockR2Bucket() + const scheduled: number[] = [] + const dsWithR2: DataSource = { + ...mockDataSource, + dumpBucket: mockBucket as unknown as R2Bucket, + rpc: { + executeQuery: vi.fn(), + setAlarm: vi.fn(), + scheduleDumpAlarm: vi.fn(async (ts: number) => { + scheduled.push(ts) + }), + } as any, + } + + const fixedNow = Date.parse('2024-01-01T17:00:00.000Z') + const req = () => + new Request('https://starbase.workers.dev/export/dump?async=true') + + // Fire 3 concurrent requests in the exact same millisecond + const responses = await Promise.all([ + dumpDatabaseRoute(dsWithR2, mockConfig, req(), { + now: () => fixedNow, + }), + dumpDatabaseRoute(dsWithR2, mockConfig, req(), { + now: () => fixedNow, + }), + dumpDatabaseRoute(dsWithR2, mockConfig, req(), { + now: () => fixedNow, + }), + ]) + const summaries = await Promise.all( + responses.map(async (r) => { + expect(r.status).toBe(202) + return ((await r.json()) as { result: any }).result + }) + ) + const keys = summaries.map((s) => s.key) + expect(new Set(keys).size).toBe(3) + expect(keys).toContain('dump_20240101-170000.sql') + // Worker enqueues only — DO alarm is sole executor + expect(summaries.every((s) => s.status === 'in_progress')).toBe(true) + expect(scheduled.length).toBe(3) + }) + + it('versioned R2 staging is crash-safe if DO restarts after R2 put and before SQL commit (zero duplicated bytes)', async () => { + const mockBucket = new MockR2Bucket() + const w1 = await R2MultipartDumpWriter.create( + mockBucket as unknown as R2Bucket, + 'dump_crash.sql', + 5 * 1024 * 1024 + ) + await w1.append('A'.repeat(70_000)) + const staging1 = await w1.checkpointPending() + expect(staging1).toBe('dump_crash.sql.pending.70000') + + const committedJob: DumpJobRecord = { + id: 'job-crash', + status: 'in_progress', + r2_key: 'dump_crash.sql', + upload_id: w1.uploadId, + tables_json: '[]', + table_index: 0, + row_offset: 0, + last_rowid: null, + parts_json: JSON.stringify(w1.parts), + pending_staging_key: staging1, + chunk_size: 500, + work_budget_ms: 5000, + breathing_interval_ms: 5000, + rows_exported: 0, + bytes_written: 70_000, + callback_url: null, + callback_status: null, + error: null, + created_at: '2024-01-01T17:00:00.000Z', + updated_at: '2024-01-01T17:00:00.000Z', + completed_at: null, + } + + // Slice 2 writes new versioned staging key, then CRASHES before SQL commit & retirePreviousStaging() + const w2 = await R2MultipartDumpWriter.resume( + mockBucket as unknown as R2Bucket, + committedJob, + 5 * 1024 * 1024 + ) + await w2.append('B'.repeat(50)) + const staging2 = await w2.checkpointPending() + expect(staging2).toBe('dump_crash.sql.pending.70050') + // Original staging key MUST still exist untouched! + expect(mockBucket.objects.has('dump_crash.sql.pending.70000')).toBe( + true + ) + + // Restart replays from committedJob (still pointing at staging1 = 70000) + const w3 = await R2MultipartDumpWriter.resume( + mockBucket as unknown as R2Bucket, + committedJob, + 5 * 1024 * 1024 + ) + await w3.append('B'.repeat(50)) + await w3.complete() + + const finalBytes = mockBucket.objects.get('dump_crash.sql')! + expect(finalBytes.byteLength).toBe(70_050) + expect(w3.bytesWritten).toBe(70_050) + }) + + it('isolates webhook bookkeeping failures so completed backups stay completed with a single webhook event', async () => { + const harness = createInMemorySqlHarness({ + tables: { + t: { + schema: 'CREATE TABLE t (i INTEGER)', + rows: makeRows([{ i: 1 }]), + }, + }, + }) + const mockBucket = new MockR2Bucket() + const upload = + await mockBucket.createMultipartUpload('dump_webhook.sql') + const job: DumpJobRecord = { + id: 'job-wh', + status: 'in_progress', + r2_key: 'dump_webhook.sql', + upload_id: upload.uploadId, + tables_json: JSON.stringify(['t']), + table_index: 0, + row_offset: 0, + last_rowid: null, + parts_json: '[]', + pending_staging_key: null, + chunk_size: 100, + work_budget_ms: 5000, + breathing_interval_ms: 5000, + rows_exported: 0, + bytes_written: 0, + callback_url: 'https://hooks.example.com/done', + callback_status: null, + error: null, + created_at: '2024-01-01T17:00:00.000Z', + updated_at: '2024-01-01T17:00:00.000Z', + completed_at: null, + } + harness.jobs.set(job.id, job) + + const events: string[] = [] + const consoleErrorSpy = vi + .spyOn(console, 'error') + .mockImplementation(() => {}) + + let failCallbackSave = true + const result = await processDumpJobSlice({ + job, + executeSql: async (sql, params) => { + if ( + failCallbackSave && + sql.includes('INSERT OR REPLACE INTO tmp_dump_jobs') && + params?.includes('delivered') + ) { + failCallbackSave = false + throw new Error('transient bookkeeping write failed') + } + return harness.executeSql(sql, params) + }, + bucket: mockBucket as unknown as R2Bucket, + fetchFn: (async (_u: any, init?: RequestInit) => { + events.push(JSON.parse(String(init?.body || '{}')).event) + return new Response('ok', { status: 200 }) + }) as unknown as typeof fetch, + }) + + expect(result.status).toBe('completed') + expect(harness.jobs.get(job.id)?.status).toBe('completed') + expect(events).toEqual(['export.dump.completed']) + consoleErrorSpy.mockRestore() + }) + + it('cleans up reserved job and aborts R2 upload if scheduleDumpAlarm throws during job creation', async () => { + const harness = createInMemorySqlHarness({ + tables: { + t: { + schema: 'CREATE TABLE t (i INTEGER)', + rows: makeRows([{ i: 1 }]), + }, + }, + }) + vi.mocked(executeOperation).mockImplementation(async (q) => + harness.executeSql(q[0].sql, q[0].params) + ) + const mockBucket = new MockR2Bucket() + const dsFailAlarm: DataSource = { + ...mockDataSource, + dumpBucket: mockBucket as unknown as R2Bucket, + rpc: { + executeQuery: vi.fn(), + setAlarm: vi.fn(), + scheduleDumpAlarm: vi.fn(async () => { + throw new Error('transient setAlarm failure') + }), + } as any, + } + const consoleErrorSpy = vi + .spyOn(console, 'error') + .mockImplementation(() => {}) + + const res = await dumpDatabaseRoute( + dsFailAlarm, + mockConfig, + new Request('https://starbase.workers.dev/export/dump?async=true') + ) + expect(res.status).toBe(500) + expect(harness.jobs.size).toBe(0) + expect(mockBucket.abortCalls).toBe(1) + consoleErrorSpy.mockRestore() + }) + + it('aborts R2 multipart upload and notifies callback when resume() fails on missing staging buffer', async () => { + const harness = createInMemorySqlHarness({ + tables: { + users: { + schema: 'CREATE TABLE users (id INTEGER)', + rows: makeRows([{ id: 1 }]), + }, + }, + }) + const mockBucket = new MockR2Bucket() + const upload = await mockBucket.createMultipartUpload( + 'dump_20240101-170000.sql' + ) + const job: DumpJobRecord = { + id: 'job-fail-1', + status: 'in_progress', + r2_key: 'dump_20240101-170000.sql', + upload_id: upload.uploadId, + tables_json: JSON.stringify(['users']), + table_index: 0, + row_offset: 0, + last_rowid: null, + parts_json: '[]', + pending_staging_key: 'dump_20240101-170000.sql.pending.16', + chunk_size: 2, + work_budget_ms: 5000, + breathing_interval_ms: 5000, + rows_exported: 0, + bytes_written: 16, + callback_url: 'https://hooks.example.com/fail', + callback_status: null, + error: null, + created_at: '2024-01-01T17:00:00.000Z', + updated_at: '2024-01-01T17:00:00.000Z', + completed_at: null, + } + harness.jobs.set(job.id, job) + + const consoleErrorSpy = vi + .spyOn(console, 'error') + .mockImplementation(() => {}) + const failWebhook = vi.fn( + async () => new Response('accepted', { status: 202 }) + ) as unknown as typeof fetch + + const failed = await processDumpJobSlice({ + job, + executeSql: harness.executeSql, + bucket: mockBucket as unknown as R2Bucket, + fetchFn: failWebhook, + }) + + expect(failed.status).toBe('failed') + expect(failed.error).toContain('Missing R2 dump staging buffer') + expect(failed.callback_status).toBe('delivered') + expect(failWebhook).toHaveBeenCalledTimes(1) + expect(mockBucket.abortCalls).toBe(1) + expect(mockBucket.activeUploads.get(upload.uploadId)?.aborted).toBe( + true + ) + consoleErrorSpy.mockRestore() + }) + + it('returns 400 on ?async=true without DUMP_BUCKET and 404 for unknown job IDs', async () => { + const harness = createInMemorySqlHarness({ tables: {} }) + vi.mocked(executeOperation).mockImplementation(async (q) => + harness.executeSql(q[0].sql, q[0].params) + ) + const noBucketRes = await dumpDatabaseRoute( + mockDataSource, + mockConfig, + new Request('https://starbase.workers.dev/export/dump?async=true') + ) + expect(noBucketRes.status).toBe(400) + + const dsWithR2: DataSource = { + ...mockDataSource, + dumpBucket: new MockR2Bucket() as unknown as R2Bucket, + } + expect( + (await getDumpJobStatusRoute('missing', dsWithR2, mockConfig)) + .status + ).toBe(404) + expect( + (await downloadDumpJobRoute('missing', dsWithR2, mockConfig)).status + ).toBe(404) + }) + + it('preserves >2^53 64-bit rowid precision and avoids shadowing user rowid / __sb_rid columns', async () => { + const harness = createInMemorySqlHarness({ + tables: { + big_ids: { + schema: 'CREATE TABLE big_ids (v TEXT)', + rows: [ + { rowid: 9007199254740995n, data: { v: 'first' } }, + { rowid: 9007199254740996n, data: { v: 'second' } }, + ], + }, + shadowed: { + schema: 'CREATE TABLE shadowed (rowid TEXT, __sb_rid TEXT, val INTEGER)', + rows: [ + { + rowid: 1, + data: { rowid: 'dup', __sb_rid: 'keep1', val: 10 }, + }, + { + rowid: 2, + data: { rowid: 'dup', __sb_rid: 'keep2', val: 20 }, + }, + ], + }, + }, + }) + vi.mocked(executeOperation).mockImplementation(async (q) => + harness.executeSql(q[0].sql, q[0].params) + ) + + const res = await dumpDatabaseRoute( + mockDataSource, + mockConfig, + undefined, + { chunkSize: 1 } + ) + expect(res.status).toBe(200) + const sql = await res.text() + expect(sql).toContain("INSERT INTO big_ids VALUES ('first');") + expect(sql).toContain("INSERT INTO big_ids VALUES ('second');") + // User's shadowed rowid and __sb_rid columns MUST BOTH be preserved intact! + expect(sql).toContain( + "INSERT INTO shadowed VALUES ('dup', 'keep1', 10);" + ) + expect(sql).toContain( + "INSERT INTO shadowed VALUES ('dup', 'keep2', 20);" + ) + }) + + it('adaptively halves page size on wide-row RPC payload errors and exports all rows', async () => { + const harness = createInMemorySqlHarness({ + tables: { + wide: { + schema: 'CREATE TABLE wide (id INTEGER, body TEXT)', + rows: makeRows([ + { id: 1, body: 'r1' }, + { id: 2, body: 'r2' }, + { id: 3, body: 'r3' }, + { id: 4, body: 'r4' }, + { id: 5, body: 'r5' }, + ]), + }, + }, + }) + const observedLimits: number[] = [] + vi.mocked(executeOperation).mockImplementation(async (q) => { + const m = q[0].sql.match(/LIMIT (\d+);$/) + if (m) { + const lim = Number(m[1]) + observedLimits.push(lim) + if (lim > 2) { + throw new Error( + 'RPC payload size limit exceeded (1 MiB max)' + ) + } + } + return harness.executeSql(q[0].sql, q[0].params) + }) + + const res = await dumpDatabaseRoute( + mockDataSource, + mockConfig, + undefined, + { chunkSize: 8 } + ) + expect(res.status).toBe(200) + const sql = await res.text() + // Must have halved 8 -> 4 -> 2 and exported all 5 rows across 3 pages! + expect(observedLimits.slice(0, 3)).toEqual([8, 4, 2]) + expect(sql.match(/INSERT INTO wide VALUES/g)?.length).toBe(5) + }) + + it('retains active R2 staging key until AFTER terminal SQL commit succeeds', async () => { + const harness = createInMemorySqlHarness({ + tables: { + t: { + schema: 'CREATE TABLE t (i INTEGER)', + rows: makeRows([{ i: 1 }]), + }, + }, + }) + const mockBucket = new MockR2Bucket() + const upload = await mockBucket.createMultipartUpload( + 'dump_term_crash.sql' + ) + await mockBucket.put( + 'dump_term_crash.sql.pending.16', + SQLITE_DUMP_HEADER + ) + const job: DumpJobRecord = { + id: 'job-term-crash', + status: 'in_progress', + r2_key: 'dump_term_crash.sql', + upload_id: upload.uploadId, + tables_json: JSON.stringify(['t']), + table_index: 0, + row_offset: 0, + last_rowid: null, + parts_json: '[]', + pending_staging_key: 'dump_term_crash.sql.pending.16', + chunk_size: 100, + work_budget_ms: 5000, + breathing_interval_ms: 5000, + rows_exported: 0, + bytes_written: 16, + callback_url: null, + callback_status: null, + error: null, + created_at: '2024-01-01T17:00:00.000Z', + updated_at: '2024-01-01T17:00:00.000Z', + completed_at: null, + } + harness.jobs.set(job.id, job) + + // Inject failure on terminal SQL commit -> staging object MUST still exist in R2! + await expect( + processDumpJobSlice({ + job, + executeSql: async (sql, params) => { + if (sql.includes('INSERT OR REPLACE INTO tmp_dump_jobs')) { + throw new Error('SQL commit interrupted') + } + return harness.executeSql(sql, params) + }, + bucket: mockBucket as unknown as R2Bucket, + }) + ).rejects.toThrow('SQL commit interrupted') + + expect(mockBucket.objects.has('dump_term_crash.sql.pending.16')).toBe( + true + ) + }) + + it('fires completion webhook on synchronous dumps (both streaming and R2-bound fast path)', async () => { + const harness = createInMemorySqlHarness({ + tables: { + items: { + schema: 'CREATE TABLE items (id INTEGER)', + rows: makeRows([{ id: 1 }, { id: 2 }]), + }, + }, + }) + vi.mocked(executeOperation).mockImplementation(async (q) => + harness.executeSql(q[0].sql, q[0].params) + ) + + const calls: { body: any; hasSignal: boolean }[] = [] + const mockFetch = vi.fn(async (_url: any, init?: RequestInit) => { + calls.push({ + body: JSON.parse(String(init?.body || '{}')), + hasSignal: init?.signal instanceof AbortSignal, + }) + return new Response('ok', { status: 200 }) + }) as unknown as typeof fetch + + const req = () => + new Request( + 'https://starbase.workers.dev/export/dump?callback_url=https://hooks.example.com/sync' + ) + + // 1. Streaming no-R2 sync dump delivers webhook upon stream completion + const rStream = await dumpDatabaseRoute( + mockDataSource, + mockConfig, + req(), + { fetchFn: mockFetch } + ) + expect(rStream.status).toBe(200) + await rStream.text() + expect(calls.length).toBe(1) + expect(calls[0].body.mode).toBe('sync') + expect(calls[0].body.rowsExported).toBe(2) + expect(calls[0].hasSignal).toBe(true) + + // 2. R2-bound fast sync dump also delivers webhook + const dsWithR2: DataSource = { + ...mockDataSource, + dumpBucket: new MockR2Bucket() as unknown as R2Bucket, + } + const rFast = await dumpDatabaseRoute(dsWithR2, mockConfig, req(), { + fetchFn: mockFetch, + }) + expect(rFast.status).toBe(200) + expect(calls.length).toBe(2) + expect(calls[1].body.mode).toBe('sync') + expect(calls[1].body.rowsExported).toBe(2) }) }) diff --git a/src/export/dump.ts b/src/export/dump.ts index 91a2e89..614632f 100644 --- a/src/export/dump.ts +++ b/src/export/dump.ts @@ -3,69 +3,1396 @@ import { StarbaseDBConfiguration } from '../handler' import { DataSource } from '../types' import { createResponse } from '../utils' -export async function dumpDatabaseRoute( +export const SQLITE_DUMP_HEADER = 'SQLite format 3\0' +export const DUMP_JOBS_TABLE = 'tmp_dump_jobs' +export const ALARM_SLOTS_TABLE = 'tmp_alarm_slots' + +export const DEFAULT_CHUNK_SIZE = 500 +export const DEFAULT_SYNC_TIMEOUT_MS = 20_000 +export const DEFAULT_MAX_SYNC_BYTES = 16 * 1024 * 1024 +export const DEFAULT_WORK_BUDGET_MS = 5_000 +export const DEFAULT_BREATHING_INTERVAL_MS = 5_000 +export const DEFAULT_R2_MIN_PART_SIZE_BYTES = 5 * 1024 * 1024 +export const DEFAULT_WEBHOOK_TIMEOUT_MS = 10_000 + +export const DUMP_JOBS_SCHEMA_SQL = ` +CREATE TABLE IF NOT EXISTS ${DUMP_JOBS_TABLE} ( + id TEXT PRIMARY KEY, + status TEXT NOT NULL, + r2_key TEXT NOT NULL UNIQUE, + upload_id TEXT, + tables_json TEXT NOT NULL, + table_index INTEGER NOT NULL DEFAULT 0, + row_offset INTEGER NOT NULL DEFAULT 0, + last_rowid TEXT, + parts_json TEXT NOT NULL DEFAULT '[]', + pending_staging_key TEXT, + chunk_size INTEGER NOT NULL DEFAULT ${DEFAULT_CHUNK_SIZE}, + work_budget_ms INTEGER NOT NULL DEFAULT ${DEFAULT_WORK_BUDGET_MS}, + breathing_interval_ms INTEGER NOT NULL DEFAULT ${DEFAULT_BREATHING_INTERVAL_MS}, + rows_exported INTEGER NOT NULL DEFAULT 0, + bytes_written INTEGER NOT NULL DEFAULT 0, + callback_url TEXT, + callback_status TEXT, + error TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + completed_at TEXT +);` + +export type DumpJobStatus = + | 'initializing' + | 'in_progress' + | 'completed' + | 'failed' + +export interface DumpJobRecord { + id: string + status: DumpJobStatus + r2_key: string + upload_id: string | null + tables_json: string + table_index: number + row_offset: number + last_rowid: string | null + parts_json: string + pending_staging_key: string | null + chunk_size: number + work_budget_ms: number + breathing_interval_ms: number + rows_exported: number + bytes_written: number + callback_url: string | null + callback_status: string | null + error: string | null + created_at: string + updated_at: string + completed_at: string | null +} + +export interface DumpJobSummary { + jobId: string + status: DumpJobStatus + key: string + statusUrl: string + downloadUrl: string + tablesTotal: number + tablesExported: number + rowsExported: number + bytesWritten: number + callbackStatus: string | null + error: string | null + createdAt: string + updatedAt: string + completedAt: string | null +} + +export interface DumpExecutionOptions { + async?: boolean + callbackUrl?: string + chunkSize?: number + syncTimeoutMs?: number + maxSyncBytes?: number + workBudgetMs?: number + breathingIntervalMs?: number + minPartSizeBytes?: number + now?: () => number + fetchFn?: typeof fetch +} + +export type DumpSqlExecutor = ( + sql: string, + params?: unknown[] +) => Promise[]> + +interface DumpCursor { + tableIndex: number + rowOffset: number + lastRowId: string | null +} + +interface DumpChunk { + sql: string + rowsInChunk: number + cursor: DumpCursor +} + +const textEncoder = new TextEncoder() +const ROWID_ALIAS = '__sb_rid' + +const INTERNAL_STARBASE_TABLES = new Set([ + DUMP_JOBS_TABLE, + ALARM_SLOTS_TABLE, + 'tmp_cache', + 'tmp_allowlist_queries', + 'tmp_allowlist_rejections', + 'tmp_rls_policies', + 'tmp_query_log', + 'tmp_cron_tasks', +]) + +const SQLITE_RESERVED_KEYWORDS = new Set( + `abort action add after all alter always analyze and as asc attach autoincrement + before begin between by cascade case cast check collate column commit conflict + constraint create cross current current_date current_time current_timestamp + database default deferrable deferred delete desc detach distinct do drop each + else end escape except exclude exclusive exists explain fail filter first + following for foreign from full generated glob group groups having if ignore + immediate in index indexed initially inner insert instead intersect into is + isnull join key last left like limit match materialized natural no not nothing + notnull null nulls of offset on or order others outer over partition plan + pragma preceding primary query raise range recursive references regexp reindex + release rename replace restrict returning right rollback row rows savepoint + select set table temp temporary then ties to transaction trigger unbounded + union unique update using vacuum values view virtual when where window with + without` + .trim() + .split(/\s+/) +) + +export function formatDumpFilename(date: Date = new Date()): string { + const p = (n: number) => String(n).padStart(2, '0') + return ( + `dump_${date.getUTCFullYear()}${p(date.getUTCMonth() + 1)}${p(date.getUTCDate())}` + + `-${p(date.getUTCHours())}${p(date.getUTCMinutes())}${p(date.getUTCSeconds())}.sql` + ) +} + +export function shouldSkipTable(tableName: string): boolean { + return ( + !tableName || + tableName.startsWith('sqlite_') || + tableName.startsWith('_cf_') || + INTERNAL_STARBASE_TABLES.has(tableName) + ) +} + +export function quoteSqlIdentifier(identifier: string): string { + return `"${identifier.replace(/"/g, '""')}"` +} + +export function formatTableIdentifier(table: string): string { + if ( + /^[A-Za-z_][A-Za-z0-9_]*$/.test(table) && + !SQLITE_RESERVED_KEYWORDS.has(table.toLowerCase()) + ) { + return table + } + return quoteSqlIdentifier(table) +} + +function bytesToHex(bytes: Uint8Array): string { + let hex = '' + for (let i = 0; i < bytes.length; i++) { + hex += bytes[i].toString(16).padStart(2, '0') + } + return hex.toUpperCase() +} + +export function formatSqlValue(value: unknown): string { + if (value === null || value === undefined) return 'NULL' + if (typeof value === 'string') return `'${value.replace(/'/g, "''")}'` + if (typeof value === 'boolean') return value ? '1' : '0' + if (typeof value === 'number') + return Number.isFinite(value) ? String(value) : 'NULL' + if (typeof value === 'bigint') return value.toString() + if (value instanceof Uint8Array) return `X'${bytesToHex(value)}'` + if (value instanceof ArrayBuffer) + return `X'${bytesToHex(new Uint8Array(value))}'` + return `'${JSON.stringify(value).replace(/'/g, "''")}'` +} + +export function formatInsertStatement( + table: string, + row: Record, + ridAlias: string = ROWID_ALIAS +): string { + const keys = Object.keys(row).filter((k) => k !== ridAlias) + const values = keys.map((k) => formatSqlValue(row[k])) + // Explicit column list when any key is a JS integer-index property (preserves exact column mapping) + const hasNumericKey = keys.some((k) => /^(0|[1-9]\d*)$/.test(k)) + const colList = hasNumericKey + ? ` (${keys.map(quoteSqlIdentifier).join(', ')})` + : '' + return `INSERT INTO ${formatTableIdentifier(table)}${colList} VALUES (${values.join(', ')});\n` +} + +export function validateCallbackUrl( + rawUrl?: string | null +): string | undefined { + if (!rawUrl || !rawUrl.trim()) return undefined + let parsed: URL + try { + parsed = new URL(rawUrl.trim()) + } catch { + throw new Error('Invalid callback_url: must be a valid HTTP(S) URL.') + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new Error( + 'Invalid callback_url: only http:// and https:// protocols are supported.' + ) + } + return parsed.toString() +} + +function parseRequestOptions( + request?: Request, + overrides?: DumpExecutionOptions +) { + let params: URLSearchParams | undefined + if (request?.url) { + try { + params = new URL(request.url).searchParams + } catch { + params = undefined + } + } + + const intOpt = ( + override: number | undefined, + key: string, + fallback: number, + min: number, + max: number + ) => { + if (override !== undefined) return override + const raw = params?.get(key) + if (!raw || !raw.trim()) return fallback + const n = Number.parseInt(raw, 10) + return Number.isFinite(n) ? Math.min(max, Math.max(min, n)) : fallback + } + + return { + async: + overrides?.async ?? + (params?.get('async') === 'true' || params?.get('async') === '1'), + callbackUrl: validateCallbackUrl( + overrides?.callbackUrl ?? + params?.get('callback_url') ?? + request?.headers.get('X-Callback-Url') + ), + chunkSize: intOpt( + overrides?.chunkSize, + 'chunk_size', + DEFAULT_CHUNK_SIZE, + 1, + 10_000 + ), + syncTimeoutMs: intOpt( + overrides?.syncTimeoutMs, + 'sync_timeout_ms', + DEFAULT_SYNC_TIMEOUT_MS, + 100, + 28_000 + ), + maxSyncBytes: overrides?.maxSyncBytes ?? DEFAULT_MAX_SYNC_BYTES, + workBudgetMs: intOpt( + overrides?.workBudgetMs, + 'work_budget_ms', + DEFAULT_WORK_BUDGET_MS, + 100, + 25_000 + ), + breathingIntervalMs: intOpt( + overrides?.breathingIntervalMs, + 'breathing_ms', + DEFAULT_BREATHING_INTERVAL_MS, + 100, + 30_000 + ), + minPartSizeBytes: + overrides?.minPartSizeBytes ?? DEFAULT_R2_MIN_PART_SIZE_BYTES, + now: overrides?.now ?? Date.now, + fetchFn: overrides?.fetchFn ?? fetch, + } +} + +function concatBytes(chunks: Uint8Array[]): Uint8Array { + if (chunks.length === 0) return new Uint8Array(0) + if (chunks.length === 1) return chunks[0] + let total = 0 + for (const c of chunks) total += c.byteLength + const merged = new Uint8Array(total) + let offset = 0 + for (const c of chunks) { + merged.set(c, offset) + offset += c.byteLength + } + return merged +} + +/** + * R2 Multipart writer with strict equal-sized non-final parts and crash-safe + * versioned R2 staging (`${key}.pending.${bytesWritten}`). + * + * Crash-safety protocol on pause: + * 1. `checkpointPending()` writes a NEW versioned staging key (leaving the old one intact). + * 2. Caller commits the new staging key + SQL cursor in `tmp_dump_jobs`. + * 3. Caller invokes `retirePreviousStaging()` to delete the superseded staging key. + */ +export class R2MultipartDumpWriter { + private pendingChunks: Uint8Array[] = [] + private pendingByteLength = 0 + private retiredStagingKeys: string[] = [] + + private constructor( + private readonly bucket: R2Bucket, + public readonly key: string, + public readonly uploadId: string, + private readonly upload: R2MultipartUpload, + private readonly minPartSizeBytes: number, + private uploadedParts: R2UploadedPart[], + private totalBytesWritten: number, + private activeStagingKey: string | null, + initialPending?: Uint8Array + ) { + if (initialPending && initialPending.byteLength > 0) { + this.pendingChunks.push(initialPending) + this.pendingByteLength = initialPending.byteLength + } + } + + static async create( + bucket: R2Bucket, + key: string, + minPartSizeBytes: number = DEFAULT_R2_MIN_PART_SIZE_BYTES + ): Promise { + const upload = await bucket.createMultipartUpload(key, { + httpMetadata: { + contentType: 'application/x-sqlite3', + contentDisposition: `attachment; filename="${key}"`, + }, + }) + return new R2MultipartDumpWriter( + bucket, + key, + upload.uploadId, + upload, + minPartSizeBytes, + [], + 0, + null + ) + } + + static async resume( + bucket: R2Bucket, + job: DumpJobRecord, + minPartSizeBytes: number = DEFAULT_R2_MIN_PART_SIZE_BYTES + ): Promise { + if (!job.upload_id) { + throw new Error(`Dump job ${job.id} is missing R2 upload_id`) + } + const upload = bucket.resumeMultipartUpload(job.r2_key, job.upload_id) + try { + const uploadedParts = JSON.parse( + job.parts_json || '[]' + ) as R2UploadedPart[] + let initialPending: Uint8Array | undefined + if (job.pending_staging_key) { + const staged = await bucket.get(job.pending_staging_key) + if (!staged) { + throw new Error( + `Missing R2 dump staging buffer: ${job.pending_staging_key}` + ) + } + initialPending = new Uint8Array(await staged.arrayBuffer()) + } + return new R2MultipartDumpWriter( + bucket, + job.r2_key, + job.upload_id, + upload, + minPartSizeBytes, + uploadedParts, + Number(job.bytes_written || 0), + job.pending_staging_key, + initialPending + ) + } catch (err) { + await upload + .abort() + .catch((e) => console.error('Failed to abort R2 upload:', e)) + if (job.pending_staging_key) { + await bucket + .delete(job.pending_staging_key) + .catch((e) => + console.error('Failed to delete R2 staging key:', e) + ) + } + throw err + } + } + + get bytesWritten(): number { + return this.totalBytesWritten + } + + get parts(): R2UploadedPart[] { + return [...this.uploadedParts] + } + + async append(text: string): Promise { + if (!text) return + const bytes = textEncoder.encode(text) + this.pendingChunks.push(bytes) + this.pendingByteLength += bytes.byteLength + this.totalBytesWritten += bytes.byteLength + if (this.pendingByteLength >= this.minPartSizeBytes) { + await this.flushExactParts() + } + } + + private async flushExactParts(): Promise { + if (this.pendingByteLength < this.minPartSizeBytes) return + const buffer = concatBytes(this.pendingChunks) + let offset = 0 + while (buffer.byteLength - offset >= this.minPartSizeBytes) { + const partSlice = buffer.subarray( + offset, + offset + this.minPartSizeBytes + ) + const uploaded = await this.upload.uploadPart( + this.uploadedParts.length + 1, + partSlice + ) + this.uploadedParts.push(uploaded) + offset += this.minPartSizeBytes + } + if (offset < buffer.byteLength) { + const remainder = buffer.slice(offset) + this.pendingChunks = [remainder] + this.pendingByteLength = remainder.byteLength + } else { + this.pendingChunks = [] + this.pendingByteLength = 0 + } + } + + /** + * Stages any sub-part-size pending tail under a versioned R2 key without + * overwriting or deleting the previously committed staging key. + */ + async checkpointPending(): Promise { + if (this.pendingByteLength >= this.minPartSizeBytes) { + await this.flushExactParts() + } + const prevKey = this.activeStagingKey + if (this.pendingByteLength === 0) { + if (prevKey) this.retiredStagingKeys.push(prevKey) + this.activeStagingKey = null + return null + } + + const newKey = `${this.key}.pending.${this.totalBytesWritten}` + if (newKey !== prevKey) { + await this.bucket.put(newKey, concatBytes(this.pendingChunks)) + if (prevKey) this.retiredStagingKeys.push(prevKey) + this.activeStagingKey = newKey + } + return newKey + } + + /** Deletes superseded staging keys ONLY after the SQL checkpoint commits. */ + async retirePreviousStaging(): Promise { + const toDelete = this.retiredStagingKeys.splice(0) + for (const key of toDelete) { + await this.bucket + .delete(key) + .catch((e) => + console.error('Failed to delete superseded R2 staging:', e) + ) + } + } + + /** Retires the active staging key along with any previously superseded staging keys. */ + async retireAllStaging(): Promise { + if (this.activeStagingKey) { + this.retiredStagingKeys.push(this.activeStagingKey) + this.activeStagingKey = null + } + await this.retirePreviousStaging() + } + + async complete(): Promise { + if (this.pendingByteLength > 0 || this.uploadedParts.length === 0) { + const payload = concatBytes(this.pendingChunks) + const uploaded = await this.upload.uploadPart( + this.uploadedParts.length + 1, + payload + ) + this.uploadedParts.push(uploaded) + this.pendingChunks = [] + this.pendingByteLength = 0 + } + // Leave activeStagingKey intact until caller commits terminal 'completed' state in SQL, + // then call `retireAllStaging()` so a failed terminal SQL write can never strand resume. + return await this.upload.complete(this.uploadedParts) + } + + async abort(): Promise { + await this.upload + .abort() + .catch((e) => + console.error('Failed to abort R2 multipart dump:', e) + ) + await this.retireAllStaging() + } +} + +/** + * Shared paginated SQL dump generator consumed by sync downloads, ReadableStream + * fallbacks (no R2), and resumable DO alarm slices. + * + * Uses `CAST(_rowid_ AS TEXT)` keyset pagination on standard tables for O(N) scans, + * exact 64-bit integer precision (> 2^53), immunity to user `rowid` shadowing and + * concurrent-delete offset drift, falling back cleanly to `LIMIT/OFFSET` on + * `WITHOUT ROWID` / `_rowid_`-shadowed tables or mocks without the alias column. + * Adaptively halves page size if an RPC payload size limit is hit on wide rows. + */ +async function* dumpChunks( + executeSql: DumpSqlExecutor, + tables: string[], + initialCursor: DumpCursor, + chunkSize: number +): AsyncGenerator { + const safeLimit = Math.max(1, Math.min(10_000, Math.trunc(chunkSize))) + let { tableIndex, rowOffset, lastRowId } = initialCursor + + while (tableIndex < tables.length) { + const table = tables[tableIndex] + const quoted = quoteSqlIdentifier(table) + const safeCommentTable = table.replace(/[\r\n]+/g, ' ') + let prefixSql = '' + let useKeyset = true + let ridAlias = ROWID_ALIAS + let currentLimit = safeLimit + + const isTableStart = rowOffset === 0 && lastRowId === null + if (isTableStart || lastRowId !== null) { + const schemaResult = await executeSql( + "SELECT sql FROM sqlite_master WHERE type='table' AND name=?;", + [table] + ) + if (schemaResult.length && schemaResult[0].sql) { + const ddl = String(schemaResult[0].sql) + .trim() + .replace(/;+$/, '') + if (isTableStart) { + prefixSql = `\n-- Table: ${safeCommentTable}\n${ddl};\n\n` + } + if (/\b(WITHOUT\s+ROWID|_rowid_)\b/i.test(ddl)) { + useKeyset = false + } + while (ddl.includes(ridAlias)) { + ridAlias += '_' + } + } + } else { + useKeyset = false + } + + while (true) { + let rows: Record[] + while (true) { + const query = useKeyset + ? lastRowId !== null && /^-?\d+$/.test(lastRowId) + ? `SELECT CAST(_rowid_ AS TEXT) AS ${ridAlias}, * FROM ${quoted} WHERE _rowid_ > ${lastRowId} ORDER BY _rowid_ ASC LIMIT ${currentLimit};` + : `SELECT CAST(_rowid_ AS TEXT) AS ${ridAlias}, * FROM ${quoted} ORDER BY _rowid_ ASC LIMIT ${currentLimit};` + : `SELECT * FROM ${quoted} LIMIT ${currentLimit} OFFSET ${Math.max(0, Math.trunc(rowOffset))};` + + try { + rows = await executeSql(query) + break + } catch (err: any) { + const msg = String(err?.message || '') + if (useKeyset && /no such column:\s*_rowid_/i.test(msg)) { + useKeyset = false + continue + } + if ( + currentLimit > 1 && + /rpc|size limit|too large|exceeded|payload/i.test(msg) + ) { + currentLimit = Math.max(1, Math.floor(currentLimit / 2)) + continue + } + throw err + } + } + + let sql = prefixSql + prefixSql = '' + for (const row of rows) { + sql += formatInsertStatement(table, row, ridAlias) + } + + const hasMore = rows.length >= currentLimit + if (hasMore) { + if (useKeyset && ridAlias in rows[rows.length - 1]) { + lastRowId = String(rows[rows.length - 1][ridAlias]) + } else { + useKeyset = false + lastRowId = null + } + rowOffset += rows.length + } else { + sql += '\n' + tableIndex += 1 + rowOffset = 0 + lastRowId = null + } + + yield { + sql, + rowsInChunk: rows.length, + cursor: { tableIndex, rowOffset, lastRowId }, + } + + if (!hasMore) break + } + } +} + +export function toDumpJobSummary(job: DumpJobRecord): DumpJobSummary { + let tablesTotal = 0 + try { + const parsed = JSON.parse(job.tables_json || '[]') + tablesTotal = Array.isArray(parsed) ? parsed.length : 0 + } catch { + tablesTotal = 0 + } + + return { + jobId: job.id, + status: job.status, + key: job.r2_key, + statusUrl: `/export/dump/${job.id}`, + downloadUrl: `/export/dump/${job.id}/download`, + tablesTotal, + tablesExported: Math.min(Number(job.table_index || 0), tablesTotal), + rowsExported: Number(job.rows_exported || 0), + bytesWritten: Number(job.bytes_written || 0), + callbackStatus: job.callback_status ?? null, + error: job.error ?? null, + createdAt: job.created_at, + updatedAt: job.updated_at, + completedAt: job.completed_at ?? null, + } +} + +async function deliverCallback( + job: DumpJobRecord, + fetchFn: typeof fetch +): Promise { + if (!job.callback_url) return null + const eventName = + job.status === 'completed' + ? 'export.dump.completed' + : 'export.dump.failed' + try { + const response = await fetchFn(job.callback_url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Starbase-Event': eventName, + }, + body: JSON.stringify({ + event: eventName, + ...toDumpJobSummary(job), + }), + signal: AbortSignal.timeout(DEFAULT_WEBHOOK_TIMEOUT_MS), + }) + const isOk = + response.ok ?? (response.status >= 200 && response.status < 300) + return isOk ? 'delivered' : `failed:${response.status}` + } catch (err: any) { + console.error('Failed to deliver dump callback:', err) + return `failed:${err?.message || 'network_error'}` + } +} + +function makeDataSourceSqlExecutor( dataSource: DataSource, config: StarbaseDBConfiguration -): Promise { - try { - // Get all table names - const tablesResult = await executeOperation( - [{ sql: "SELECT name FROM sqlite_master WHERE type='table';" }], +): DumpSqlExecutor { + return async (sql: string, params?: unknown[]) => { + const result = await executeOperation( + params && params.length ? [{ sql, params }] : [{ sql }], dataSource, config ) + return (result ?? []) as Record[] + } +} - const tables = tablesResult.map((row: any) => row.name) - let dumpContent = 'SQLite format 3\0' // SQLite file header +async function saveJob( + executeSql: DumpSqlExecutor, + job: DumpJobRecord +): Promise { + const cols = Object.keys(job) as (keyof DumpJobRecord)[] + await executeSql( + `INSERT OR REPLACE INTO ${DUMP_JOBS_TABLE} (${cols.join(', ')}) VALUES (${cols.map(() => '?').join(', ')});`, + cols.map((c) => job[c]) + ) +} - // Iterate through all tables - for (const table of tables) { - // Get table schema - const schemaResult = await executeOperation( - [ - { - sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`, - }, - ], - dataSource, - config +/** + * Atomically reserves a unique `r2_key` in `tmp_dump_jobs` via the `UNIQUE` + * constraint BEFORE creating any R2 multipart upload, preventing concurrent + * same-second dump requests from colliding on the same R2 object or staging key. + */ +async function reserveJobKey( + executeSql: DumpSqlExecutor, + baseJob: DumpJobRecord +): Promise { + const insert = async (candidate: DumpJobRecord) => { + const cols = Object.keys(candidate) as (keyof DumpJobRecord)[] + await executeSql( + `INSERT INTO ${DUMP_JOBS_TABLE} (${cols.join(', ')}) VALUES (${cols.map(() => '?').join(', ')});`, + cols.map((c) => candidate[c]) + ) + return candidate + } + + try { + return await insert(baseJob) + } catch (err: any) { + const msg = String(err?.message || '') + if (!/unique/i.test(msg)) throw err + const suffix = baseJob.id.replace(/[^a-zA-Z0-9]/g, '').slice(0, 8) + return await insert({ + ...baseJob, + r2_key: baseJob.r2_key.replace(/\.sql$/i, `_${suffix}.sql`), + }) + } +} + +async function finishJob( + executeSql: DumpSqlExecutor, + job: DumpJobRecord, + status: 'completed' | 'failed', + now: () => number, + fetchFn: typeof fetch, + errorMessage?: string +): Promise { + const ts = new Date(now()).toISOString() + job.status = status + job.pending_staging_key = null + job.error = errorMessage ?? null + job.updated_at = ts + job.completed_at = ts + + // Commit terminal state FIRST so webhook receivers can download immediately without 409 races. + await saveJob(executeSql, job) + + // Isolate webhook delivery & callback_status bookkeeping so a transient bookkeeping write + // failure can NEVER invalidate an already-finalized backup or emit contradictory events. + if (job.callback_url) { + try { + job.callback_status = await deliverCallback(job, fetchFn) + job.updated_at = new Date(now()).toISOString() + await saveJob(executeSql, job) + } catch (bookkeepingErr) { + console.error( + 'Failed to persist dump callback status:', + bookkeepingErr ) + } + } + + return job +} + +/** + * Executes a single time-bounded slice of an active R2 dump job inside + * `StarbaseDBDurableObject.alarm()`. The DO alarm handler is the sole executor + * of async dump jobs, eliminating concurrent Worker-vs-alarm part races. + */ +export async function processDumpJobSlice(params: { + job: DumpJobRecord + executeSql: DumpSqlExecutor + bucket?: R2Bucket + scheduleAlarm?: (scheduledTime: number) => Promise + minPartSizeBytes?: number + now?: () => number + fetchFn?: typeof fetch +}): Promise { + const { + executeSql, + bucket, + scheduleAlarm, + minPartSizeBytes = DEFAULT_R2_MIN_PART_SIZE_BYTES, + now = Date.now, + fetchFn = fetch, + } = params + + const job: DumpJobRecord = { ...params.job, status: 'in_progress' } + if (!bucket) { + return finishJob( + executeSql, + job, + 'failed', + now, + fetchFn, + 'R2 DUMP_BUCKET binding is not configured on Durable Object' + ) + } + + const tables = JSON.parse(job.tables_json || '[]') as string[] + const sliceStart = now() + let writer: R2MultipartDumpWriter | undefined - if (schemaResult.length) { - const schema = schemaResult[0].sql - dumpContent += `\n-- Table: ${table}\n${schema};\n\n` + try { + writer = await R2MultipartDumpWriter.resume( + bucket, + job, + minPartSizeBytes + ) + + let pausedForBreathing = false + let chunksInSlice = 0 + const gen = dumpChunks( + executeSql, + tables, + { + tableIndex: job.table_index, + rowOffset: job.row_offset, + lastRowId: job.last_rowid, + }, + job.chunk_size + ) + + for await (const chunk of gen) { + await writer.append(chunk.sql) + job.rows_exported += chunk.rowsInChunk + job.table_index = chunk.cursor.tableIndex + job.row_offset = chunk.cursor.rowOffset + job.last_rowid = chunk.cursor.lastRowId + chunksInSlice++ + + if ( + job.table_index < tables.length && + now() - sliceStart >= job.work_budget_ms + ) { + pausedForBreathing = true + await gen.return(undefined) + break } + } - // Get table data - const dataResult = await executeOperation( - [{ sql: `SELECT * FROM ${table};` }], - dataSource, - config - ) + if (pausedForBreathing && chunksInSlice > 0) { + const stagingKey = await writer.checkpointPending() + job.status = 'in_progress' + job.parts_json = JSON.stringify(writer.parts) + job.pending_staging_key = stagingKey + job.bytes_written = writer.bytesWritten + job.updated_at = new Date(now()).toISOString() - for (const row of dataResult) { - const values = Object.values(row).map((value) => - typeof value === 'string' - ? `'${value.replace(/'/g, "''")}'` - : value - ) - dumpContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n` + // Crash-safe order: write new staging -> persist SQL -> retire old staging + await saveJob(executeSql, job) + await writer.retirePreviousStaging() + + if (scheduleAlarm) { + await scheduleAlarm(now() + job.breathing_interval_ms) } + return job + } - dumpContent += '\n' + await writer.complete() + job.parts_json = JSON.stringify(writer.parts) + job.bytes_written = writer.bytesWritten + } catch (error: any) { + if (writer) { + await writer.abort() } + return finishJob( + executeSql, + job, + 'failed', + now, + fetchFn, + error?.message || 'Database dump failed' + ) + } - // Create a Blob from the dump content - const blob = new Blob([dumpContent], { type: 'application/x-sqlite3' }) + // Commit terminal 'completed' state in SQL FIRST, then retire remaining staging keys + const completedJob = await finishJob( + executeSql, + job, + 'completed', + now, + fetchFn + ) + await writer.retireAllStaging() + return completedJob +} - const headers = new Headers({ +async function createR2DumpJob(params: { + tables: string[] + cursor: DumpCursor + rowsExported: number + initialChunks: string[] + executeSql: DumpSqlExecutor + bucket: R2Bucket + scheduleAlarm: (scheduledTime: number) => Promise + alarmDelayMs: number + opts: ReturnType +}): Promise { + const { + tables, + cursor, + rowsExported, + initialChunks, + executeSql, + bucket, + scheduleAlarm, + alarmDelayMs, + opts, + } = params + + const startedMs = opts.now() + const startedIso = new Date(startedMs).toISOString() + const reserved = await reserveJobKey(executeSql, { + id: crypto.randomUUID(), + status: 'initializing', + r2_key: formatDumpFilename(new Date(startedMs)), + upload_id: null, + tables_json: JSON.stringify(tables), + table_index: cursor.tableIndex, + row_offset: cursor.rowOffset, + last_rowid: cursor.lastRowId, + parts_json: '[]', + pending_staging_key: null, + chunk_size: opts.chunkSize, + work_budget_ms: opts.workBudgetMs, + breathing_interval_ms: opts.breathingIntervalMs, + rows_exported: rowsExported, + bytes_written: 0, + callback_url: opts.callbackUrl ?? null, + callback_status: null, + error: null, + created_at: startedIso, + updated_at: startedIso, + completed_at: null, + }) + + let writer: R2MultipartDumpWriter | undefined + try { + writer = await R2MultipartDumpWriter.create( + bucket, + reserved.r2_key, + opts.minPartSizeBytes + ) + for (const chunk of initialChunks) { + await writer.append(chunk) + } + const stagingKey = await writer.checkpointPending() + + const job: DumpJobRecord = { + ...reserved, + status: 'in_progress', + upload_id: writer.uploadId, + parts_json: JSON.stringify(writer.parts), + pending_staging_key: stagingKey, + bytes_written: writer.bytesWritten, + updated_at: new Date(opts.now()).toISOString(), + } + + await saveJob(executeSql, job) + await writer.retirePreviousStaging() + await scheduleAlarm(opts.now() + alarmDelayMs) + return job + } catch (error) { + if (writer) await writer.abort() + await executeSql(`DELETE FROM ${DUMP_JOBS_TABLE} WHERE id = ?;`, [ + reserved.id, + ]).catch((e) => console.error('Failed to clean up failed dump job:', e)) + throw error + } +} + +async function deliverSyncCallback( + opts: ReturnType, + dataSource: DataSource, + tablesCount: number, + rowsExported: number, + bytesWritten: number +): Promise { + if (!opts.callbackUrl) return + const p = opts + .fetchFn(opts.callbackUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Starbase-Event': 'export.dump.completed', + }, + body: JSON.stringify({ + event: 'export.dump.completed', + status: 'completed', + mode: 'sync', + tablesTotal: tablesCount, + tablesExported: tablesCount, + rowsExported, + bytesWritten, + completedAt: new Date(opts.now()).toISOString(), + }), + signal: AbortSignal.timeout(DEFAULT_WEBHOOK_TIMEOUT_MS), + }) + .then(() => undefined) + .catch((err) => { + console.error('Failed to deliver sync dump callback:', err) + }) + + if (dataSource.executionContext?.waitUntil) { + dataSource.executionContext.waitUntil(p) + } else { + await p + } +} + +function createStreamingDumpResponse( + gen: AsyncGenerator, + firstChunk: IteratorResult, + onComplete?: (rowsExported: number, bytesWritten: number) => Promise +): Response { + let headerSent = false + let pendingFirst: IteratorResult | null = firstChunk + let rowsExported = 0 + let bytesWritten = 0 + + const stream = new ReadableStream({ + async pull(controller) { + try { + if (!headerSent) { + headerSent = true + const hdr = textEncoder.encode(SQLITE_DUMP_HEADER) + bytesWritten += hdr.byteLength + controller.enqueue(hdr) + if (pendingFirst && !pendingFirst.done) { + rowsExported += pendingFirst.value.rowsInChunk + if (pendingFirst.value.sql) { + const chunkBytes = textEncoder.encode( + pendingFirst.value.sql + ) + bytesWritten += chunkBytes.byteLength + controller.enqueue(chunkBytes) + } + pendingFirst = null + return + } + if (pendingFirst?.done) { + pendingFirst = null + if (onComplete) { + await onComplete(rowsExported, bytesWritten) + } + controller.close() + return + } + } + const next = await gen.next() + if (next.done) { + if (onComplete) { + await onComplete(rowsExported, bytesWritten) + } + controller.close() + } else { + rowsExported += next.value.rowsInChunk + if (next.value.sql) { + const chunkBytes = textEncoder.encode(next.value.sql) + bytesWritten += chunkBytes.byteLength + controller.enqueue(chunkBytes) + } + } + } catch (err) { + controller.error(err) + } + }, + async cancel() { + await gen.return(undefined) + }, + }) + + return new Response(stream, { + headers: { 'Content-Type': 'application/x-sqlite3', 'Content-Disposition': 'attachment; filename="database_dump.sql"', - }) + }, + }) +} + +export async function dumpDatabaseRoute( + dataSource: DataSource, + config: StarbaseDBConfiguration, + request?: Request, + overrides?: DumpExecutionOptions +): Promise { + let opts: ReturnType + try { + opts = parseRequestOptions(request, overrides) + } catch (err: any) { + return createResponse( + undefined, + err?.message || 'Invalid export dump parameters', + 400 + ) + } + + const isAdmin = config.role === 'admin' + if (opts.async && !isAdmin) { + return createResponse( + undefined, + 'Asynchronous R2 database dumps require admin role.', + 403 + ) + } + + const executeSql = makeDataSourceSqlExecutor(dataSource, config) + const scheduleDumpAlarm = (ts: number) => + dataSource.rpc.scheduleDumpAlarm(ts) - return new Response(blob, { headers }) + try { + const tablesResult = await executeSql( + "SELECT name FROM sqlite_master WHERE type='table';" + ) + const tables = tablesResult + .map((row: any) => String(row.name ?? '')) + .filter((name) => !shouldSkipTable(name)) + + // 1. Explicit async mode: enqueue job for DO alarm execution and return 202 immediately + if (opts.async) { + if (!dataSource.dumpBucket) { + return createResponse( + undefined, + 'R2 DUMP_BUCKET binding is required for asynchronous database dumps.', + 400 + ) + } + const job = await createR2DumpJob({ + tables, + cursor: { tableIndex: 0, rowOffset: 0, lastRowId: null }, + rowsExported: 0, + initialChunks: [SQLITE_DUMP_HEADER], + executeSql, + bucket: dataSource.dumpBucket, + scheduleAlarm: scheduleDumpAlarm, + alarmDelayMs: 0, + opts, + }) + return createResponse(toDumpJobSummary(job), undefined, 202) + } + + // 2. Empty database fast path + if (tables.length === 0) { + await deliverSyncCallback( + opts, + dataSource, + 0, + 0, + textEncoder.encode(SQLITE_DUMP_HEADER).byteLength + ) + return new Response( + new Blob([SQLITE_DUMP_HEADER], { + type: 'application/x-sqlite3', + }), + { + headers: { + 'Content-Type': 'application/x-sqlite3', + 'Content-Disposition': + 'attachment; filename="database_dump.sql"', + }, + } + ) + } + + const gen = dumpChunks( + executeSql, + tables, + { tableIndex: 0, rowOffset: 0, lastRowId: null }, + opts.chunkSize + ) + + // 3. No R2 bucket OR non-admin role (preserves RLS/allowlist): stream via ReadableStream + if (!dataSource.dumpBucket || !isAdmin) { + const firstChunk = await gen.next() + return createStreamingDumpResponse( + gen, + firstChunk, + opts.callbackUrl + ? (rows, bytes) => + deliverSyncCallback( + opts, + dataSource, + tables.length, + rows, + bytes + ) + : undefined + ) + } + + // 4. Admin + R2 bound: fast sync path with automatic promotion to R2 + DO alarm + const startTime = opts.now() + const syncChunks: string[] = [SQLITE_DUMP_HEADER] + let syncBytes = textEncoder.encode(SQLITE_DUMP_HEADER).byteLength + let rowsExported = 0 + let cursor: DumpCursor = { + tableIndex: 0, + rowOffset: 0, + lastRowId: null, + } + + const exceededSyncLimits = () => + opts.now() - startTime >= opts.syncTimeoutMs || + syncBytes >= opts.maxSyncBytes + + for await (const chunk of gen) { + syncChunks.push(chunk.sql) + syncBytes += textEncoder.encode(chunk.sql).byteLength + rowsExported += chunk.rowsInChunk + cursor = chunk.cursor + + if (exceededSyncLimits()) { + await gen.return(undefined) + const job = await createR2DumpJob({ + tables, + cursor, + rowsExported, + initialChunks: syncChunks, + executeSql, + bucket: dataSource.dumpBucket, + scheduleAlarm: scheduleDumpAlarm, + alarmDelayMs: + cursor.tableIndex < tables.length + ? opts.breathingIntervalMs + : 0, + opts, + }) + return createResponse(toDumpJobSummary(job), undefined, 202) + } + } + + await deliverSyncCallback( + opts, + dataSource, + tables.length, + rowsExported, + syncBytes + ) + return new Response( + new Blob(syncChunks, { type: 'application/x-sqlite3' }), + { + headers: { + 'Content-Type': 'application/x-sqlite3', + 'Content-Disposition': + 'attachment; filename="database_dump.sql"', + }, + } + ) } catch (error: any) { console.error('Database Dump Error:', error) return createResponse(undefined, 'Failed to create database dump', 500) } } + +async function loadJob( + jobId: string, + dataSource: DataSource, + config: StarbaseDBConfiguration +): Promise { + if (config.role !== 'admin') { + return createResponse( + undefined, + 'Dump job endpoints require admin role.', + 403 + ) + } + if (!jobId || !jobId.trim()) { + return createResponse(undefined, 'Dump job ID is required', 400) + } + const executeSql = makeDataSourceSqlExecutor(dataSource, config) + const rows = await executeSql( + `SELECT * FROM ${DUMP_JOBS_TABLE} WHERE id = ? LIMIT 1;`, + [jobId.trim()] + ) + if (!rows.length) { + return createResponse(undefined, `Dump job '${jobId}' not found`, 404) + } + return rows[0] as DumpJobRecord +} + +export async function getDumpJobStatusRoute( + jobId: string, + dataSource: DataSource, + config: StarbaseDBConfiguration +): Promise { + try { + const jobOrRes = await loadJob(jobId, dataSource, config) + if (jobOrRes instanceof Response) return jobOrRes + return createResponse(toDumpJobSummary(jobOrRes), undefined, 200) + } catch (error: any) { + console.error('Dump Job Status Error:', error) + return createResponse( + undefined, + 'Failed to retrieve dump job status', + 500 + ) + } +} + +export async function downloadDumpJobRoute( + jobId: string, + dataSource: DataSource, + config: StarbaseDBConfiguration +): Promise { + if (config.role !== 'admin') { + return createResponse( + undefined, + 'Dump job endpoints require admin role.', + 403 + ) + } + if (!dataSource.dumpBucket) { + return createResponse( + undefined, + 'R2 DUMP_BUCKET binding is not configured.', + 503 + ) + } + try { + const jobOrRes = await loadJob(jobId, dataSource, config) + if (jobOrRes instanceof Response) return jobOrRes + if (jobOrRes.status !== 'completed') { + return createResponse( + toDumpJobSummary(jobOrRes), + `Dump job '${jobId}' is not completed (current status: ${jobOrRes.status})`, + 409 + ) + } + + const object = await dataSource.dumpBucket.get(jobOrRes.r2_key) + if (!object) { + return createResponse( + undefined, + `Dump artifact '${jobOrRes.r2_key}' not found in R2 storage`, + 404 + ) + } + + const headers = new Headers({ + 'Content-Type': 'application/x-sqlite3', + 'Content-Disposition': `attachment; filename="${jobOrRes.r2_key}"`, + }) + if (typeof object.size === 'number') { + headers.set('Content-Length', String(object.size)) + } + return new Response(object.body, { headers }) + } catch (error: any) { + console.error('Dump Job Download Error:', error) + return createResponse( + undefined, + 'Failed to download database dump', + 500 + ) + } +} diff --git a/src/handler.ts b/src/handler.ts index 3fa0085..4eab251 100644 --- a/src/handler.ts +++ b/src/handler.ts @@ -6,7 +6,11 @@ import { DataSource } from './types' import { LiteREST } from './literest' import { executeQuery, executeTransaction } from './operation' import { createResponse, QueryRequest, QueryTransactionRequest } from './utils' -import { dumpDatabaseRoute } from './export/dump' +import { + downloadDumpJobRoute, + dumpDatabaseRoute, + getDumpJobStatusRoute, +} from './export/dump' import { exportTableToJsonRoute } from './export/json' import { exportTableToCsvRoute } from './export/csv' import { importDumpRoute } from './import/dump' @@ -120,10 +124,40 @@ export class StarbaseDB { } if (this.getFeature('export')) { - this.app.get('/export/dump', this.isInternalSource, async () => { - return dumpDatabaseRoute(this.dataSource, this.config) + this.app.get('/export/dump', this.isInternalSource, async (c) => { + return dumpDatabaseRoute( + this.dataSource, + this.config, + c.req.raw + ) }) + this.app.get( + '/export/dump/:jobId', + this.isInternalSource, + async (c) => { + const jobId = c.req.param('jobId') + return getDumpJobStatusRoute( + jobId, + this.dataSource, + this.config + ) + } + ) + + this.app.get( + '/export/dump/:jobId/download', + this.isInternalSource, + async (c) => { + const jobId = c.req.param('jobId') + return downloadDumpJobRoute( + jobId, + this.dataSource, + this.config + ) + } + ) + this.app.get( '/export/json/:tableName', this.isInternalSource, diff --git a/src/index.ts b/src/index.ts index 4d08932..a71d184 100644 --- a/src/index.ts +++ b/src/index.ts @@ -55,6 +55,7 @@ export interface Env { AUTH_JWKS_ENDPOINT?: string HYPERDRIVE: Hyperdrive + DUMP_BUCKET?: R2Bucket // ## DO NOT REMOVE: TEMPLATE INTERFACE ## } @@ -121,6 +122,7 @@ export default { ...context, }, executionContext: ctx, + dumpBucket: env.DUMP_BUCKET, } if (env.EXTERNAL_DB_TYPE === 'postgresql') { diff --git a/src/types.ts b/src/types.ts index b0a59f4..f8a95ea 100644 --- a/src/types.ts +++ b/src/types.ts @@ -64,6 +64,7 @@ export type DataSource = { cacheTTL?: number registry?: StarbasePluginRegistry executionContext?: ExecutionContext + dumpBucket?: R2Bucket } export enum RegionLocationHint { diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index 6c35c6f..c7fbea2 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -13,4 +13,5 @@ interface Env { DATABASE_DURABLE_OBJECT: DurableObjectNamespace< import('./src/index').StarbaseDBDurableObject > + DUMP_BUCKET?: R2Bucket } diff --git a/wrangler.toml b/wrangler.toml index 395c4ac..f6b63c3 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -78,3 +78,9 @@ AUTH_JWKS_ENDPOINT = "" # [[hyperdrive]] # binding = "HYPERDRIVE" # id = "" + +# Bind an R2 bucket for resumable large database dumps (>30s or async exports). +# Docs: https://developers.cloudflare.com/r2/api/workers/workers-api-reference/ +# [[r2_buckets]] +# binding = "DUMP_BUCKET" +# bucket_name = "starbasedb-dumps"