Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
452 changes: 452 additions & 0 deletions src/do.test.ts

Large diffs are not rendered by default.

291 changes: 246 additions & 45 deletions src/do.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,19 @@
import { DurableObject } from 'cloudflare:workers'
import {
ALARM_SLOTS_TABLE,
DUMP_JOBS_SCHEMA_SQL,
DUMP_JOBS_TABLE,
DumpJobRecord,
processDumpJobSlice,
} from './export/dump'

const ALARM_SLOTS_SCHEMA_SQL = `
CREATE TABLE IF NOT EXISTS ${ALARM_SLOTS_TABLE} (
slot TEXT PRIMARY KEY,
scheduled_at INTEGER NOT NULL
);`

type AlarmSlot = 'cron' | 'dump'

export class StarbaseDBDurableObject extends DurableObject {
// Durable storage for the SQL database
Expand All @@ -9,6 +24,10 @@ export class StarbaseDBDurableObject extends DurableObject {
public connections = new Map<string, WebSocket>()
// Store the client auth token for requests back to our Worker
private clientAuthToken: string
// Optional R2 bucket binding for large resumable database dumps
private dumpBucket?: R2Bucket
// Cached alarm slots to skip no-op SQL writes on repeated CronPlugin middleware calls
private cachedAlarmSlots?: Record<AlarmSlot, number | null>

/**
* The constructor is invoked once upon creation of the Durable Object, i.e. the first call to
Expand All @@ -20,6 +39,7 @@ export class StarbaseDBDurableObject extends DurableObject {
constructor(ctx: DurableObjectState, env: Env) {
super(ctx, env)
this.clientAuthToken = env.CLIENT_AUTHORIZATION_TOKEN
this.dumpBucket = env.DUMP_BUCKET
this.sql = ctx.storage.sql
this.storage = ctx.storage

Expand Down Expand Up @@ -63,12 +83,15 @@ export class StarbaseDBDurableObject extends DurableObject {
this.executeQuery({ sql: allowlistStatement })
this.executeQuery({ sql: allowlistRejectedStatement })
this.executeQuery({ sql: rlsStatement })
this.sql.exec(DUMP_JOBS_SCHEMA_SQL)
this.sql.exec(ALARM_SLOTS_SCHEMA_SQL)
}

init() {
return {
getAlarm: this.getAlarm.bind(this),
setAlarm: this.setAlarm.bind(this),
scheduleDumpAlarm: this.scheduleDumpAlarm.bind(this),
deleteAlarm: this.deleteAlarm.bind(this),
getStatistics: this.getStatistics.bind(this),
executeQuery: this.executeQuery.bind(this),
Expand All @@ -79,73 +102,251 @@ export class StarbaseDBDurableObject extends DurableObject {
return await this.storage.getAlarm()
}

private async readAlarmSlots(): Promise<Record<AlarmSlot, number | null>> {
if (this.cachedAlarmSlots) {
return { ...this.cachedAlarmSlots }
}
const slots: Record<AlarmSlot, number | null> = {
cron: null,
dump: null,
}
try {
const rows = (await this.executeQuery({
sql: `SELECT slot, scheduled_at FROM ${ALARM_SLOTS_TABLE};`,
isRaw: false,
})) as Record<string, SqlStorageValue>[]
for (const row of rows) {
const slot = String(row.slot ?? '')
const ts = Number(row.scheduled_at)
if (
(slot === 'cron' || slot === 'dump') &&
Number.isFinite(ts)
) {
slots[slot] = ts
}
}
} catch (err) {
console.error('Error reading alarm slots:', err)
}
this.cachedAlarmSlots = { ...slots }
return slots
}

private async upsertAlarmSlot(
slot: AlarmSlot,
scheduledAt: number
): Promise<void> {
await this.executeQuery({
sql: `INSERT OR REPLACE INTO ${ALARM_SLOTS_TABLE} (slot, scheduled_at) VALUES (?, ?);`,
params: [slot, scheduledAt],
})
if (this.cachedAlarmSlots) {
this.cachedAlarmSlots[slot] = scheduledAt
}
}

private async clearAlarmSlot(slot: AlarmSlot): Promise<void> {
await this.executeQuery({
sql: `DELETE FROM ${ALARM_SLOTS_TABLE} WHERE slot = ?;`,
params: [slot],
})
if (this.cachedAlarmSlots) {
this.cachedAlarmSlots[slot] = null
}
}

private async syncPhysicalAlarm(
options?: DurableObjectSetAlarmOptions,
forceRearm = false
): Promise<void> {
const slots = await this.readAlarmSlots()
const candidates = [slots.cron, slots.dump].filter(
(v): v is number => v !== null && Number.isFinite(v)
)
if (candidates.length === 0) return

const earliest = Math.min(...candidates)
if (!forceRearm) {
// Never push back an already-scheduled earlier/due physical alarm (prevents starvation under traffic).
const currentPhysical = await this.storage
.getAlarm()
.catch(() => null)
if (currentPhysical !== null && currentPhysical <= earliest) {
return
}
}
await this.storage.setAlarm(earliest, options)
}

private async scheduleSlotAlarm(
slot: AlarmSlot,
scheduledTime: number | Date,
options?: DurableObjectSetAlarmOptions
): Promise<void> {
const now = Date.now()
const inputTime =
scheduledTime instanceof Date
? scheduledTime.getTime()
: scheduledTime
const finalTime = Math.max(inputTime, now + 1000)

const slots = await this.readAlarmSlots()
if (
slot === 'cron' &&
slots.cron !== null &&
inputTime <= slots.cron &&
finalTime >= slots.cron &&
slots.cron >= now
) {
// Preserve unchanged or final-second cron deadlines without clamping them past
// the already-armed physical alarm, and skip redundant SQLite writes.
await this.syncPhysicalAlarm(options)
return
}

// Preserve an earlier pending dump wake-up so a slower job never delays a faster one
const nextSlotTime =
slot === 'dump' && slots.dump !== null
? Math.min(slots.dump, finalTime)
: finalTime

if (slots[slot] !== nextSlotTime) {
await this.upsertAlarmSlot(slot, nextSlotTime)
}
await this.syncPhysicalAlarm(options)
}

public async setAlarm(
scheduledTime: number | Date,
options?: DurableObjectSetAlarmOptions
): Promise<void> {
try {
const now = Date.now()
const inputTime =
scheduledTime instanceof Date
? scheduledTime.getTime()
: scheduledTime

// Ensure the time is in the future and at least 1 second from now
const minimumTime = now + 1000
const finalTime = Math.max(inputTime, minimumTime)
await this.storage.setAlarm(finalTime, options)
await this.scheduleSlotAlarm('cron', scheduledTime, options)
} catch (e) {
console.error('Error setting alarm: ', e)
throw e
}
}

public deleteAlarm(options?: DurableObjectSetAlarmOptions): Promise<void> {
public async scheduleDumpAlarm(
scheduledTime: number | Date,
options?: DurableObjectSetAlarmOptions
): Promise<void> {
try {
await this.scheduleSlotAlarm('dump', scheduledTime, options)
} catch (e) {
console.error('Error setting dump alarm: ', e)
throw e
}
}

public async deleteAlarm(
options?: DurableObjectSetAlarmOptions
): Promise<void> {
await this.clearAlarmSlot('cron')
const slots = await this.readAlarmSlots()
if (slots.dump !== null) {
await this.syncPhysicalAlarm(options, true)
return
}
return this.storage.deleteAlarm(options)
}

async alarm() {
try {
// Fetch all the tasks that are marked to emit an event for this cycle.
const task = (await this.executeQuery({
sql: 'SELECT * FROM tmp_cron_tasks WHERE is_active = 1;',
isRaw: false,
})) as Record<string, SqlStorageValue>[]

if (!task.length) {
return
}

const now = Date.now()
// Refresh cached slots on alarm entry to reflect durable SQLite state
this.cachedAlarmSlots = undefined
const initialSlots = await this.readAlarmSlots()
const dumpDue = initialSlots.dump === null || initialSlots.dump <= now
let ranDumpJob = false

// 1. Continue any active resumable database dump jobs with breathing intervals
if (dumpDue) {
try {
const firstTask = task[0]
await fetch(`${firstTask.callback_host}/cron/callback`, {
method: 'POST',
headers: {
Authorization: `Bearer ${this.clientAuthToken}`,
'Content-Type': 'application/json',
},
body: JSON.stringify(task ?? []),
})
} catch (error) {
console.error('Failed to call the alarm/cron callback:', error)

// If the callback fails, we should try to reschedule to prevent the chain from breaking
try {
await this.setAlarm(Date.now() + 60000)
} catch (retryError) {
console.error('Failed to set recovery alarm:', retryError)
const activeDumpJobs = (await this.executeQuery({
sql: `SELECT * FROM ${DUMP_JOBS_TABLE} WHERE status = 'in_progress' ORDER BY created_at ASC LIMIT 1;`,
isRaw: false,
})) as Record<string, SqlStorageValue>[]

if (activeDumpJobs.length > 0) {
ranDumpJob = true
await this.clearAlarmSlot('dump')
const job = activeDumpJobs[0] as unknown as DumpJobRecord
const updated = await processDumpJobSlice({
job,
executeSql: async (sql, params) =>
(await this.executeQuery({
sql,
params,
isRaw: false,
})) as Record<string, any>[],
bucket: this.dumpBucket,
scheduleAlarm: async (scheduledTime) => {
await this.scheduleDumpAlarm(scheduledTime)
},
})

if (updated.status !== 'in_progress') {
const remaining = (await this.executeQuery({
sql: `SELECT id FROM ${DUMP_JOBS_TABLE} WHERE status = 'in_progress' LIMIT 1;`,
isRaw: false,
})) as Record<string, SqlStorageValue>[]
if (remaining.length > 0) {
await this.scheduleDumpAlarm(Date.now() + 1000)
}
}
} else if (initialSlots.dump !== null) {
await this.clearAlarmSlot('dump')
}
} catch (dumpErr) {
console.error('Error processing dump job alarm:', dumpErr)
}
} catch (e) {
console.error('There was an error processing an alarm: ', e)
}

// Try to recover by scheduling a retry in 1 minute
// 2. Process active cron plugin tasks ONLY when cron slot is actually due (strict <= now)
const cronDue =
initialSlots.cron !== null ? initialSlots.cron <= now : !ranDumpJob

if (cronDue) {
if (initialSlots.cron !== null) {
await this.clearAlarmSlot('cron')
}
try {
await this.setAlarm(Date.now() + 60000)
} catch (retryError) {
console.error('Failed to set recovery alarm:', retryError)
const task = (await this.executeQuery({
sql: 'SELECT * FROM tmp_cron_tasks WHERE is_active = 1;',
isRaw: false,
})) as Record<string, SqlStorageValue>[]

if (task.length) {
const firstTask = task[0]
await fetch(`${firstTask.callback_host}/cron/callback`, {
method: 'POST',
headers: {
Authorization: `Bearer ${this.clientAuthToken}`,
'Content-Type': 'application/json',
},
body: JSON.stringify(task ?? []),
})
}
} catch (e: any) {
if (
!(
typeof e?.message === 'string' &&
e.message.includes('no such table')
)
) {
console.error('There was an error processing an alarm: ', e)
await this.setAlarm(Date.now() + 60000).catch((retryErr) =>
console.error('Failed to set recovery alarm:', retryErr)
)
}
}
}

// 3. Re-arm the single physical DO alarm to the earliest remaining slot
await this.syncPhysicalAlarm(undefined, true).catch((syncErr) =>
console.error('Failed to synchronize remaining alarm:', syncErr)
)
}

public async getStatistics(): Promise<{
Expand Down
Loading