From 5ed74e5850150482e0c97cf6520f243e15ad556c Mon Sep 17 00:00:00 2001 From: andreikharytonenka-lab Date: Sun, 22 Nov 2026 10:30:22 +0300 Subject: [PATCH] feat(plugins): add external-to-internal data replication plugin (#72) --- dist/plugins.ts | 10 + plugins/data-sync/README.md | 88 ++ plugins/data-sync/index.test.ts | 952 +++++++++++++ plugins/data-sync/index.ts | 1976 +++++++++++++++++++++++++++ plugins/data-sync/meta.json | 27 + plugins/postgres-sync/README.md | 36 + plugins/postgres-sync/index.test.ts | 640 +++++++++ plugins/postgres-sync/index.ts | 477 +++++++ plugins/postgres-sync/meta.json | 16 + 9 files changed, 4222 insertions(+) create mode 100644 plugins/data-sync/README.md create mode 100644 plugins/data-sync/index.test.ts create mode 100644 plugins/data-sync/index.ts create mode 100644 plugins/data-sync/meta.json create mode 100644 plugins/postgres-sync/README.md create mode 100644 plugins/postgres-sync/index.test.ts create mode 100644 plugins/postgres-sync/index.ts create mode 100644 plugins/postgres-sync/meta.json diff --git a/dist/plugins.ts b/dist/plugins.ts index 7dd252a..7b35889 100644 --- a/dist/plugins.ts +++ b/dist/plugins.ts @@ -6,3 +6,13 @@ export { ChangeDataCapturePlugin } from '../plugins/cdc' export { QueryLogPlugin } from '../plugins/query-log' export { ResendPlugin } from '../plugins/resend' export { ClerkPlugin } from '../plugins/clerk' +export { + DataSyncPlugin, + ReplicationPlugin, + DatabaseSyncSource, + MySQLSyncSource, + MySQLSync, + SQLiteSyncSource, + SQLiteSync, +} from '../plugins/data-sync' +export { PostgresSyncSource, PostgresSync } from '../plugins/postgres-sync' diff --git a/plugins/data-sync/README.md b/plugins/data-sync/README.md new file mode 100644 index 0000000..d644eb1 --- /dev/null +++ b/plugins/data-sync/README.md @@ -0,0 +1,88 @@ +# Data Sync Plugin + +The Data Sync Plugin for StarbaseDB replicates tables from an external database source (PostgreSQL, MySQL, Turso, Cloudflare D1, StarbaseDB, Cloudflare Hyperdrive, or Outerbase API) into the internal Durable Object SQLite database, enabling low-latency edge queries against replicated data. + +## Features + +- **Database Sync Source Abstraction**: Works with `PostgresSyncSource`, `MySQLSyncSource`, `SQLiteSyncSource`, or custom `DatabaseSyncSource` implementations. +- **Automatic Schema Discovery & Table Creation**: Introspects external table columns, normalizes types and defaults to SQLite, and creates internal tables automatically. +- **Smart Schema Mapping**: + - Default/public schema tables (e.g. `users`, `person`, `public.users`) are created directly in SQLite as `users` and `person`. + - Custom schema tables (e.g. `analytics.events`) are created as `analytics_events` (or a custom `target_table`). + - Transparently rewrites `public.` and `.
` references in internal SQLite queries via `beforeQuery`. +- **Flexible Incremental & Full-Table Sync**: + - Supports incremental sync via `timestamp_column`, `id_column`, or any custom `cursor_column` / `track_column`. + - Automatically handles tables without `created_at` or `id` columns via full-table pull. +- **Plugin System Metadata Table**: Tracks replication watermarks and errors in `tmp_data_sync_metadata`. + +## Usage + +### With an Explicit Sync Source + +```typescript +import { DataSyncPlugin } from './plugins/data-sync' +import { PostgresSyncSource } from './plugins/postgres-sync' + +const syncStore = new PostgresSyncSource({ + dialect: 'postgresql', +}) + +const plugins = [ + // ... other plugins + new DataSyncPlugin(syncStore, { + sync_interval: 60, // Sync every 60 seconds + tables: [ + 'users', + 'person', + 'analytics.events', + { + name: 'orders', + timestamp_column: 'updated_at', + id_column: 'order_id', + batch_size: 500, + }, + ], + }), +] satisfies StarbasePlugin[] +``` + +### With Automatic Sync Source Selection + +```typescript +import { DataSyncPlugin } from './plugins/data-sync' + +const plugins = [ + new DataSyncPlugin({ + sync_interval: 60, + tables: ['users', 'person'], + }), +] satisfies StarbasePlugin[] +``` + +## Configuration Options + +### Plugin Options (`DataSyncOptions`) + +| Option | Type | Default | Description | +| --------------- | ------------------------------- | ------- | -------------------------------------------------- | +| `sync_interval` | `number` | `60` | Interval in seconds between automatic sync runs | +| `tables` | `(string \| TableSyncConfig)[]` | `[]` | List of table names or detailed table sync configs | +| `auto_sync` | `boolean` | `true` | Whether to automatically sync tables on interval | + +### Table Configuration (`TableSyncConfig`) + +| Option | Type | Default | Description | +| ------------------ | -------- | -------------- | -------------------------------------------------------------- | +| `name` | `string` | Required | Name of the external table to replicate | +| `schema` | `string` | `'public'` | Optional source schema name | +| `target_table` | `string` | Auto-generated | Optional custom SQLite table name | +| `cursor_column` | `string` | `undefined` | Custom monotonic column used to track the latest synced record | +| `timestamp_column` | `string` | `'created_at'` | Timestamp column for incremental replication | +| `id_column` | `string` | `'id'` | ID column for incremental replication | +| `batch_size` | `number` | `1000` | Maximum number of rows fetched per sync batch | + +## Monitoring & Control Endpoints + +- `GET /sync-status` (or `GET /sync/status`): Returns current replication configuration and per-table sync watermarks/errors. +- `POST /sync` (or `POST /sync/trigger`): Triggers an immediate sync of all configured tables (or a single table when `{ "table": "users" }` is passed in the JSON body). +- `GET /sync-data` (or `GET /sync/data`): Previews synced rows and metadata across replicated tables. diff --git a/plugins/data-sync/index.test.ts b/plugins/data-sync/index.test.ts new file mode 100644 index 0000000..60dcf7b --- /dev/null +++ b/plugins/data-sync/index.test.ts @@ -0,0 +1,952 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { StarbaseApp } from '../../src/handler' +import { DataSource } from '../../src/types' +import { PostgresSyncSource } from '../postgres-sync' +import { + ColumnDefinition, + DatabaseSyncSource, + DataSyncPlugin, + MySQLSync, + MySQLSyncSource, + normalizeToQueryResult, + QueryResult, + ReplicationPlugin, + sanitizeSQLiteValue, + SQLiteSync, + SQLiteSyncSource, + TableMetadata, + TableSyncConfig, +} from './index' + +class MockSyncSource extends DatabaseSyncSource { + get dialect(): string { + return 'postgresql' + } + + validateConnection = vi.fn().mockResolvedValue(true) + + getTableSchema = vi.fn().mockResolvedValue([ + { name: 'id', type: 'integer', nullable: false, primaryKey: true }, + { name: 'name', type: 'varchar(255)', nullable: true }, + { + name: 'created_at', + type: 'timestamp with time zone', + nullable: false, + }, + ] as ColumnDefinition[]) + + getIncrementalData = vi.fn().mockResolvedValue({ + rows: [ + [1, 'John', '2023-01-01T00:00:00Z'], + [2, 'Jane', '2023-01-02T00:00:00Z'], + ], + columns: ['id', 'name', 'created_at'], + } as QueryResult) + + mapDataType = vi.fn().mockImplementation((sourceType: string) => { + const typeMap: Record = { + integer: 'INTEGER', + 'varchar(255)': 'TEXT', + 'timestamp with time zone': 'TEXT', + boolean: 'INTEGER', + jsonb: 'TEXT', + text: 'TEXT', + } + return typeMap[sourceType] || 'TEXT' + }) + + validateTableStructure = vi.fn().mockResolvedValue({ + valid: true, + errors: [], + }) +} + +describe('DataSyncPlugin & Sync Sources', () => { + let dataSyncPlugin: DataSyncPlugin + let mockSyncSource: MockSyncSource + let mockDataSource: DataSource + let mockExecuteQuery: ReturnType + let mockApp: StarbaseApp + let registeredRoutes: { + get: Record + post: Record + middlewares: Function[] + } + + beforeEach(() => { + vi.clearAllMocks() + + mockExecuteQuery = vi.fn().mockResolvedValue({ + rows: [], + columns: [], + }) + + mockDataSource = { + source: 'internal', + external: { + dialect: 'postgresql', + connection: { + host: 'localhost', + port: 5432, + database: 'test', + user: 'postgres', + password: 'password', + }, + } as any, + rpc: { + executeQuery: mockExecuteQuery, + } as any, + } + + registeredRoutes = { + get: {}, + post: {}, + middlewares: [], + } + + mockApp = { + use: vi.fn((fn: Function) => { + registeredRoutes.middlewares.push(fn) + }), + get: vi.fn((path: string, handler: Function) => { + registeredRoutes.get[path] = handler + }), + post: vi.fn((path: string, handler: Function) => { + registeredRoutes.post[path] = handler + }), + } as unknown as StarbaseApp + + mockSyncSource = new MockSyncSource({ dialect: 'postgresql' }) + dataSyncPlugin = new DataSyncPlugin(mockSyncSource, { + sync_interval: 60, + tables: ['users', 'orders'], + auto_sync: false, + }) + }) + + afterEach(() => { + dataSyncPlugin.stopPeriodicSync() + }) + + describe('Utility Functions (normalizeToQueryResult & sanitizeSQLiteValue)', () => { + it('should normalize both object arrays and tabular results to QueryResult', () => { + expect(normalizeToQueryResult(null)).toEqual({ + rows: [], + columns: [], + }) + expect(normalizeToQueryResult([])).toEqual({ + rows: [], + columns: [], + }) + + const sdkObjectRows = [ + { id: 1, name: 'Alice' }, + { id: 2, name: 'Bob' }, + ] + expect(normalizeToQueryResult(sdkObjectRows)).toEqual({ + columns: ['id', 'name'], + rows: [ + [1, 'Alice'], + [2, 'Bob'], + ], + }) + + const tabularResult = { + columns: ['id', 'name'], + rows: [[1, 'Alice']], + } + expect(normalizeToQueryResult(tabularResult)).toEqual(tabularResult) + + const wrappedObjectRows = { + rows: [{ id: 10, title: 'Hello' }], + columns: [], + } + expect(normalizeToQueryResult(wrappedObjectRows)).toEqual({ + columns: ['id', 'title'], + rows: [[10, 'Hello']], + }) + }) + + it('should sanitize values properly for Durable Object SQLite', () => { + expect(sanitizeSQLiteValue(undefined)).toBeNull() + expect(sanitizeSQLiteValue(null)).toBeNull() + expect(sanitizeSQLiteValue(true)).toBe(1) + expect(sanitizeSQLiteValue(false)).toBe(0) + expect(sanitizeSQLiteValue(BigInt(42))).toBe(42) + expect(sanitizeSQLiteValue(BigInt('9007199254740993123'))).toBe( + '9007199254740993123' + ) + + const date = new Date('2024-06-01T12:00:00.000Z') + expect(sanitizeSQLiteValue(date)).toBe('2024-06-01T12:00:00.000Z') + + const bytes = new Uint8Array([1, 2, 3]) + expect(sanitizeSQLiteValue(bytes)).toBe(bytes) + + expect(sanitizeSQLiteValue({ role: 'admin', tags: ['a'] })).toBe( + '{"role":"admin","tags":["a"]}' + ) + expect(sanitizeSQLiteValue(['tag1', 'tag2'])).toBe( + '["tag1","tag2"]' + ) + expect(sanitizeSQLiteValue('plain text')).toBe('plain text') + }) + }) + + describe('Initialization & Configuration', () => { + it('should support new DataSyncPlugin(syncSource, opts) (@Brayden syntax)', () => { + const pgSource = new PostgresSyncSource({ dialect: 'postgresql' }) + const plugin = new DataSyncPlugin(pgSource, { + sync_interval: 60, + tables: ['users', 'person'], + }) + + expect(ReplicationPlugin).toBe(DataSyncPlugin) + const config = plugin.getConfig() + expect(config.sync_interval).toBe(60) + expect(config.auto_sync).toBe(true) + expect(config.tables).toEqual([ + { + name: 'users', + timestamp_column: 'created_at', + id_column: 'id', + batch_size: 1000, + }, + { + name: 'person', + timestamp_column: 'created_at', + id_column: 'id', + batch_size: 1000, + }, + ]) + }) + + it('should support new DataSyncPlugin(opts) without explicit syncSource and auto-select dialect adapter', async () => { + const plugin = new DataSyncPlugin({ + sync_interval: 120, + tables: ['users'], + auto_sync: false, + }) + + expect(plugin.getConfig().sync_interval).toBe(120) + + await plugin.setDataSource({ + source: 'internal', + external: { dialect: 'mysql' } as any, + rpc: { executeQuery: mockExecuteQuery } as any, + }) + expect(plugin['syncSource']).toBeInstanceOf(MySQLSyncSource) + + await plugin.setDataSource({ + source: 'internal', + external: { dialect: 'sqlite' } as any, + rpc: { executeQuery: mockExecuteQuery } as any, + }) + expect(plugin['syncSource']).toBeInstanceOf(SQLiteSyncSource) + + await plugin.setDataSource({ + source: 'internal', + external: { dialect: 'postgresql' } as any, + rpc: { executeQuery: mockExecuteQuery } as any, + }) + expect(plugin['syncSource']).toBeInstanceOf(PostgresSyncSource) + }) + + it('should map table names cleanly in internal SQLite without tmp_ prefix or literal dots', () => { + const plugin = new DataSyncPlugin(mockSyncSource, { + tables: [ + 'users', + 'public.products', + 'analytics.events', + { + name: 'orders', + schema: 'sales', + timestamp_column: 'updated_at', + id_column: 'order_id', + batch_size: 500, + }, + { + name: 'legacy_accounts', + target_table: 'accounts_replica', + cursor_column: 'seq_no', + }, + ], + auto_sync: false, + }) + + const tableKeys = Array.from(plugin['tables'].keys()) + expect(tableKeys).toEqual([ + 'users', + 'products', + 'analytics_events', + 'sales_orders', + 'accounts_replica', + ]) + }) + }) + + describe('Plugin Registration & Metadata Initialization', () => { + it('should register routes, validate connection, and create tmp_data_sync_metadata when dataSource is set', async () => { + await dataSyncPlugin.setDataSource(mockDataSource) + await dataSyncPlugin.register(mockApp) + + expect(mockApp.use).toHaveBeenCalled() + expect(mockApp.get).toHaveBeenCalledWith( + '/sync-status', + expect.any(Function) + ) + expect(mockApp.get).toHaveBeenCalledWith( + '/sync/status', + expect.any(Function) + ) + expect(mockApp.post).toHaveBeenCalledWith( + '/sync', + expect.any(Function) + ) + expect(mockApp.post).toHaveBeenCalledWith( + '/sync/trigger', + expect.any(Function) + ) + expect(mockApp.get).toHaveBeenCalledWith( + '/sync-data', + expect.any(Function) + ) + expect(mockApp.get).toHaveBeenCalledWith( + '/sync/data', + expect.any(Function) + ) + + expect(mockSyncSource.validateConnection).toHaveBeenCalled() + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: expect.stringContaining( + 'CREATE TABLE IF NOT EXISTS tmp_data_sync_metadata' + ), + params: [], + }) + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: 'SELECT * FROM tmp_data_sync_metadata', + params: [], + }) + }) + + it('should throw error during register() when pre-set dataSource fails connection validation', async () => { + mockSyncSource.validateConnection.mockResolvedValueOnce(false) + await dataSyncPlugin.setDataSource(mockDataSource) + + await expect(dataSyncPlugin.register(mockApp)).rejects.toThrow( + 'Failed to validate connection to external postgresql database' + ) + }) + + it('should lazily initialize and sync inside Hono middleware when dataSource is provided per request', async () => { + const lazyPlugin = new DataSyncPlugin(mockSyncSource, { + sync_interval: 60, + tables: ['users'], + auto_sync: true, + }) + + await lazyPlugin.register(mockApp) + expect(mockSyncSource.getTableSchema).not.toHaveBeenCalled() + + const middleware = registeredRoutes.middlewares[0] + const mockContext = { + get: vi.fn((key: string) => { + if (key === 'dataSource') return mockDataSource + if (key === 'config') return { role: 'admin' } + return undefined + }), + } + const next = vi.fn().mockResolvedValue(undefined) + + await middleware(mockContext, next) + + expect(next).toHaveBeenCalled() + expect(mockSyncSource.getTableSchema).toHaveBeenCalledWith( + 'users', + undefined + ) + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: expect.stringContaining( + 'CREATE TABLE IF NOT EXISTS tmp_data_sync_metadata' + ), + params: [], + }) + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: expect.stringContaining( + 'CREATE TABLE IF NOT EXISTS users' + ), + params: [], + }) + }) + }) + + describe('End-to-End Table Sync Operations (syncTable & syncAll)', () => { + beforeEach(async () => { + await dataSyncPlugin.setDataSource(mockDataSource) + await dataSyncPlugin.register(mockApp) + mockExecuteQuery.mockClear() + }) + + it('should sync table successfully, creating SQLite table with PRIMARY KEY and updating tmp_data_sync_metadata', async () => { + const result = await dataSyncPlugin.syncTable('users') + + expect(result.table).toBe('users') + expect(result.rowsSynced).toBe(2) + expect(result.lastSyncId).toBe(2) + + expect(mockSyncSource.validateTableStructure).toHaveBeenCalledWith( + 'users', + expect.objectContaining({ name: 'users' }) + ) + expect(mockSyncSource.getTableSchema).toHaveBeenCalledWith( + 'users', + undefined + ) + + // Verify table is created as 'users' (NOT 'tmp_users') with PRIMARY KEY + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: 'CREATE TABLE IF NOT EXISTS users (id INTEGER PRIMARY KEY NOT NULL, name TEXT, created_at TEXT NOT NULL)', + params: [], + }) + + // Verify rows are upserted into 'users' + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: 'INSERT OR REPLACE INTO users (id, name, created_at) VALUES (?, ?, ?)', + params: [1, 'John', '2023-01-01T00:00:00Z'], + }) + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: 'INSERT OR REPLACE INTO users (id, name, created_at) VALUES (?, ?, ?)', + params: [2, 'Jane', '2023-01-02T00:00:00Z'], + }) + + // Verify metadata is persisted in tmp_data_sync_metadata + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: expect.stringContaining( + 'INSERT OR REPLACE INTO tmp_data_sync_metadata' + ), + params: [ + 'users', + 'users', + new Date('2023-01-02T00:00:00Z').getTime(), + '2', + null, + null, + 2, + null, + ], + }) + }) + + it('should sync tables without created_at or id columns seamlessly (like @Brayden person table)', async () => { + const personPlugin = new DataSyncPlugin(mockSyncSource, { + sync_interval: 60, + tables: ['person'], + auto_sync: false, + }) + await personPlugin.setDataSource(mockDataSource) + + mockSyncSource.getTableSchema.mockResolvedValueOnce([ + { name: 'first_name', type: 'text', nullable: true }, + { name: 'last_name', type: 'text', nullable: true }, + ]) + mockSyncSource.getIncrementalData.mockResolvedValueOnce({ + columns: ['first_name', 'last_name'], + rows: [ + ['Brayden', 'Wilmoth'], + ['Ada', 'Lovelace'], + ], + }) + + const syncResult = await personPlugin.syncTable('person') + + expect(syncResult.table).toBe('person') + expect(syncResult.rowsSynced).toBe(2) + + // Verify getIncrementalData was called with timestamp_column and id_column cleared + expect(mockSyncSource.getIncrementalData).toHaveBeenCalledWith( + 'person', + expect.any(Object), + expect.objectContaining({ + name: 'person', + timestamp_column: undefined, + id_column: undefined, + }) + ) + + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: 'CREATE TABLE IF NOT EXISTS person (first_name TEXT, last_name TEXT)', + params: [], + }) + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: 'INSERT OR REPLACE INTO person (first_name, last_name) VALUES (?, ?)', + params: ['Brayden', 'Wilmoth'], + }) + }) + + it('should sanitize Date, boolean, JSON, and undefined values during syncTable insert', async () => { + mockSyncSource.getTableSchema.mockResolvedValueOnce([ + { + name: 'id', + type: 'integer', + nullable: false, + primaryKey: true, + }, + { name: 'is_active', type: 'boolean', nullable: false }, + { name: 'profile', type: 'jsonb', nullable: true }, + { + name: 'created_at', + type: 'timestamp with time zone', + nullable: false, + }, + ]) + + const dateObj = new Date('2025-02-20T15:30:00.000Z') + mockSyncSource.getIncrementalData.mockResolvedValueOnce({ + columns: ['id', 'is_active', 'profile', 'created_at'], + rows: [[10, true, { plan: 'pro' }, dateObj]], + }) + + await dataSyncPlugin.syncTable('users') + + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: 'INSERT OR REPLACE INTO users (id, is_active, profile, created_at) VALUES (?, ?, ?, ?)', + params: [10, 1, '{"plan":"pro"}', '2025-02-20T15:30:00.000Z'], + }) + }) + + it('should track custom cursor_column watermarks in tmp_data_sync_metadata', async () => { + const cursorPlugin = new DataSyncPlugin(mockSyncSource, { + tables: [ + { + name: 'events', + schema: 'analytics', + cursor_column: 'event_seq', + primary_key: 'event_seq', + }, + ], + auto_sync: false, + }) + await cursorPlugin.setDataSource(mockDataSource) + + mockSyncSource.getTableSchema.mockResolvedValueOnce([ + { name: 'event_seq', type: 'integer', nullable: false }, + { name: 'event_type', type: 'text', nullable: false }, + ]) + mockSyncSource.getIncrementalData.mockResolvedValueOnce({ + columns: ['event_seq', 'event_type'], + rows: [ + [501, 'signup'], + [502, 'purchase'], + ], + }) + + const res = await cursorPlugin.syncTable('analytics.events') + + expect(res.table).toBe('analytics_events') + expect(res.lastCursorValue).toBe(502) + expect( + cursorPlugin.getMetadata()['analytics_events'] + ).toMatchObject({ + lastCursorValue: 502, + cursorColumn: 'event_seq', + rowsSynced: 2, + }) + }) + + it('should handle table validation errors and persist sync_errors in tmp_data_sync_metadata', async () => { + mockSyncSource.validateTableStructure.mockResolvedValueOnce({ + valid: false, + errors: ['Table public.users does not exist'], + }) + + await expect(dataSyncPlugin.syncTable('users')).rejects.toThrow( + 'Table structure validation failed for users: Table public.users does not exist' + ) + + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: expect.stringContaining( + 'INSERT OR REPLACE INTO tmp_data_sync_metadata' + ), + params: [ + 'users', + 'users', + 0, + null, + null, + null, + 0, + 'Table structure validation failed for users: Table public.users does not exist', + ], + }) + }) + + it('should sync all configured tables via syncAll() and continue if one table fails', async () => { + mockSyncSource.validateTableStructure + .mockResolvedValueOnce({ + valid: false, + errors: ['Table users broken'], + }) + .mockResolvedValueOnce({ + valid: true, + errors: [], + }) + + const results = await dataSyncPlugin.syncAll() + + expect(results).toHaveLength(2) + expect(results[0].table).toBe('users') + expect(results[0].error).toContain('Table users broken') + expect(results[1].table).toBe('orders') + expect(results[1].rowsSynced).toBe(2) + }) + }) + + describe('Query Transformation (beforeQuery & afterQuery)', () => { + let queryPlugin: DataSyncPlugin + + beforeEach(() => { + queryPlugin = new DataSyncPlugin(mockSyncSource, { + tables: [ + 'users', + 'public.products', + 'analytics.events', + { + name: 'accounts', + target_table: 'custom_accounts', + }, + ], + auto_sync: false, + }) + }) + + it('should omit public. schema prefix in internal SQLite queries', async () => { + const res1 = await queryPlugin.beforeQuery({ + sql: 'SELECT * FROM public.users WHERE id = 1', + params: [], + }) + expect(res1.sql).toBe('SELECT * FROM users WHERE id = 1') + + const res2 = await queryPlugin.beforeQuery({ + sql: 'SELECT * FROM "public"."users" LIMIT 50', + params: [], + }) + expect(res2.sql).toBe('SELECT * FROM "users" LIMIT 50') + + const res3 = await queryPlugin.beforeQuery({ + sql: 'SELECT * FROM public.unconfigured_table', + params: [], + }) + expect(res3.sql).toBe('SELECT * FROM unconfigured_table') + }) + + it('should rewrite custom schema.table references to schema_table in internal SQLite', async () => { + const res1 = await queryPlugin.beforeQuery({ + sql: 'SELECT * FROM analytics.events ORDER BY id DESC', + params: [], + }) + expect(res1.sql).toBe( + 'SELECT * FROM analytics_events ORDER BY id DESC' + ) + + const res2 = await queryPlugin.beforeQuery({ + sql: 'SELECT * FROM "analytics"."events" LIMIT 10', + params: [], + }) + expect(res2.sql).toBe('SELECT * FROM "analytics_events" LIMIT 10') + }) + + it('should rewrite standalone table references when explicit target_table override is configured', async () => { + const res = await queryPlugin.beforeQuery({ + sql: 'SELECT * FROM accounts WHERE active = 1', + params: [], + }) + expect(res.sql).toBe( + 'SELECT * FROM custom_accounts WHERE active = 1' + ) + }) + + it('should preserve single-quoted string literals without modification', async () => { + const res = await queryPlugin.beforeQuery({ + sql: "SELECT * FROM public.users WHERE origin = 'public.users' AND tag = 'analytics.events'", + params: [], + }) + expect(res.sql).toBe( + "SELECT * FROM users WHERE origin = 'public.users' AND tag = 'analytics.events'" + ) + }) + + it('should never rewrite sqlite_master, pragma_table_info, PRAGMA, or external source queries', async () => { + const schemaQuery = + "SELECT type, name, tbl_name, sql FROM sqlite_master WHERE type='table'" + const res1 = await queryPlugin.beforeQuery({ + sql: schemaQuery, + params: [], + }) + expect(res1.sql).toBe(schemaQuery) + + const pragmaQuery = 'PRAGMA table_info(users)' + const res2 = await queryPlugin.beforeQuery({ + sql: pragmaQuery, + params: [], + }) + expect(res2.sql).toBe(pragmaQuery) + + const extRes = await queryPlugin.beforeQuery({ + sql: 'SELECT * FROM public.users', + params: [], + dataSource: { source: 'external' } as any, + }) + expect(extRes.sql).toBe('SELECT * FROM public.users') + }) + + it('should return query result unmodified in afterQuery', async () => { + const rawResult = { rows: [[1, 'John']], columns: ['id', 'name'] } + const out = await queryPlugin.afterQuery({ + sql: 'SELECT * FROM users', + result: rawResult, + isRaw: true, + }) + expect(out).toBe(rawResult) + }) + }) + + describe('HTTP Monitoring & Control Endpoints', () => { + beforeEach(async () => { + await dataSyncPlugin.setDataSource(mockDataSource) + await dataSyncPlugin.register(mockApp) + }) + + it('should return sync configuration and table metadata on GET /sync-status', async () => { + await dataSyncPlugin.syncTable('users') + + const statusHandler = registeredRoutes.get['/sync-status'] + const response = await statusHandler() + const body = await response.json() + + expect(response.status).toBe(200) + expect(body.result.config.sync_interval).toBe(60) + expect(body.result.tables.users).toBeDefined() + expect(body.result.tables.users.lastSyncId).toBe(2) + }) + + it('should trigger manual sync on POST /sync for all tables or a specific table', async () => { + const triggerHandler = registeredRoutes.post['/sync'] + + const singleTableCtx = { + req: { + json: vi.fn().mockResolvedValue({ table: 'users' }), + }, + } + const res1 = await triggerHandler(singleTableCtx) + const body1 = await res1.json() + expect(res1.status).toBe(200) + expect(body1.result.status).toBe('Sync triggered') + + const allTablesCtx = { + req: { + json: vi.fn().mockRejectedValue(new Error('No body')), + }, + } + const res2 = await triggerHandler(allTablesCtx) + const body2 = await res2.json() + expect(res2.status).toBe(200) + expect(body2.result.status).toBe('Sync triggered') + }) + + it('should return replicated data preview on GET /sync-data', async () => { + mockExecuteQuery.mockResolvedValue({ + columns: ['id', 'name'], + rows: [[1, 'John']], + }) + + const dataHandler = registeredRoutes.get['/sync-data'] + const response = await dataHandler() + const body = await response.json() + + expect(response.status).toBe(200) + expect(body.result.tables.users).toEqual({ + columns: ['id', 'name'], + rows: [[1, 'John']], + count: 1, + }) + }) + }) + + describe('MySQLSyncSource Adapter', () => { + let mysqlSource: MySQLSyncSource + let mysqlExec: ReturnType + + beforeEach(async () => { + mysqlExec = vi.fn() + mysqlSource = new MySQLSyncSource({ + schema: 'app_db', + queryExecutor: mysqlExec, + }) + await mysqlSource.setDataSource({ + source: 'internal', + external: { dialect: 'mysql' } as any, + rpc: { executeQuery: vi.fn() } as any, + }) + }) + + it('should validate MySQL connection and map data types', async () => { + expect(MySQLSync).toBe(MySQLSyncSource) + expect(mysqlSource.dialect).toBe('mysql') + + mysqlExec.mockResolvedValueOnce([{ 'VERSION()': '8.0.35' }]) + expect(await mysqlSource.validateConnection()).toBe(true) + + expect(mysqlSource.mapDataType('int(11)')).toBe('INTEGER') + expect(mysqlSource.mapDataType('tinyint(1)')).toBe('INTEGER') + expect(mysqlSource.mapDataType('decimal(10,2)')).toBe('REAL') + expect(mysqlSource.mapDataType('double')).toBe('REAL') + expect(mysqlSource.mapDataType('longblob')).toBe('BLOB') + expect(mysqlSource.mapDataType('varchar(255)')).toBe('TEXT') + expect(mysqlSource.mapDataType('datetime')).toBe('TEXT') + }) + + it('should discover MySQL table schema and fetch incremental data', async () => { + mysqlExec.mockResolvedValueOnce([ + { + COLUMN_NAME: 'id', + DATA_TYPE: 'int', + COLUMN_TYPE: 'int(11)', + IS_NULLABLE: 'NO', + COLUMN_DEFAULT: null, + CHARACTER_MAXIMUM_LENGTH: null, + COLUMN_KEY: 'PRI', + }, + { + COLUMN_NAME: 'email', + DATA_TYPE: 'varchar', + COLUMN_TYPE: 'varchar(255)', + IS_NULLABLE: 'YES', + COLUMN_DEFAULT: null, + CHARACTER_MAXIMUM_LENGTH: 255, + COLUMN_KEY: '', + }, + ]) + + const cols = await mysqlSource.getTableSchema('customers') + expect(cols).toEqual([ + { + name: 'id', + type: 'int', + nullable: false, + defaultValue: undefined, + primaryKey: true, + }, + { + name: 'email', + type: 'varchar', + nullable: true, + defaultValue: undefined, + primaryKey: false, + }, + ]) + + mysqlExec.mockResolvedValueOnce([ + { id: 5, email: 'test@outerbase.com' }, + ]) + const data = await mysqlSource.getIncrementalData( + 'customers', + { lastSyncTimestamp: 0, lastSyncId: 4 }, + { name: 'customers', id_column: 'id', batch_size: 100 } + ) + + expect(data).toEqual({ + columns: ['id', 'email'], + rows: [[5, 'test@outerbase.com']], + }) + expect(mysqlExec).toHaveBeenLastCalledWith( + 'SELECT * FROM customers WHERE id > ? ORDER BY id ASC LIMIT 100', + [4] + ) + }) + }) + + describe('SQLiteSyncSource Adapter (Turso / Cloudflare D1 / StarbaseDB)', () => { + let sqliteSource: SQLiteSyncSource + let sqliteExec: ReturnType + + beforeEach(async () => { + sqliteExec = vi.fn() + sqliteSource = new SQLiteSyncSource({ + queryExecutor: sqliteExec, + }) + await sqliteSource.setDataSource({ + source: 'internal', + external: { dialect: 'sqlite' } as any, + rpc: { executeQuery: vi.fn() } as any, + }) + }) + + it('should validate SQLite connection and map data types', async () => { + expect(SQLiteSync).toBe(SQLiteSyncSource) + expect(sqliteSource.dialect).toBe('sqlite') + + sqliteExec.mockResolvedValueOnce([{ 'sqlite_version()': '3.45.0' }]) + expect(await sqliteSource.validateConnection()).toBe(true) + + expect(sqliteSource.mapDataType('INTEGER')).toBe('INTEGER') + expect(sqliteSource.mapDataType('BIGINT')).toBe('INTEGER') + expect(sqliteSource.mapDataType('REAL')).toBe('REAL') + expect(sqliteSource.mapDataType('FLOAT')).toBe('REAL') + expect(sqliteSource.mapDataType('BLOB')).toBe('BLOB') + expect(sqliteSource.mapDataType('TEXT')).toBe('TEXT') + }) + + it('should discover SQLite table schema via pragma_table_info and fetch incremental data', async () => { + sqliteExec.mockResolvedValueOnce([ + { + name: 'id', + type: 'INTEGER', + notnull: 1, + dflt_value: null, + pk: 1, + }, + { + name: 'title', + type: 'TEXT', + notnull: 0, + dflt_value: "'Untitled'", + pk: 0, + }, + ]) + + const cols = await sqliteSource.getTableSchema('posts') + expect(cols).toEqual([ + { + name: 'id', + type: 'INTEGER', + nullable: false, + defaultValue: undefined, + primaryKey: true, + }, + { + name: 'title', + type: 'TEXT', + nullable: true, + defaultValue: "'Untitled'", + primaryKey: false, + }, + ]) + + sqliteExec.mockResolvedValueOnce([{ id: 1, title: 'Hello Edge' }]) + const data = await sqliteSource.getIncrementalData( + 'posts', + { lastSyncTimestamp: 0 }, + { name: 'posts', id_column: 'id', batch_size: 50 } + ) + + expect(data).toEqual({ + columns: ['id', 'title'], + rows: [[1, 'Hello Edge']], + }) + expect(sqliteExec).toHaveBeenLastCalledWith( + 'SELECT * FROM posts ORDER BY id ASC LIMIT 50', + [] + ) + }) + }) +}) diff --git a/plugins/data-sync/index.ts b/plugins/data-sync/index.ts new file mode 100644 index 0000000..d3a82da --- /dev/null +++ b/plugins/data-sync/index.ts @@ -0,0 +1,1976 @@ +import postgres from 'postgres' +import { StarbaseApp, StarbaseDBConfiguration } from '../../src/handler' +import { executeExternalQuery as executeOperationExternalQuery } from '../../src/operation' +import { StarbasePlugin } from '../../src/plugin' +import { DataSource } from '../../src/types' +import { createResponse } from '../../src/utils' +import { PostgresSyncSource } from '../postgres-sync' + +export { PostgresSyncSource, PostgresSyncSource as PostgresSync } + +export interface TableSyncConfig { + name: string + schema?: string + target_table?: string + targetTable?: string + timestamp_column?: string + timestampColumn?: string + id_column?: string + idColumn?: string + cursor_column?: string + cursorColumn?: string + track_column?: string + trackColumn?: string + primary_key?: string + primaryKey?: string + batch_size?: number + batchSize?: number + hasExplicitSyncColumns?: boolean +} + +export interface SyncConfig { + sync_interval: number + tables: (string | TableSyncConfig)[] + auto_sync?: boolean +} + +export interface DataSyncOptions { + sync_interval?: number + syncInterval?: number + tables?: (string | TableSyncConfig)[] + auto_sync?: boolean + autoSync?: boolean + pathPrefix?: string + dataSource?: DataSource + config?: StarbaseDBConfiguration + syncSource?: DatabaseSyncSource +} + +export interface TableMetadata { + lastSyncTimestamp: number + lastSyncId?: string | number + lastCursorValue?: string | number + cursorColumn?: string + rowsSynced?: number + sync_errors?: string +} + +export interface QueryResult { + rows: any[][] + columns: string[] +} + +export interface ColumnDefinition { + name: string + type: string + nullable?: boolean + defaultValue?: string + primaryKey?: boolean +} + +export interface SyncSourceConfig { + dialect?: string + schema?: string + queryExecutor?: (sql: string, params?: unknown[]) => Promise + [key: string]: any +} + +export interface TableSyncResult { + table: string + sourceTable: string + rowsSynced: number + lastSyncTimestamp: number + lastSyncId?: string | number + lastCursorValue?: string | number + error?: string +} + +/** + * Normalizes query results from either array-of-objects (`Record[]`) + * or raw tabular format (`{ rows: any[][], columns: string[] }`) into a unified `QueryResult`. + */ +export function normalizeToQueryResult(rawResult: any): QueryResult { + if (!rawResult) { + return { rows: [], columns: [] } + } + + if (Array.isArray(rawResult)) { + if (rawResult.length === 0) { + return { rows: [], columns: [] } + } + + if (Array.isArray(rawResult[0])) { + return { + rows: rawResult as any[][], + columns: [], + } + } + + if (typeof rawResult[0] === 'object' && rawResult[0] !== null) { + const columnSet = new Set() + for (const item of rawResult) { + if (item && typeof item === 'object' && !Array.isArray(item)) { + for (const key of Object.keys(item)) { + columnSet.add(key) + } + } + } + const columns = Array.from(columnSet) + const rows = rawResult.map((item) => + columns.map((col) => + item && typeof item === 'object' ? item[col] : undefined + ) + ) + return { rows, columns } + } + + return { rows: [], columns: [] } + } + + if (typeof rawResult === 'object' && Array.isArray(rawResult.rows)) { + const rawRows = rawResult.rows + const rawCols: string[] = Array.isArray(rawResult.columns) + ? rawResult.columns + : [] + + if (rawRows.length === 0 || Array.isArray(rawRows[0])) { + return { + rows: rawRows, + columns: rawCols, + } + } + + if (typeof rawRows[0] === 'object' && rawRows[0] !== null) { + let columns = rawCols + if (columns.length === 0) { + const columnSet = new Set() + for (const item of rawRows) { + if ( + item && + typeof item === 'object' && + !Array.isArray(item) + ) { + for (const key of Object.keys(item)) { + columnSet.add(key) + } + } + } + columns = Array.from(columnSet) + } + const rows = rawRows.map((item: Record) => + columns.map((col) => + item && typeof item === 'object' ? item[col] : undefined + ) + ) + return { rows, columns } + } + } + + return { rows: [], columns: [] } +} + +/** + * Sanitizes parameter values before writing to Cloudflare Durable Object SQLite (`SqlStorage`). + */ +export function sanitizeSQLiteValue(value: unknown): unknown { + if (value === undefined || value === null) { + return null + } + if (typeof value === 'boolean') { + return value ? 1 : 0 + } + if (typeof value === 'bigint') { + const asNumber = Number(value) + return Number.isSafeInteger(asNumber) ? asNumber : value.toString() + } + if (value instanceof Date) { + return value.toISOString() + } + if ( + value instanceof Uint8Array || + (typeof ArrayBuffer !== 'undefined' && value instanceof ArrayBuffer) + ) { + return value + } + if (typeof value === 'object') { + return JSON.stringify(value) + } + return value +} + +/** + * Validates and escapes a SQL identifier (table, schema, or column name) + * to prevent SQL injection while preserving simple identifier formatting. + */ +export function escapeIdentifier(identifier: string): string { + const trimmed = String(identifier).trim() + if (!trimmed || /[\0\r\n;]/.test(trimmed)) { + throw new Error(`Invalid SQL identifier: ${identifier}`) + } + if (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(trimmed)) { + return trimmed + } + return `"${trimmed.replace(/"/g, '""')}"` +} + +/** + * Transforms SQL segments outside single-quoted string literals. + */ +function transformOutsideSingleQuotes( + sql: string, + transformer: (segment: string) => string +): string { + const parts = sql.split(/('(?:''|[^'])*')/g) + return parts + .map((part, index) => (index % 2 === 1 ? part : transformer(part))) + .join('') +} + +/** + * Abstract base class for external database sync sources. + */ +export abstract class DatabaseSyncSource { + protected dataSource?: DataSource + protected starbaseConfig?: StarbaseDBConfiguration + protected config: SyncSourceConfig + + constructor(config: SyncSourceConfig = {}) { + this.config = config + } + + abstract get dialect(): string + + async setDataSource( + dataSource: DataSource, + starbaseConfig?: StarbaseDBConfiguration + ): Promise { + this.dataSource = dataSource + if (starbaseConfig) { + this.starbaseConfig = starbaseConfig + } + } + + protected getExternalDataSource(): DataSource | undefined { + if (!this.dataSource?.external) { + return undefined + } + return this.dataSource + } + + protected getRowValue( + row: any[] | Record, + columns: string[], + columnName: string, + fallbackIndex: number + ): any { + if (row && !Array.isArray(row) && typeof row === 'object') { + if (columnName in row) { + return row[columnName] + } + const lower = columnName.toLowerCase() + const matchedKey = Object.keys(row).find( + (k) => k.toLowerCase() === lower + ) + if (matchedKey !== undefined) { + return row[matchedKey] + } + } + + if (Array.isArray(row)) { + if (columns && columns.length > 0) { + const lower = columnName.toLowerCase() + const colIdx = columns.findIndex( + (c) => c.toLowerCase() === lower + ) + if (colIdx !== -1) { + return row[colIdx] + } + } + return row[fallbackIndex] + } + + return undefined + } + + /** + * Executes a query against the external data source. + */ + public async executeExternalQuery( + sql: string, + params: unknown[] = [] + ): Promise { + if (typeof this.config.queryExecutor === 'function') { + const raw = await this.config.queryExecutor(sql, params) + return normalizeToQueryResult(raw) + } + + const externalDs = this.getExternalDataSource() + if (!externalDs || !externalDs.external) { + throw new Error('External data source not configured') + } + + const extAny = externalDs.external as any + if (typeof extAny.queryExecutor === 'function') { + const raw = await extAny.queryExecutor(sql, params) + return normalizeToQueryResult(raw) + } + + if (typeof extAny.rpc?.executeQuery === 'function') { + const raw = await extAny.rpc.executeQuery({ sql, params }) + return normalizeToQueryResult(raw) + } + + const isMockRpc = + typeof externalDs.rpc?.executeQuery === 'function' && + 'mock' in (externalDs.rpc.executeQuery as any) + + if (this.starbaseConfig?.outerbaseApiKey) { + const raw = await executeOperationExternalQuery({ + sql, + params, + dataSource: externalDs, + config: this.starbaseConfig, + }) + return normalizeToQueryResult(raw) + } + + if (isMockRpc) { + const raw = await externalDs.rpc.executeQuery({ sql, params }) + return normalizeToQueryResult(raw) + } + + if ( + 'connectionString' in externalDs.external && + externalDs.external.connectionString + ) { + const pgClient = postgres(externalDs.external.connectionString, { + max: 5, + fetch_types: false, + }) + + try { + let queryStr = sql + if (params.length > 0 && queryStr.includes('?')) { + let paramIdx = 1 + queryStr = transformOutsideSingleQuotes(queryStr, (seg) => + seg.replace(/\?/g, () => `$${paramIdx++}`) + ) + } + + const raw = await pgClient.unsafe(queryStr, params as any[]) + if (externalDs.executionContext) { + externalDs.executionContext.waitUntil(pgClient.end()) + } else { + await pgClient.end() + } + return normalizeToQueryResult(raw) + } catch (error) { + await pgClient.end().catch(() => {}) + throw error + } + } + + const raw = await executeOperationExternalQuery({ + sql, + params, + dataSource: externalDs, + config: this.starbaseConfig || { role: 'admin' }, + }) + return normalizeToQueryResult(raw) + } + + abstract validateConnection(): Promise + abstract getTableSchema( + tableName: string, + schema?: string + ): Promise + abstract getIncrementalData( + tableName: string, + lastSync: TableMetadata, + tableConfig: TableSyncConfig + ): Promise + abstract mapDataType(sourceType: string): string + abstract validateTableStructure( + tableName: string, + tableConfig: TableSyncConfig + ): Promise<{ valid: boolean; errors: string[] }> +} + +/** + * MySQL Sync Source adapter for pulling schema and data from external MySQL databases. + */ +export class MySQLSyncSource extends DatabaseSyncSource { + private schema?: string + + constructor(config: SyncSourceConfig = {}) { + super({ + ...config, + dialect: 'mysql', + }) + this.schema = config.schema + } + + get dialect(): string { + return 'mysql' + } + + async validateConnection(): Promise { + const externalDataSource = this.getExternalDataSource() + if ( + !externalDataSource || + externalDataSource.external?.dialect !== 'mysql' + ) { + return false + } + + try { + await this.executeExternalQuery('SELECT VERSION()') + return true + } catch (error) { + console.error('MySQL connection validation failed:', error) + return false + } + } + + async getTableSchema( + tableName: string, + schema?: string + ): Promise { + const externalDataSource = this.getExternalDataSource() + if (!externalDataSource) { + throw new Error('External data source not configured') + } + + let targetSchema = + schema || + this.schema || + (externalDataSource.external as any)?.database + let cleanTableName = tableName.replace(/^tmp_/, '') + if (cleanTableName.includes('.')) { + const parts = cleanTableName.split('.', 2) + targetSchema = schema || parts[0] + cleanTableName = parts[1] + } + + const result = await this.executeExternalQuery( + `SELECT + COLUMN_NAME, + DATA_TYPE, + COLUMN_TYPE, + IS_NULLABLE, + COLUMN_DEFAULT, + CHARACTER_MAXIMUM_LENGTH, + COLUMN_KEY + FROM information_schema.columns + WHERE TABLE_SCHEMA = COALESCE(?, DATABASE()) AND TABLE_NAME = ? + ORDER BY ORDINAL_POSITION`, + [targetSchema || null, cleanTableName] + ) + + return (result.rows || []).map((row: any[]) => { + const name = String( + this.getRowValue(row, result.columns, 'COLUMN_NAME', 0) + ) + const dataType = String( + this.getRowValue(row, result.columns, 'DATA_TYPE', 1) || 'text' + ) + const isNullable = this.getRowValue( + row, + result.columns, + 'IS_NULLABLE', + 3 + ) + const colDefault = this.getRowValue( + row, + result.columns, + 'COLUMN_DEFAULT', + 4 + ) + const colKey = this.getRowValue( + row, + result.columns, + 'COLUMN_KEY', + 6 + ) + + return { + name, + type: dataType, + nullable: + isNullable === 'YES' || + isNullable === true || + isNullable === 1, + defaultValue: + colDefault !== null && colDefault !== undefined + ? String(colDefault) + : undefined, + primaryKey: colKey === 'PRI', + } + }) + } + + mapDataType(mysqlType: string): string { + const baseType = mysqlType + .toLowerCase() + .replace(/\(.*\)/, '') + .trim() + const typeMap: Record = { + int: 'INTEGER', + integer: 'INTEGER', + tinyint: 'INTEGER', + smallint: 'INTEGER', + mediumint: 'INTEGER', + bigint: 'INTEGER', + bit: 'INTEGER', + bool: 'INTEGER', + boolean: 'INTEGER', + year: 'INTEGER', + float: 'REAL', + double: 'REAL', + 'double precision': 'REAL', + decimal: 'REAL', + dec: 'REAL', + numeric: 'REAL', + fixed: 'REAL', + real: 'REAL', + blob: 'BLOB', + tinyblob: 'BLOB', + mediumblob: 'BLOB', + longblob: 'BLOB', + binary: 'BLOB', + varbinary: 'BLOB', + varchar: 'TEXT', + char: 'TEXT', + text: 'TEXT', + tinytext: 'TEXT', + mediumtext: 'TEXT', + longtext: 'TEXT', + json: 'TEXT', + date: 'TEXT', + datetime: 'TEXT', + timestamp: 'TEXT', + time: 'TEXT', + enum: 'TEXT', + set: 'TEXT', + } + + return typeMap[baseType] || 'TEXT' + } + + async validateTableStructure( + tableName: string, + tableConfig: TableSyncConfig + ): Promise<{ valid: boolean; errors: string[] }> { + const errors: string[] = [] + const sourceTable = tableConfig.name || tableName + const columns = await this.getTableSchema( + sourceTable, + tableConfig.schema + ) + + if (columns.length === 0) { + return { + valid: false, + errors: [`Table ${sourceTable} does not exist`], + } + } + + if (tableConfig.hasExplicitSyncColumns === false) { + return { valid: true, errors: [] } + } + + const cursorCol = + tableConfig.cursor_column || + tableConfig.cursorColumn || + tableConfig.track_column || + tableConfig.trackColumn + + if ( + cursorCol && + (tableConfig as any).explicitCursorColumn !== false && + !columns.some((c) => c.name === cursorCol) + ) { + errors.push( + `Cursor column ${cursorCol} not found in table ${sourceTable}` + ) + } + + const tsCol = + tableConfig.timestamp_column || tableConfig.timestampColumn + if ( + tsCol && + (tableConfig as any).explicitTimestampColumn !== false && + !columns.some((c) => c.name === tsCol) + ) { + errors.push( + `Timestamp column ${tsCol} not found in table ${sourceTable}` + ) + } + + const idCol = tableConfig.id_column || tableConfig.idColumn + if ( + idCol && + (tableConfig as any).explicitIdColumn !== false && + !columns.some((c) => c.name === idCol) + ) { + errors.push(`ID column ${idCol} not found in table ${sourceTable}`) + } + + return { + valid: errors.length === 0, + errors, + } + } + + async getIncrementalData( + tableName: string, + lastSync: TableMetadata, + tableConfig: TableSyncConfig + ): Promise { + const externalDataSource = this.getExternalDataSource() + if (!externalDataSource) { + throw new Error('External data source not configured') + } + + const sourceTable = tableConfig.name || tableName.replace(/^tmp_/, '') + const safeSourceTable = escapeIdentifier(sourceTable) + const qualifiedTable = + tableConfig.schema && tableConfig.schema !== 'public' + ? `${escapeIdentifier(tableConfig.schema)}.${safeSourceTable}` + : safeSourceTable + + const cursorCol = + tableConfig.cursor_column || + tableConfig.cursorColumn || + tableConfig.track_column || + tableConfig.trackColumn + const timestampCol = + tableConfig.timestamp_column || tableConfig.timestampColumn + const idCol = tableConfig.id_column || tableConfig.idColumn + + let query = `SELECT * FROM ${qualifiedTable}` + const conditions: string[] = [] + const params: any[] = [] + + if (cursorCol) { + if ( + lastSync.lastCursorValue !== undefined && + lastSync.lastCursorValue !== null + ) { + conditions.push(`${escapeIdentifier(cursorCol)} > ?`) + params.push(lastSync.lastCursorValue) + } + } else { + if (lastSync.lastSyncTimestamp && timestampCol) { + conditions.push(`${escapeIdentifier(timestampCol)} > ?`) + params.push(new Date(lastSync.lastSyncTimestamp).toISOString()) + } + if ( + lastSync.lastSyncId !== undefined && + lastSync.lastSyncId !== null && + idCol + ) { + conditions.push(`${escapeIdentifier(idCol)} > ?`) + params.push(lastSync.lastSyncId) + } + } + + if (conditions.length > 0) { + query += ' WHERE ' + conditions.join(' AND ') + } + + const orderColumns = ( + cursorCol ? [cursorCol] : [timestampCol, idCol].filter(Boolean) + ).map((c) => escapeIdentifier(c!)) + if (orderColumns.length > 0) { + query += ' ORDER BY ' + orderColumns.join(', ') + ' ASC' + } + + const batchSize = Math.max( + 1, + Math.floor( + Number(tableConfig.batch_size || tableConfig.batchSize || 1000) + ) + ) + query += ` LIMIT ${batchSize}` + + return await this.executeExternalQuery(query, params) + } +} + +export { MySQLSyncSource as MySQLSync } + +/** + * SQLite Sync Source adapter for pulling schema and data from external SQLite databases + * (Turso, Cloudflare D1, or external StarbaseDB). + */ +export class SQLiteSyncSource extends DatabaseSyncSource { + constructor(config: SyncSourceConfig = {}) { + super({ + ...config, + dialect: 'sqlite', + }) + } + + get dialect(): string { + return 'sqlite' + } + + async validateConnection(): Promise { + const externalDataSource = this.getExternalDataSource() + if ( + !externalDataSource || + externalDataSource.external?.dialect !== 'sqlite' + ) { + return false + } + + try { + await this.executeExternalQuery('SELECT sqlite_version()') + return true + } catch (error) { + console.error('SQLite connection validation failed:', error) + return false + } + } + + async getTableSchema(tableName: string): Promise { + const externalDataSource = this.getExternalDataSource() + if (!externalDataSource) { + throw new Error('External data source not configured') + } + + let cleanTableName = tableName.replace(/^tmp_/, '') + if (cleanTableName.includes('.')) { + cleanTableName = cleanTableName.split('.', 2)[1] + } + + const result = await this.executeExternalQuery( + `SELECT name, type, "notnull", dflt_value, pk FROM pragma_table_info(?)`, + [cleanTableName] + ) + + return (result.rows || []).map((row: any[]) => { + const hasCidOffset = + result.columns.length === 0 && + row.length >= 6 && + typeof row[0] === 'number' + const name = String( + this.getRowValue( + row, + result.columns, + 'name', + hasCidOffset ? 1 : 0 + ) + ) + const type = String( + this.getRowValue( + row, + result.columns, + 'type', + hasCidOffset ? 2 : 1 + ) || 'TEXT' + ) + const notNullVal = this.getRowValue( + row, + result.columns, + 'notnull', + hasCidOffset ? 3 : 2 + ) + const defaultVal = this.getRowValue( + row, + result.columns, + 'dflt_value', + hasCidOffset ? 4 : 3 + ) + const pkVal = this.getRowValue( + row, + result.columns, + 'pk', + hasCidOffset ? 5 : 4 + ) + + return { + name, + type, + nullable: + notNullVal === 0 || + notNullVal === false || + notNullVal === '0' || + notNullVal === undefined, + defaultValue: + defaultVal !== null && defaultVal !== undefined + ? String(defaultVal) + : undefined, + primaryKey: pkVal === 1 || pkVal === true || Number(pkVal) > 0, + } + }) + } + + mapDataType(sqliteType: string): string { + const upper = sqliteType.toUpperCase().trim() + if (upper.includes('INT') || upper === 'BOOLEAN' || upper === 'BOOL') { + return 'INTEGER' + } + if ( + upper.includes('REAL') || + upper.includes('FLOA') || + upper.includes('DOUB') || + upper.includes('NUMERIC') || + upper.includes('DECIMAL') + ) { + return 'REAL' + } + if (upper.includes('BLOB') || upper.includes('BINARY')) { + return 'BLOB' + } + return 'TEXT' + } + + async validateTableStructure( + tableName: string, + tableConfig: TableSyncConfig + ): Promise<{ valid: boolean; errors: string[] }> { + const errors: string[] = [] + const sourceTable = tableConfig.name || tableName + const columns = await this.getTableSchema(sourceTable) + + if (columns.length === 0) { + return { + valid: false, + errors: [`Table ${sourceTable} does not exist`], + } + } + + if (tableConfig.hasExplicitSyncColumns === false) { + return { valid: true, errors: [] } + } + + const cursorCol = + tableConfig.cursor_column || + tableConfig.cursorColumn || + tableConfig.track_column || + tableConfig.trackColumn + + if ( + cursorCol && + (tableConfig as any).explicitCursorColumn !== false && + !columns.some((c) => c.name === cursorCol) + ) { + errors.push( + `Cursor column ${cursorCol} not found in table ${sourceTable}` + ) + } + + const tsCol = + tableConfig.timestamp_column || tableConfig.timestampColumn + if ( + tsCol && + (tableConfig as any).explicitTimestampColumn !== false && + !columns.some((c) => c.name === tsCol) + ) { + errors.push( + `Timestamp column ${tsCol} not found in table ${sourceTable}` + ) + } + + const idCol = tableConfig.id_column || tableConfig.idColumn + if ( + idCol && + (tableConfig as any).explicitIdColumn !== false && + !columns.some((c) => c.name === idCol) + ) { + errors.push(`ID column ${idCol} not found in table ${sourceTable}`) + } + + return { + valid: errors.length === 0, + errors, + } + } + + async getIncrementalData( + tableName: string, + lastSync: TableMetadata, + tableConfig: TableSyncConfig + ): Promise { + const externalDataSource = this.getExternalDataSource() + if (!externalDataSource) { + throw new Error('External data source not configured') + } + + const sourceTable = tableConfig.name || tableName.replace(/^tmp_/, '') + const cursorCol = + tableConfig.cursor_column || + tableConfig.cursorColumn || + tableConfig.track_column || + tableConfig.trackColumn + const timestampCol = + tableConfig.timestamp_column || tableConfig.timestampColumn + const idCol = tableConfig.id_column || tableConfig.idColumn + + let query = `SELECT * FROM ${escapeIdentifier(sourceTable)}` + const conditions: string[] = [] + const params: any[] = [] + + if (cursorCol) { + if ( + lastSync.lastCursorValue !== undefined && + lastSync.lastCursorValue !== null + ) { + conditions.push(`${escapeIdentifier(cursorCol)} > ?`) + params.push(lastSync.lastCursorValue) + } + } else { + if (lastSync.lastSyncTimestamp && timestampCol) { + conditions.push(`${escapeIdentifier(timestampCol)} > ?`) + params.push(new Date(lastSync.lastSyncTimestamp).toISOString()) + } + if ( + lastSync.lastSyncId !== undefined && + lastSync.lastSyncId !== null && + idCol + ) { + conditions.push(`${escapeIdentifier(idCol)} > ?`) + params.push(lastSync.lastSyncId) + } + } + + if (conditions.length > 0) { + query += ' WHERE ' + conditions.join(' AND ') + } + + const orderColumns = ( + cursorCol ? [cursorCol] : [timestampCol, idCol].filter(Boolean) + ).map((c) => escapeIdentifier(c!)) + if (orderColumns.length > 0) { + query += ' ORDER BY ' + orderColumns.join(', ') + ' ASC' + } + + const batchSize = Math.max( + 1, + Math.floor( + Number(tableConfig.batch_size || tableConfig.batchSize || 1000) + ) + ) + query += ` LIMIT ${batchSize}` + + return await this.executeExternalQuery(query, params) + } +} + +export { SQLiteSyncSource as SQLiteSync } + +/** + * StarbaseDB plugin that replicates data from an external data source + * (PostgreSQL, MySQL, Turso, Cloudflare D1, StarbaseDB, Hyperdrive, or Outerbase API) + * into the internal Durable Object SQLite database. + */ +export class DataSyncPlugin extends StarbasePlugin { + public pathPrefix: string = '/sync' + private syncInterval: number + private tables: Map + private schemaTableMap: Map + private autoSync: boolean + private dataSource?: DataSource + private starbaseConfig?: StarbaseDBConfiguration + private syncSource: DatabaseSyncSource + private hasExplicitSyncSource: boolean + private timerId?: ReturnType + private syncMetadata: Map = new Map() + private lastSyncAttempt: Map = new Map() + private isMetadataInitialized: boolean = false + + constructor( + syncSourceOrOpts?: DatabaseSyncSource | DataSyncOptions, + opts?: DataSyncOptions + ) { + super('starbasedb:data-sync', { requiresAuth: true }, '/sync') + + const isSyncSource = + syncSourceOrOpts instanceof DatabaseSyncSource || + Boolean( + syncSourceOrOpts && + typeof syncSourceOrOpts === 'object' && + 'validateConnection' in syncSourceOrOpts && + 'getTableSchema' in syncSourceOrOpts + ) + + let resolvedOpts: DataSyncOptions + if (isSyncSource) { + this.syncSource = syncSourceOrOpts as DatabaseSyncSource + this.hasExplicitSyncSource = true + resolvedOpts = opts || {} + } else { + resolvedOpts = (syncSourceOrOpts as DataSyncOptions) || opts || {} + this.hasExplicitSyncSource = Boolean(resolvedOpts.syncSource) + this.syncSource = + resolvedOpts.syncSource || new PostgresSyncSource() + } + + if (resolvedOpts.pathPrefix) { + this.pathPrefix = resolvedOpts.pathPrefix + } + + const intervalSeconds = + resolvedOpts.sync_interval ?? resolvedOpts.syncInterval ?? 60 + this.syncInterval = intervalSeconds * 1000 + this.autoSync = resolvedOpts.auto_sync ?? resolvedOpts.autoSync ?? true + this.tables = new Map() + this.schemaTableMap = new Map() + + const configuredTables = resolvedOpts.tables || [] + for (const table of configuredTables) { + this.registerTableConfig(table) + } + + if (resolvedOpts.dataSource) { + this.dataSource = resolvedOpts.dataSource + this.starbaseConfig = resolvedOpts.config + } + } + + private registerTableConfig(table: string | TableSyncConfig): void { + if (typeof table === 'string') { + let parsedSchema: string | undefined + let parsedName = table.trim() + + if (parsedName.includes('.')) { + const parts = parsedName.split('.', 2) + parsedSchema = parts[0] + parsedName = parts[1] + } + + const configObj: TableSyncConfig = { + name: parsedName, + ...(parsedSchema && parsedSchema !== 'public' + ? { schema: parsedSchema } + : {}), + timestamp_column: 'created_at', + id_column: 'id', + batch_size: 1000, + } + + Object.defineProperties(configObj, { + hasExplicitSyncColumns: { + value: false, + enumerable: false, + writable: true, + configurable: true, + }, + explicitTimestampColumn: { + value: false, + enumerable: false, + writable: true, + configurable: true, + }, + explicitIdColumn: { + value: false, + enumerable: false, + writable: true, + configurable: true, + }, + explicitCursorColumn: { + value: false, + enumerable: false, + writable: true, + configurable: true, + }, + }) + + const sqliteTableName = this.getSQLiteTableName( + configObj, + parsedSchema + ) + this.tables.set(sqliteTableName, configObj) + this.registerSchemaMapping( + parsedSchema, + parsedName, + sqliteTableName + ) + return + } + + let parsedSchema = table.schema + let parsedName = table.name.trim() + if (!parsedSchema && parsedName.includes('.')) { + const parts = parsedName.split('.', 2) + parsedSchema = parts[0] + parsedName = parts[1] + } + + const rawTsCol = table.timestamp_column ?? table.timestampColumn + const rawIdCol = table.id_column ?? table.idColumn + const rawCursorCol = + table.cursor_column ?? + table.cursorColumn ?? + table.track_column ?? + table.trackColumn + const rawTargetTable = table.target_table ?? table.targetTable + const rawPrimaryKey = table.primary_key ?? table.primaryKey + const rawBatchSize = table.batch_size ?? table.batchSize ?? 1000 + + const hasExplicitTs = Boolean(rawTsCol) + const hasExplicitId = Boolean(rawIdCol) + const hasExplicitCursor = Boolean(rawCursorCol) + const hasExplicit = + table.hasExplicitSyncColumns !== undefined + ? table.hasExplicitSyncColumns + : hasExplicitTs || hasExplicitId || hasExplicitCursor + + const configObj: TableSyncConfig = { + name: parsedName, + ...(parsedSchema ? { schema: parsedSchema } : {}), + ...(rawTargetTable ? { target_table: rawTargetTable } : {}), + ...(rawCursorCol ? { cursor_column: rawCursorCol } : {}), + ...(rawPrimaryKey ? { primary_key: rawPrimaryKey } : {}), + timestamp_column: rawTsCol || 'created_at', + id_column: rawIdCol || 'id', + batch_size: rawBatchSize, + } + + Object.defineProperties(configObj, { + hasExplicitSyncColumns: { + value: hasExplicit, + enumerable: false, + writable: true, + configurable: true, + }, + explicitTimestampColumn: { + value: hasExplicitTs, + enumerable: false, + writable: true, + configurable: true, + }, + explicitIdColumn: { + value: hasExplicitId, + enumerable: false, + writable: true, + configurable: true, + }, + explicitCursorColumn: { + value: hasExplicitCursor, + enumerable: false, + writable: true, + configurable: true, + }, + }) + + const sqliteTableName = this.getSQLiteTableName(configObj, parsedSchema) + this.tables.set(sqliteTableName, configObj) + this.registerSchemaMapping(parsedSchema, parsedName, sqliteTableName) + } + + public getSQLiteTableName( + config: TableSyncConfig, + explicitSchema?: string + ): string { + const targetOverride = config.target_table || config.targetTable + if (targetOverride) { + return targetOverride + } + + const schema = explicitSchema || config.schema + if (!schema || schema === 'public' || schema === 'main') { + return config.name + } + + return `${schema}_${config.name}` + } + + private registerSchemaMapping( + schema: string | undefined, + tableName: string, + sqliteTableName: string + ): void { + const effectiveSchema = schema || 'public' + this.schemaTableMap.set( + `${effectiveSchema}.${tableName}`.toLowerCase(), + sqliteTableName + ) + this.schemaTableMap.set( + `public.${tableName}`.toLowerCase(), + sqliteTableName + ) + } + + private resolveTableEntry( + tableName: string + ): { sqliteTableName: string; config: TableSyncConfig } | undefined { + const direct = this.tables.get(tableName) + if (direct) { + return { sqliteTableName: tableName, config: direct } + } + + const cleaned = tableName.replace(/^tmp_/, '') + const directCleaned = this.tables.get(cleaned) + if (directCleaned) { + return { sqliteTableName: cleaned, config: directCleaned } + } + + const mappedName = this.schemaTableMap.get(cleaned.toLowerCase()) + if (mappedName && this.tables.has(mappedName)) { + return { + sqliteTableName: mappedName, + config: this.tables.get(mappedName)!, + } + } + + for (const [sqliteName, cfg] of this.tables.entries()) { + if ( + cfg.name === cleaned || + `${cfg.schema || 'public'}.${cfg.name}` === cleaned + ) { + return { sqliteTableName: sqliteName, config: cfg } + } + } + + return undefined + } + + public async setDataSource( + dataSource: DataSource, + starbaseConfig?: StarbaseDBConfiguration + ): Promise { + this.dataSource = dataSource + if (starbaseConfig) { + this.starbaseConfig = starbaseConfig + } + + if (!this.hasExplicitSyncSource && dataSource?.external?.dialect) { + const extDialect = dataSource.external.dialect + if (extDialect === 'mysql' && this.syncSource.dialect !== 'mysql') { + this.syncSource = new MySQLSyncSource() + } else if ( + extDialect === 'sqlite' && + this.syncSource.dialect !== 'sqlite' + ) { + this.syncSource = new SQLiteSyncSource() + } else if ( + extDialect === 'postgresql' && + this.syncSource.dialect !== 'postgresql' + ) { + this.syncSource = new PostgresSyncSource() + } + } + + await this.syncSource.setDataSource(dataSource, this.starbaseConfig) + } + + private async createMetadataTable(): Promise { + if (!this.dataSource) return + + const sql = ` + CREATE TABLE IF NOT EXISTS tmp_data_sync_metadata ( + table_name TEXT PRIMARY KEY, + source_table TEXT, + last_sync_timestamp INTEGER, + last_sync_id TEXT, + last_cursor_value TEXT, + cursor_column TEXT, + rows_synced INTEGER DEFAULT 0, + sync_errors TEXT + ) + ` + await this.dataSource.rpc.executeQuery({ sql, params: [] }) + } + + private async loadSyncMetadata(): Promise { + if (!this.dataSource) return + + const sql = `SELECT * FROM tmp_data_sync_metadata` + const rawResult = await this.dataSource.rpc.executeQuery({ + sql, + params: [], + }) + const result = normalizeToQueryResult(rawResult) + + for (const row of result.rows) { + const isLegacyFourCol = + result.columns.length === 0 && row.length === 4 + const tableName = String( + this.syncSource['getRowValue']( + row, + result.columns, + 'table_name', + 0 + ) + ) + const lastSyncTsRaw = this.syncSource['getRowValue']( + row, + result.columns, + 'last_sync_timestamp', + isLegacyFourCol ? 1 : 2 + ) + const lastSyncIdRaw = this.syncSource['getRowValue']( + row, + result.columns, + 'last_sync_id', + isLegacyFourCol ? 2 : 3 + ) + const lastCursorRaw = isLegacyFourCol + ? undefined + : this.syncSource['getRowValue']( + row, + result.columns, + 'last_cursor_value', + 4 + ) + const cursorColRaw = isLegacyFourCol + ? undefined + : this.syncSource['getRowValue']( + row, + result.columns, + 'cursor_column', + 5 + ) + const rowsSyncedRaw = isLegacyFourCol + ? undefined + : this.syncSource['getRowValue']( + row, + result.columns, + 'rows_synced', + 6 + ) + const syncErrorsRaw = this.syncSource['getRowValue']( + row, + result.columns, + 'sync_errors', + isLegacyFourCol ? 3 : 7 + ) + + this.syncMetadata.set(tableName, { + lastSyncTimestamp: Number(lastSyncTsRaw) || 0, + lastSyncId: + lastSyncIdRaw !== null && lastSyncIdRaw !== undefined + ? lastSyncIdRaw + : undefined, + lastCursorValue: + lastCursorRaw !== null && lastCursorRaw !== undefined + ? lastCursorRaw + : undefined, + cursorColumn: + cursorColRaw !== null && cursorColRaw !== undefined + ? String(cursorColRaw) + : undefined, + rowsSynced: + rowsSyncedRaw !== null && rowsSyncedRaw !== undefined + ? Number(rowsSyncedRaw) + : undefined, + sync_errors: + syncErrorsRaw !== null && syncErrorsRaw !== undefined + ? String(syncErrorsRaw) + : undefined, + }) + } + } + + private async updateSyncMetadata( + tableName: string, + metadata: TableMetadata, + sourceTable?: string + ): Promise { + if (!this.dataSource) return + + this.syncMetadata.set(tableName, metadata) + + const sql = ` + INSERT OR REPLACE INTO tmp_data_sync_metadata + (table_name, source_table, last_sync_timestamp, last_sync_id, last_cursor_value, cursor_column, rows_synced, sync_errors) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ` + const params = [ + tableName, + sourceTable || tableName, + metadata.lastSyncTimestamp ?? 0, + metadata.lastSyncId !== undefined && metadata.lastSyncId !== null + ? String(metadata.lastSyncId) + : null, + metadata.lastCursorValue !== undefined && + metadata.lastCursorValue !== null + ? String(metadata.lastCursorValue) + : null, + metadata.cursorColumn ?? null, + metadata.rowsSynced ?? 0, + metadata.sync_errors ?? null, + ] + + try { + await this.dataSource.rpc.executeQuery({ sql, params }) + } catch (error) { + const msg = error instanceof Error ? error.message : String(error) + if (msg.toLowerCase().includes('no such table')) { + await this.createMetadataTable() + await this.dataSource.rpc.executeQuery({ sql, params }) + } else { + throw error + } + } + } + + private async createTempTable( + tableName: string, + columns: ColumnDefinition[], + tableConfig?: TableSyncConfig + ): Promise { + if (!this.dataSource) { + throw new Error('Internal data source not configured') + } + + const configuredPk = tableConfig?.primary_key || tableConfig?.primaryKey + + const columnDefs = columns + .map((col) => { + const type = this.syncSource.mapDataType(col.type) + const isPk = + col.primaryKey || + (configuredPk && + col.name.toLowerCase() === configuredPk.toLowerCase()) + const pkClause = isPk ? ' PRIMARY KEY' : '' + const nullable = col.nullable === false ? ' NOT NULL' : '' + const defaultValue = col.defaultValue + ? ` DEFAULT ${col.defaultValue}` + : '' + return `${escapeIdentifier(col.name)} ${type}${pkClause}${nullable}${defaultValue}` + }) + .join(', ') + + const sql = `CREATE TABLE IF NOT EXISTS ${escapeIdentifier(tableName)} (${columnDefs})` + await this.dataSource.rpc.executeQuery({ sql, params: [] }) + } + + private async ensureInitialized(): Promise { + if (!this.dataSource || this.isMetadataInitialized) return + await this.createMetadataTable() + await this.loadSyncMetadata() + this.isMetadataInitialized = true + } + + private async syncDueTables(): Promise { + if (!this.dataSource?.external) return + + const now = Date.now() + for (const sqliteTableName of this.tables.keys()) { + const meta = this.syncMetadata.get(sqliteTableName) + const lastAttempt = this.lastSyncAttempt.get(sqliteTableName) || 0 + const lastSyncTs = Math.max( + meta?.lastSyncTimestamp || 0, + lastAttempt + ) + + if (lastSyncTs === 0 || now - lastSyncTs >= this.syncInterval) { + this.lastSyncAttempt.set(sqliteTableName, now) + try { + await this.syncTable(sqliteTableName) + } catch (error) { + console.error( + `Background sync failed for table ${sqliteTableName}:`, + error + ) + } + } + } + } + + override async register(app: StarbaseApp): Promise { + app.use(async (c, next) => { + const reqDataSource = c?.get('dataSource') + const reqConfig = c?.get('config') + + if (reqDataSource) { + await this.setDataSource(reqDataSource, reqConfig) + } + + if (this.dataSource && this.autoSync && this.dataSource.external) { + try { + await this.ensureInitialized() + + const hasUnsyncedTable = Array.from( + this.tables.keys() + ).some((t) => { + const meta = this.syncMetadata.get(t) + const lastAttempt = this.lastSyncAttempt.get(t) || 0 + return !meta?.lastSyncTimestamp && lastAttempt === 0 + }) + + if (hasUnsyncedTable) { + await this.syncDueTables() + } else if (this.dataSource.executionContext) { + this.dataSource.executionContext.waitUntil( + this.syncDueTables() + ) + } else { + await this.syncDueTables() + } + } catch (error) { + console.error('DataSyncPlugin middleware error:', error) + } + } + + await next() + }) + + const statusHandler = async () => { + const status = { + config: this.getConfig(), + tables: Object.fromEntries(this.syncMetadata), + } + return createResponse(status, undefined, 200) + } + + const triggerHandler = async (c: any) => { + try { + let targetTable: string | undefined + try { + const body = await c.req.json() + targetTable = body?.table + } catch { + // Request body is optional for POST /sync + } + + if (targetTable) { + await this.syncTable(targetTable) + } else { + await this.syncAll() + } + return createResponse( + { + status: 'Sync triggered', + tables: Object.fromEntries(this.syncMetadata), + }, + undefined, + 200 + ) + } catch (error) { + return createResponse(undefined, (error as Error).message, 500) + } + } + + const dataHandler = async () => { + try { + const tableData: Record = {} + for (const [sqliteTableName, tableConfig] of this.tables) { + const raw = await this.dataSource?.rpc.executeQuery({ + sql: `SELECT * FROM ${escapeIdentifier(sqliteTableName)} LIMIT 100`, + params: [], + }) + const result = normalizeToQueryResult(raw) + tableData[tableConfig.name] = { + columns: result.columns, + rows: result.rows, + count: result.rows.length, + } + } + return createResponse( + { + tables: tableData, + metadata: Object.fromEntries(this.syncMetadata), + }, + undefined, + 200 + ) + } catch (error) { + return createResponse(undefined, (error as Error).message, 500) + } + } + + app.get('/sync-status', statusHandler) + app.get(`${this.pathPrefix}/status`, statusHandler) + + app.post('/sync', triggerHandler) + app.post(`${this.pathPrefix}/trigger`, triggerHandler) + + app.get('/sync-data', dataHandler) + app.get(`${this.pathPrefix}/data`, dataHandler) + + // If dataSource was already provided before register() (e.g. via setDataSource), + // validate connection, initialize metadata, and start periodic sync immediately. + if (this.dataSource) { + const isValid = await this.syncSource.validateConnection() + if (!isValid) { + throw new Error( + `Failed to validate connection to external ${this.syncSource.dialect} database` + ) + } + + await this.createMetadataTable() + await this.loadSyncMetadata() + this.isMetadataInitialized = true + + if (this.autoSync) { + await this.startPeriodicSync() + } + } + } + + public async startPeriodicSync(): Promise { + if (this.timerId) { + clearInterval(this.timerId) + this.timerId = undefined + } + + await this.syncAll() + + this.timerId = setInterval(async () => { + await this.syncAll() + }, this.syncInterval) + } + + public stopPeriodicSync(): void { + if (this.timerId) { + clearInterval(this.timerId) + this.timerId = undefined + } + } + + public async syncTable(tableName: string): Promise { + if (!this.dataSource) { + throw new Error('Data source not configured') + } + + await this.syncSource.setDataSource( + this.dataSource, + this.starbaseConfig + ) + + const resolved = this.resolveTableEntry(tableName) + if (!resolved) { + throw new Error( + `Table ${tableName} not found in sync configuration` + ) + } + + const { sqliteTableName, config: tableConfig } = resolved + + try { + const validation = await this.syncSource.validateTableStructure( + tableConfig.name, + tableConfig + ) + if (!validation.valid) { + throw new Error( + `Table structure validation failed for ${tableName}: ${validation.errors.join(', ')}` + ) + } + + const schema = await this.syncSource.getTableSchema( + tableConfig.name, + tableConfig.schema + ) + + await this.createTempTable(sqliteTableName, schema, tableConfig) + + const schemaColMap = new Map() + for (const col of schema) { + schemaColMap.set(col.name.toLowerCase(), col) + } + + let cursorCol = + tableConfig.cursor_column || + tableConfig.cursorColumn || + tableConfig.track_column || + tableConfig.trackColumn + if (cursorCol && !schemaColMap.has(cursorCol.toLowerCase())) { + cursorCol = undefined + } else if (cursorCol) { + cursorCol = schemaColMap.get(cursorCol.toLowerCase())!.name + } + + let timestampCol = + tableConfig.timestamp_column || tableConfig.timestampColumn + if (timestampCol && !schemaColMap.has(timestampCol.toLowerCase())) { + const autoTs = schema.find((c) => + ['created_at', 'updated_at'].includes(c.name.toLowerCase()) + ) + timestampCol = autoTs ? autoTs.name : undefined + } else if (timestampCol) { + timestampCol = schemaColMap.get( + timestampCol.toLowerCase() + )!.name + } + + let idCol = tableConfig.id_column || tableConfig.idColumn + if (idCol && !schemaColMap.has(idCol.toLowerCase())) { + const autoPk = schema.find((c) => c.primaryKey) + idCol = autoPk ? autoPk.name : undefined + } else if (idCol) { + idCol = schemaColMap.get(idCol.toLowerCase())!.name + } + + const effectiveConfig: TableSyncConfig = { + ...tableConfig, + cursor_column: cursorCol, + timestamp_column: timestampCol, + id_column: idCol, + } + + const existingMetadata = this.syncMetadata.get(sqliteTableName) + const metadata: TableMetadata = existingMetadata + ? { ...existingMetadata } + : { lastSyncTimestamp: 0 } + + const rawNewData = await this.syncSource.getIncrementalData( + tableConfig.name, + metadata, + effectiveConfig + ) + const newData = normalizeToQueryResult(rawNewData) + + if (newData.rows.length > 0) { + const insertColumns = + newData.columns.length > 0 + ? newData.columns + : schema.map((c) => c.name) + const columnsList = insertColumns + .map((c) => escapeIdentifier(c)) + .join(', ') + const placeholders = insertColumns.map(() => '?').join(', ') + const insertSql = `INSERT OR REPLACE INTO ${escapeIdentifier(sqliteTableName)} (${columnsList}) VALUES (${placeholders})` + + const cursorIdx = cursorCol + ? insertColumns.findIndex( + (c) => c.toLowerCase() === cursorCol!.toLowerCase() + ) + : -1 + const timestampIdx = timestampCol + ? insertColumns.findIndex( + (c) => c.toLowerCase() === timestampCol!.toLowerCase() + ) + : -1 + const idIdx = idCol + ? insertColumns.findIndex( + (c) => c.toLowerCase() === idCol!.toLowerCase() + ) + : -1 + + for (const row of newData.rows) { + const sanitizedParams = insertColumns.map((_, idx) => + sanitizeSQLiteValue(row[idx]) + ) + + await this.dataSource.rpc.executeQuery({ + sql: insertSql, + params: sanitizedParams, + }) + + if ( + cursorIdx !== -1 && + row[cursorIdx] !== undefined && + row[cursorIdx] !== null + ) { + const rawVal = row[cursorIdx] + metadata.lastCursorValue = + rawVal instanceof Date + ? rawVal.toISOString() + : rawVal + metadata.cursorColumn = cursorCol + } + + if ( + timestampIdx !== -1 && + row[timestampIdx] !== undefined && + row[timestampIdx] !== null + ) { + const rawTs = row[timestampIdx] + const parsedTs = + rawTs instanceof Date + ? rawTs.getTime() + : new Date(rawTs).getTime() + if (!isNaN(parsedTs)) { + metadata.lastSyncTimestamp = parsedTs + } + } + + if ( + idIdx !== -1 && + row[idIdx] !== undefined && + row[idIdx] !== null + ) { + metadata.lastSyncId = row[idIdx] + } + } + + if (!metadata.lastSyncTimestamp) { + metadata.lastSyncTimestamp = Date.now() + } + metadata.rowsSynced = + (metadata.rowsSynced || 0) + newData.rows.length + metadata.sync_errors = undefined + + await this.updateSyncMetadata( + sqliteTableName, + metadata, + tableConfig.name + ) + } + + return { + table: sqliteTableName, + sourceTable: tableConfig.name, + rowsSynced: newData.rows.length, + lastSyncTimestamp: metadata.lastSyncTimestamp, + lastSyncId: metadata.lastSyncId, + lastCursorValue: metadata.lastCursorValue, + } + } catch (error) { + const errorMsg = (error as Error).message + const metadata: TableMetadata = this.syncMetadata.get( + sqliteTableName + ) || { + lastSyncTimestamp: 0, + } + metadata.sync_errors = errorMsg + await this.updateSyncMetadata( + sqliteTableName, + metadata, + tableConfig.name + ) + throw error + } + } + + public async syncAll(): Promise { + const results: TableSyncResult[] = [] + for (const sqliteTableName of this.tables.keys()) { + try { + const res = await this.syncTable(sqliteTableName) + results.push(res) + } catch (error) { + console.error(`Failed to sync table ${sqliteTableName}:`, error) + const cfg = this.tables.get(sqliteTableName) + results.push({ + table: sqliteTableName, + sourceTable: cfg?.name || sqliteTableName, + rowsSynced: 0, + lastSyncTimestamp: + this.syncMetadata.get(sqliteTableName) + ?.lastSyncTimestamp || 0, + error: (error as Error).message, + }) + } + } + return results + } + + public getConfig(): SyncConfig { + return { + sync_interval: this.syncInterval / 1000, + tables: Array.from(this.tables.values()), + auto_sync: this.autoSync, + } + } + + public getMetadata(): Record { + return Object.fromEntries(this.syncMetadata) + } + + override async beforeQuery(opts: { + sql: string + params?: unknown[] + dataSource?: DataSource + config?: StarbaseDBConfiguration + }): Promise<{ sql: string; params?: unknown[] }> { + let { sql, params, dataSource } = opts + + // Only rewrite queries targeting the internal SQLite database + if (dataSource?.source && dataSource.source !== 'internal') { + return { sql, params } + } + + // Never rewrite SQLite system catalog / introspection queries + if ( + /\b(sqlite_master|sqlite_schema|pragma_table_info|tmp_data_sync_metadata)\b/i.test( + sql + ) || + /^\s*PRAGMA\b/i.test(sql) + ) { + return { sql, params } + } + + const updatedSql = transformOutsideSingleQuotes(sql, (segment) => { + let transformed = segment + + // 1. Rewrite configured schema.table references (e.g. analytics.events -> analytics_events, public.users -> users) + for (const [qualifiedKey, targetTable] of this.schemaTableMap) { + const [schemaName, tableName] = qualifiedKey.split('.', 2) + const escapedSchema = schemaName.replace( + /[.*+?^${}()|[\]\\]/g, + '\\$&' + ) + const escapedTable = tableName.replace( + /[.*+?^${}()|[\]\\]/g, + '\\$&' + ) + + // Double-quoted "schema"."table" -> "targetTable" + const quotedRegex = new RegExp( + `"${escapedSchema}"\\s*\\.\\s*"${escapedTable}"`, + 'gi' + ) + transformed = transformed.replace( + quotedRegex, + `"${targetTable}"` + ) + + // Mixed or unquoted schema.table -> targetTable + const unquotedRegex = new RegExp( + `(?:"${escapedSchema}"|\\b${escapedSchema}\\b)\\s*\\.\\s*(?:"${escapedTable}"|\\b${escapedTable}\\b)`, + 'gi' + ) + transformed = transformed.replace(unquotedRegex, targetTable) + } + + // 2. General public. prefix omission for internal SQLite queries + transformed = transformed.replace( + /"public"\s*\.\s*"([a-zA-Z_][a-zA-Z0-9_]*)"/gi, + '"$1"' + ) + transformed = transformed.replace( + /(?:"public"|\bpublic\b)\s*\.\s*([a-zA-Z_][a-zA-Z0-9_]*)\b/gi, + '$1' + ) + + // 3. Rewrite standalone table names when an explicit target_table override differs from table.name + for (const [sqliteTableName, cfg] of this.tables.entries()) { + if ( + sqliteTableName !== cfg.name && + (!cfg.schema || + cfg.schema === 'public' || + cfg.schema === 'main') + ) { + const escapedName = cfg.name.replace( + /[.*+?^${}()|[\]\\]/g, + '\\$&' + ) + const standaloneRegex = new RegExp( + `\\b${escapedName}\\b`, + 'g' + ) + transformed = transformed.replace( + standaloneRegex, + sqliteTableName + ) + } + } + + return transformed + }) + + return { + sql: updatedSql, + params, + } + } + + override async afterQuery(opts: { + sql: string + result: any + isRaw: boolean + dataSource?: DataSource + config?: StarbaseDBConfiguration + }): Promise { + return opts.result + } +} + +export { DataSyncPlugin as ReplicationPlugin } diff --git a/plugins/data-sync/meta.json b/plugins/data-sync/meta.json new file mode 100644 index 0000000..4a34bf4 --- /dev/null +++ b/plugins/data-sync/meta.json @@ -0,0 +1,27 @@ +{ + "version": "1.0.0", + "resources": { + "tables": { + "tmp_data_sync_metadata": [ + "table_name", + "source_table", + "last_sync_timestamp", + "last_sync_id", + "last_cursor_value", + "cursor_column", + "rows_synced", + "sync_errors" + ] + }, + "secrets": {}, + "variables": { + "sync_interval": "number", + "tables": "string[]" + } + }, + "dependencies": { + "tables": {}, + "secrets": {}, + "variables": {} + } +} diff --git a/plugins/postgres-sync/README.md b/plugins/postgres-sync/README.md new file mode 100644 index 0000000..98501f4 --- /dev/null +++ b/plugins/postgres-sync/README.md @@ -0,0 +1,36 @@ +# PostgreSQL Sync Source (`PostgresSyncSource`) + +The PostgreSQL Sync Source adapter for StarbaseDB provides schema introspection, data type mapping, table structure validation, and incremental data fetching for external PostgreSQL databases (including Supabase, Neon, AWS RDS, Cloudflare Hyperdrive, and Outerbase API connections). + +## Usage + +```typescript +import { DataSyncPlugin } from '../data-sync' +import { PostgresSyncSource } from '../postgres-sync' + +const syncStore = new PostgresSyncSource({ + dialect: 'postgresql', + schema: 'public', +}) + +const dataSyncPlugin = new DataSyncPlugin(syncStore, { + sync_interval: 60, + tables: ['users', 'person'], +}) +``` + +## Configuration Options + +| Option | Type | Default | Description | +| --------- | -------------- | -------------- | -------------------------------------------------- | +| `dialect` | `'postgresql'` | `'postgresql'` | Database dialect identifier | +| `schema` | `string` | `'public'` | Default PostgreSQL schema to replicate tables from | + +## Data Type Mapping + +| PostgreSQL Types | SQLite Type | +| ------------------------------------------------------------------------------------------------------ | ----------- | +| `integer`, `bigint`, `smallint`, `serial`, `bigserial`, `boolean`, `bit` | `INTEGER` | +| `numeric`, `decimal`, `real`, `double precision`, `money` | `REAL` | +| `bytea` | `BLOB` | +| `text`, `varchar`, `char`, `date`, `timestamp`, `timestamptz`, `json`, `jsonb`, `uuid`, arrays (`*[]`) | `TEXT` | diff --git a/plugins/postgres-sync/index.test.ts b/plugins/postgres-sync/index.test.ts new file mode 100644 index 0000000..c48f571 --- /dev/null +++ b/plugins/postgres-sync/index.test.ts @@ -0,0 +1,640 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { DataSource } from '../../src/types' +import { TableMetadata, TableSyncConfig } from '../data-sync' +import { PostgresSync, PostgresSyncSource } from './index' + +describe('PostgresSyncSource', () => { + let postgresSyncSource: PostgresSyncSource + let mockDataSource: DataSource + let mockExecuteQuery: ReturnType + + beforeEach(() => { + vi.clearAllMocks() + + mockExecuteQuery = vi.fn() + mockDataSource = { + source: 'internal', + external: { + dialect: 'postgresql', + connection: { + host: 'localhost', + port: 5432, + database: 'test', + user: 'postgres', + password: 'password', + }, + } as any, + rpc: { + executeQuery: mockExecuteQuery, + } as any, + } + + postgresSyncSource = new PostgresSyncSource({ + dialect: 'postgresql', + schema: 'public', + }) + }) + + describe('Constructor and Dialect', () => { + it('should initialize with default public schema and postgresql dialect', () => { + const defaultSource = new PostgresSyncSource() + expect(defaultSource.dialect).toBe('postgresql') + expect(PostgresSync).toBe(PostgresSyncSource) + }) + + it('should accept a custom default schema', () => { + const customSource = new PostgresSyncSource({ + dialect: 'postgresql', + schema: 'analytics', + }) + expect(customSource.dialect).toBe('postgresql') + }) + }) + + describe('Connection Validation', () => { + it('should validate PostgreSQL connection successfully with tabular rows', async () => { + mockExecuteQuery.mockResolvedValueOnce({ + rows: [['PostgreSQL 15.0']], + columns: ['version'], + }) + + await postgresSyncSource.setDataSource(mockDataSource) + const isValid = await postgresSyncSource.validateConnection() + + expect(isValid).toBe(true) + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: 'SELECT version()', + params: [], + }) + }) + + it('should validate PostgreSQL connection successfully with @outerbase/sdk object rows', async () => { + mockExecuteQuery.mockResolvedValueOnce([ + { version: 'PostgreSQL 16.2 on x86_64-pc-linux-gnu' }, + ]) + + await postgresSyncSource.setDataSource(mockDataSource) + const isValid = await postgresSyncSource.validateConnection() + + expect(isValid).toBe(true) + }) + + it('should fail validation when external data source is not configured', async () => { + const invalidDataSource: DataSource = { + source: 'internal', + rpc: { + executeQuery: mockExecuteQuery, + } as any, + } + + await postgresSyncSource.setDataSource(invalidDataSource) + const isValid = await postgresSyncSource.validateConnection() + + expect(isValid).toBe(false) + expect(mockExecuteQuery).not.toHaveBeenCalled() + }) + + it('should fail validation when external dialect is not postgresql', async () => { + const mysqlDataSource: DataSource = { + source: 'internal', + external: { + dialect: 'mysql', + } as any, + rpc: { + executeQuery: mockExecuteQuery, + } as any, + } + + await postgresSyncSource.setDataSource(mysqlDataSource) + const isValid = await postgresSyncSource.validateConnection() + + expect(isValid).toBe(false) + expect(mockExecuteQuery).not.toHaveBeenCalled() + }) + + it('should fail validation when connection query throws an error', async () => { + mockExecuteQuery.mockRejectedValueOnce( + new Error('Connection refused') + ) + + await postgresSyncSource.setDataSource(mockDataSource) + const isValid = await postgresSyncSource.validateConnection() + + expect(isValid).toBe(false) + }) + }) + + describe('Schema Operations (getTableSchema)', () => { + beforeEach(async () => { + await postgresSyncSource.setDataSource(mockDataSource) + }) + + it('should retrieve table schema correctly from tabular rows', async () => { + mockExecuteQuery.mockResolvedValueOnce({ + rows: [ + [ + 'id', + 'integer', + 'int4', + 'NO', + "nextval('users_id_seq'::regclass)", + null, + ], + ['name', 'character varying', 'varchar', 'YES', null, 255], + [ + 'created_at', + 'timestamp with time zone', + 'timestamptz', + 'NO', + 'now()', + null, + ], + ['tags', 'ARRAY', '_text', 'YES', null, null], + [ + 'status', + 'character varying', + 'varchar', + 'NO', + "'active'::character varying", + 50, + ], + ['score', 'integer', 'int4', 'NO', '0', null], + ], + columns: [ + 'column_name', + 'data_type', + 'udt_name', + 'is_nullable', + 'column_default', + 'character_maximum_length', + ], + }) + + const schema = await postgresSyncSource.getTableSchema('users') + + expect(schema).toEqual([ + { + name: 'id', + type: 'integer', + nullable: false, + defaultValue: undefined, + }, + { + name: 'name', + type: 'character varying(255)', + nullable: true, + defaultValue: undefined, + }, + { + name: 'created_at', + type: 'timestamp with time zone', + nullable: false, + defaultValue: undefined, + }, + { + name: 'tags', + type: 'text[]', + nullable: true, + defaultValue: undefined, + }, + { + name: 'status', + type: 'character varying(50)', + nullable: false, + defaultValue: "'active'", + }, + { + name: 'score', + type: 'integer', + nullable: false, + defaultValue: '0', + }, + ]) + + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: expect.stringContaining('FROM information_schema.columns'), + params: ['public', 'users'], + }) + }) + + it('should retrieve table schema correctly from @outerbase/sdk object rows', async () => { + mockExecuteQuery.mockResolvedValueOnce([ + { + column_name: 'first_name', + data_type: 'text', + udt_name: 'text', + is_nullable: 'YES', + column_default: null, + character_maximum_length: null, + }, + { + column_name: 'last_name', + data_type: 'text', + udt_name: 'text', + is_nullable: 'NO', + column_default: "'Doe'::text", + character_maximum_length: null, + }, + ]) + + const schema = + await postgresSyncSource.getTableSchema('analytics.person') + + expect(schema).toEqual([ + { + name: 'first_name', + type: 'text', + nullable: true, + defaultValue: undefined, + }, + { + name: 'last_name', + type: 'text', + nullable: false, + defaultValue: "'Doe'", + }, + ]) + + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: expect.stringContaining('FROM information_schema.columns'), + params: ['analytics', 'person'], + }) + }) + + it('should strip tmp_ prefix when querying external information_schema', async () => { + mockExecuteQuery.mockResolvedValueOnce({ + rows: [['id', 'integer', 'int4', 'NO', null, null]], + columns: [ + 'column_name', + 'data_type', + 'udt_name', + 'is_nullable', + 'column_default', + 'character_maximum_length', + ], + }) + + await postgresSyncSource.getTableSchema('tmp_users') + + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: expect.stringContaining('FROM information_schema.columns'), + params: ['public', 'users'], + }) + }) + }) + + describe('Data Type Mapping (mapDataType)', () => { + it('should map PostgreSQL types to SQLite types accurately', () => { + const testCases: [string, string][] = [ + ['integer', 'INTEGER'], + ['int', 'INTEGER'], + ['int2', 'INTEGER'], + ['int4', 'INTEGER'], + ['int8', 'INTEGER'], + ['bigint', 'INTEGER'], + ['smallint', 'INTEGER'], + ['serial', 'INTEGER'], + ['bigserial', 'INTEGER'], + ['smallserial', 'INTEGER'], + ['boolean', 'INTEGER'], + ['bool', 'INTEGER'], + ['bit', 'INTEGER'], + ['bit varying', 'INTEGER'], + ['numeric', 'REAL'], + ['decimal(10,2)', 'REAL'], + ['real', 'REAL'], + ['double precision', 'REAL'], + ['float4', 'REAL'], + ['float8', 'REAL'], + ['money', 'REAL'], + ['character varying(255)', 'TEXT'], + ['varchar', 'TEXT'], + ['character(10)', 'TEXT'], + ['char', 'TEXT'], + ['text', 'TEXT'], + ['bytea', 'BLOB'], + ['timestamp with time zone', 'TEXT'], + ['timestamp without time zone', 'TEXT'], + ['timestamptz', 'TEXT'], + ['date', 'TEXT'], + ['time', 'TEXT'], + ['interval', 'TEXT'], + ['json', 'TEXT'], + ['jsonb', 'TEXT'], + ['uuid', 'TEXT'], + ['inet', 'TEXT'], + ['cidr', 'TEXT'], + ['macaddr', 'TEXT'], + ['xml', 'TEXT'], + ['integer[]', 'TEXT'], + ['text[]', 'TEXT'], + ['varchar[]', 'TEXT'], + ['custom_domain_type', 'TEXT'], + ] + + for (const [pgType, expectedSqliteType] of testCases) { + expect(postgresSyncSource.mapDataType(pgType)).toBe( + expectedSqliteType + ) + } + }) + }) + + describe('Table Structure Validation (validateTableStructure)', () => { + beforeEach(async () => { + await postgresSyncSource.setDataSource(mockDataSource) + }) + + it('should validate existing table structure with standard created_at and id columns', async () => { + mockExecuteQuery + .mockResolvedValueOnce({ + rows: [[true]], + columns: ['exists'], + }) + .mockResolvedValueOnce({ + rows: [ + ['id', 'integer', 'int4', 'NO', null, null], + [ + 'created_at', + 'timestamp with time zone', + 'timestamptz', + 'NO', + null, + null, + ], + ], + columns: [ + 'column_name', + 'data_type', + 'udt_name', + 'is_nullable', + 'column_default', + 'character_maximum_length', + ], + }) + + const tableConfig: TableSyncConfig = { + name: 'users', + timestamp_column: 'created_at', + id_column: 'id', + hasExplicitSyncColumns: true, + } + + const validation = await postgresSyncSource.validateTableStructure( + 'users', + tableConfig + ) + + expect(validation.valid).toBe(true) + expect(validation.errors).toHaveLength(0) + }) + + it('should validate table without created_at or id when configured via string name (hasExplicitSyncColumns: false)', async () => { + mockExecuteQuery + .mockResolvedValueOnce([{ exists: true }]) + .mockResolvedValueOnce([ + { + column_name: 'first_name', + data_type: 'text', + udt_name: 'text', + is_nullable: 'YES', + column_default: null, + character_maximum_length: null, + }, + { + column_name: 'last_name', + data_type: 'text', + udt_name: 'text', + is_nullable: 'YES', + column_default: null, + character_maximum_length: null, + }, + ]) + + const tableConfig: TableSyncConfig = { + name: 'person', + timestamp_column: 'created_at', + id_column: 'id', + hasExplicitSyncColumns: false, + } + + const validation = await postgresSyncSource.validateTableStructure( + 'person', + tableConfig + ) + + expect(validation.valid).toBe(true) + expect(validation.errors).toHaveLength(0) + }) + + it('should detect non-existent table', async () => { + mockExecuteQuery.mockResolvedValueOnce({ + rows: [[false]], + columns: ['exists'], + }) + + const tableConfig: TableSyncConfig = { + name: 'non_existent', + timestamp_column: 'created_at', + id_column: 'id', + } + + const validation = await postgresSyncSource.validateTableStructure( + 'non_existent', + tableConfig + ) + + expect(validation.valid).toBe(false) + expect(validation.errors).toContain( + 'Table public.non_existent does not exist' + ) + }) + + it('should detect missing explicitly required columns and invalid timestamp column types', async () => { + mockExecuteQuery + .mockResolvedValueOnce({ + rows: [[true]], + columns: ['exists'], + }) + .mockResolvedValueOnce({ + rows: [ + [ + 'name', + 'character varying', + 'varchar', + 'YES', + null, + 255, + ], + ['bad_ts', 'integer', 'int4', 'NO', null, null], + ], + columns: [ + 'column_name', + 'data_type', + 'udt_name', + 'is_nullable', + 'column_default', + 'character_maximum_length', + ], + }) + + const tableConfig: TableSyncConfig = { + name: 'users', + timestamp_column: 'bad_ts', + id_column: 'id', + cursor_column: 'missing_cursor', + hasExplicitSyncColumns: true, + } + + const validation = await postgresSyncSource.validateTableStructure( + 'users', + tableConfig + ) + + expect(validation.valid).toBe(false) + expect(validation.errors).toContain( + 'Cursor column missing_cursor not found in table public.users' + ) + expect(validation.errors).toContain( + 'Column bad_ts in table public.users is not a timestamp type (found: integer)' + ) + expect(validation.errors).toContain( + 'ID column id not found in table public.users' + ) + }) + }) + + describe('Incremental Data Sync (getIncrementalData)', () => { + beforeEach(async () => { + await postgresSyncSource.setDataSource(mockDataSource) + }) + + it('should fetch initial data when no lastSyncTimestamp or lastSyncId exists', async () => { + const mockResult = { + rows: [ + [1, 'John', '2023-01-01T00:00:00Z'], + [2, 'Jane', '2023-01-02T00:00:00Z'], + ], + columns: ['id', 'name', 'created_at'], + } + mockExecuteQuery.mockResolvedValueOnce(mockResult) + + const lastSync: TableMetadata = { + lastSyncTimestamp: 0, + } + const tableConfig: TableSyncConfig = { + name: 'users', + timestamp_column: 'created_at', + id_column: 'id', + batch_size: 1000, + } + + const result = await postgresSyncSource.getIncrementalData( + 'users', + lastSync, + tableConfig + ) + + expect(result).toEqual(mockResult) + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: 'SELECT * FROM public.users ORDER BY created_at, id ASC LIMIT 1000', + params: [], + }) + }) + + it('should fetch incremental data using timestamp and id watermarks', async () => { + const mockResult = { + rows: [[3, 'Alice', '2023-01-03T00:00:00Z']], + columns: ['id', 'name', 'created_at'], + } + mockExecuteQuery.mockResolvedValueOnce(mockResult) + + const lastSync: TableMetadata = { + lastSyncTimestamp: 1640995200000, // 2022-01-01T00:00:00.000Z + lastSyncId: '2', + } + const tableConfig: TableSyncConfig = { + name: 'users', + timestamp_column: 'created_at', + id_column: 'id', + batch_size: 500, + } + + const result = await postgresSyncSource.getIncrementalData( + 'users', + lastSync, + tableConfig + ) + + expect(result).toEqual(mockResult) + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: 'SELECT * FROM public.users WHERE created_at > ? AND id > ? ORDER BY created_at, id ASC LIMIT 500', + params: ['2022-01-01T00:00:00.000Z', '2'], + }) + }) + + it('should fetch incremental data using custom cursor_column', async () => { + mockExecuteQuery.mockResolvedValueOnce([ + { seq_id: 105, payload: 'event-105' }, + ]) + + const lastSync: TableMetadata = { + lastSyncTimestamp: 1700000000000, + lastCursorValue: 104, + cursorColumn: 'seq_id', + } + const tableConfig: TableSyncConfig = { + name: 'events', + schema: 'analytics', + cursor_column: 'seq_id', + batch_size: 250, + } + + const result = await postgresSyncSource.getIncrementalData( + 'events', + lastSync, + tableConfig + ) + + expect(result).toEqual({ + columns: ['seq_id', 'payload'], + rows: [[105, 'event-105']], + }) + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: 'SELECT * FROM analytics.events WHERE seq_id > ? ORDER BY seq_id ASC LIMIT 250', + params: [104], + }) + }) + + it('should fetch full batch without WHERE or ORDER BY when table has no cursor, timestamp, or id columns', async () => { + mockExecuteQuery.mockResolvedValueOnce([ + { first_name: 'Brayden', last_name: 'Wilmoth' }, + ]) + + const lastSync: TableMetadata = { + lastSyncTimestamp: 1700000000000, + } + const tableConfig: TableSyncConfig = { + name: 'person', + timestamp_column: undefined, + id_column: undefined, + batch_size: 1000, + } + + const result = await postgresSyncSource.getIncrementalData( + 'person', + lastSync, + tableConfig + ) + + expect(result).toEqual({ + columns: ['first_name', 'last_name'], + rows: [['Brayden', 'Wilmoth']], + }) + expect(mockExecuteQuery).toHaveBeenCalledWith({ + sql: 'SELECT * FROM public.person LIMIT 1000', + params: [], + }) + }) + }) +}) diff --git a/plugins/postgres-sync/index.ts b/plugins/postgres-sync/index.ts new file mode 100644 index 0000000..7a1f11d --- /dev/null +++ b/plugins/postgres-sync/index.ts @@ -0,0 +1,477 @@ +import { + ColumnDefinition, + DatabaseSyncSource, + escapeIdentifier, + QueryResult, + SyncSourceConfig, + TableMetadata, + TableSyncConfig, +} from '../data-sync' + +export interface PostgresSyncConfig extends SyncSourceConfig { + dialect: 'postgresql' + schema?: string +} + +/** + * PostgreSQL Sync Source adapter for replicating schema and data from an external + * PostgreSQL database (Supabase, Neon, RDS, Hyperdrive, or Outerbase API) into + * StarbaseDB's internal SQLite database. + */ +export class PostgresSyncSource extends DatabaseSyncSource { + private schema: string + + constructor(config: Partial = {}) { + super({ + ...config, + dialect: 'postgresql', + }) + this.schema = config.schema || 'public' + } + + get dialect(): string { + return 'postgresql' + } + + async validateConnection(): Promise { + const externalDataSource = this.getExternalDataSource() + if ( + !externalDataSource || + externalDataSource.external?.dialect !== 'postgresql' + ) { + return false + } + + try { + await this.executeExternalQuery('SELECT version()') + return true + } catch (error) { + console.error('PostgreSQL connection validation failed:', error) + return false + } + } + + async getTableSchema( + tableName: string, + schema?: string + ): Promise { + const externalDataSource = this.getExternalDataSource() + if (!externalDataSource) { + throw new Error('External data source not configured') + } + + let targetSchema = + schema || + (externalDataSource.external as any)?.defaultSchema || + this.schema + let cleanTableName = tableName.replace(/^tmp_/, '') + if (cleanTableName.includes('.')) { + const parts = cleanTableName.split('.', 2) + targetSchema = schema || parts[0] + cleanTableName = parts[1] + } + + const result = await this.executeExternalQuery( + `SELECT + column_name, + data_type, + udt_name, + is_nullable, + column_default, + character_maximum_length + FROM information_schema.columns + WHERE table_schema = ? AND table_name = ? + ORDER BY ordinal_position`, + [targetSchema, cleanTableName] + ) + + return (result.rows || []).map((row: any[]) => { + const colName = String( + this.getRowValue(row, result.columns, 'column_name', 0) + ) + const dataType = String( + this.getRowValue(row, result.columns, 'data_type', 1) || 'text' + ) + const udtName = this.getRowValue(row, result.columns, 'udt_name', 2) + const isNullable = this.getRowValue( + row, + result.columns, + 'is_nullable', + 3 + ) + const colDefault = this.getRowValue( + row, + result.columns, + 'column_default', + 4 + ) + const maxLength = this.getRowValue( + row, + result.columns, + 'character_maximum_length', + 5 + ) + + let fullType = dataType + if ( + dataType === 'ARRAY' && + typeof udtName === 'string' && + udtName.startsWith('_') + ) { + fullType = `${udtName.substring(1)}[]` + } else if ( + dataType === 'USER-DEFINED' && + typeof udtName === 'string' + ) { + fullType = udtName + } else if ( + (dataType === 'character varying' || dataType === 'varchar') && + maxLength + ) { + fullType = `${dataType}(${maxLength})` + } else if ( + dataType === 'timestamp with time zone' || + udtName === 'timestamptz' + ) { + fullType = 'timestamp with time zone' + } + + return { + name: colName, + type: fullType, + nullable: + isNullable === 'YES' || + isNullable === true || + isNullable === 1, + defaultValue: this.normalizeDefaultValue(colDefault, fullType), + } + }) + } + + private normalizeDefaultValue( + defaultValue: string | null | undefined, + dataType: string + ): string | undefined { + if (!defaultValue) return undefined + + if (defaultValue.includes('nextval')) { + return undefined + } + + if (defaultValue.includes('(') && defaultValue.includes(')')) { + return undefined + } + + if (defaultValue.startsWith("'") && defaultValue.includes("'::")) { + return defaultValue.split("'::")[0] + "'" + } + + const baseType = dataType + .toLowerCase() + .replace(/\(.*\)/, '') + .trim() + if ( + [ + 'integer', + 'int', + 'int2', + 'int4', + 'int8', + 'bigint', + 'smallint', + 'numeric', + 'decimal', + 'real', + 'double precision', + ].includes(baseType) + ) { + const num = Number(defaultValue) + return isNaN(num) ? undefined : String(num) + } + + return defaultValue + } + + async getIncrementalData( + tableName: string, + lastSync: TableMetadata, + tableConfig: TableSyncConfig + ): Promise { + const externalDataSource = this.getExternalDataSource() + if (!externalDataSource) { + throw new Error('External data source not configured') + } + + let targetSchema = + tableConfig.schema || + (externalDataSource.external as any)?.defaultSchema || + this.schema + let sourceTable = tableConfig.name || tableName.replace(/^tmp_/, '') + if (sourceTable.includes('.')) { + const parts = sourceTable.split('.', 2) + targetSchema = tableConfig.schema || parts[0] + sourceTable = parts[1] + } + + const cursorCol = + tableConfig.cursor_column || + tableConfig.cursorColumn || + tableConfig.track_column || + tableConfig.trackColumn + const timestampCol = + tableConfig.timestamp_column || tableConfig.timestampColumn + const idCol = tableConfig.id_column || tableConfig.idColumn + + let query = `SELECT * FROM ${escapeIdentifier(targetSchema)}.${escapeIdentifier(sourceTable)}` + const conditions: string[] = [] + const params: any[] = [] + + if (cursorCol) { + if ( + lastSync.lastCursorValue !== undefined && + lastSync.lastCursorValue !== null + ) { + conditions.push(`${escapeIdentifier(cursorCol)} > ?`) + params.push(lastSync.lastCursorValue) + } + } else { + if (lastSync.lastSyncTimestamp && timestampCol) { + conditions.push(`${escapeIdentifier(timestampCol)} > ?`) + params.push(new Date(lastSync.lastSyncTimestamp).toISOString()) + } + + if ( + lastSync.lastSyncId !== undefined && + lastSync.lastSyncId !== null && + idCol + ) { + conditions.push(`${escapeIdentifier(idCol)} > ?`) + params.push(lastSync.lastSyncId) + } + } + + if (conditions.length > 0) { + query += ' WHERE ' + conditions.join(' AND ') + } + + const orderColumns = ( + cursorCol ? [cursorCol] : [timestampCol, idCol].filter(Boolean) + ).map((c) => escapeIdentifier(c!)) + + if (orderColumns.length > 0) { + query += ' ORDER BY ' + orderColumns.join(', ') + ' ASC' + } + + const batchSize = Math.max( + 1, + Math.floor( + Number(tableConfig.batch_size || tableConfig.batchSize || 1000) + ) + ) + query += ` LIMIT ${batchSize}` + + return await this.executeExternalQuery(query, params) + } + + mapDataType(postgresType: string): string { + if (postgresType.endsWith('[]')) { + return 'TEXT' + } + + const baseType = postgresType + .toLowerCase() + .replace(/\(.*\)/, '') + .trim() + + const typeMap: Record = { + integer: 'INTEGER', + int: 'INTEGER', + int2: 'INTEGER', + int4: 'INTEGER', + int8: 'INTEGER', + bigint: 'INTEGER', + smallint: 'INTEGER', + serial: 'INTEGER', + bigserial: 'INTEGER', + smallserial: 'INTEGER', + numeric: 'REAL', + decimal: 'REAL', + real: 'REAL', + 'double precision': 'REAL', + float: 'REAL', + float4: 'REAL', + float8: 'REAL', + money: 'REAL', + 'character varying': 'TEXT', + varchar: 'TEXT', + character: 'TEXT', + char: 'TEXT', + text: 'TEXT', + bytea: 'BLOB', + date: 'TEXT', + time: 'TEXT', + 'time without time zone': 'TEXT', + 'time with time zone': 'TEXT', + timetz: 'TEXT', + timestamp: 'TEXT', + 'timestamp with time zone': 'TEXT', + 'timestamp without time zone': 'TEXT', + timestamptz: 'TEXT', + interval: 'TEXT', + boolean: 'INTEGER', + bool: 'INTEGER', + json: 'TEXT', + jsonb: 'TEXT', + uuid: 'TEXT', + point: 'TEXT', + line: 'TEXT', + lseg: 'TEXT', + box: 'TEXT', + path: 'TEXT', + polygon: 'TEXT', + circle: 'TEXT', + cidr: 'TEXT', + inet: 'TEXT', + macaddr: 'TEXT', + macaddr8: 'TEXT', + bit: 'INTEGER', + 'bit varying': 'INTEGER', + xml: 'TEXT', + tsvector: 'TEXT', + tsquery: 'TEXT', + } + + return typeMap[baseType] || 'TEXT' + } + + async validateTableStructure( + tableName: string, + tableConfig: TableSyncConfig + ): Promise<{ valid: boolean; errors: string[] }> { + const externalDataSource = this.getExternalDataSource() + if (!externalDataSource) { + throw new Error('External data source not configured') + } + + let targetSchema = + tableConfig.schema || + (externalDataSource.external as any)?.defaultSchema || + this.schema + let sourceTableName = tableConfig.name || tableName.replace(/^tmp_/, '') + if (sourceTableName.includes('.')) { + const parts = sourceTableName.split('.', 2) + targetSchema = tableConfig.schema || parts[0] + sourceTableName = parts[1] + } + + const errors: string[] = [] + + const tableExists = await this.executeExternalQuery( + `SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = ? + AND table_name = ? + )`, + [targetSchema, sourceTableName] + ) + + const existsVal = tableExists.rows?.[0]?.[0] + const isExisting = + existsVal === true || + existsVal === 't' || + existsVal === 'true' || + existsVal === 1 || + existsVal === '1' + + if (!tableExists.rows?.length || !isExisting) { + return { + valid: false, + errors: [ + `Table ${targetSchema}.${sourceTableName} does not exist`, + ], + } + } + + const columns = await this.getTableSchema(sourceTableName, targetSchema) + + // When a table is configured by string name without explicit sync columns, + // do not fail validation if default created_at or id columns are absent. + if (tableConfig.hasExplicitSyncColumns === false) { + return { + valid: true, + errors: [], + } + } + + const cursorColName = + tableConfig.cursor_column || + tableConfig.cursorColumn || + tableConfig.track_column || + tableConfig.trackColumn + if ( + cursorColName && + (tableConfig as any).explicitCursorColumn !== false + ) { + const cursorCol = columns.find((col) => col.name === cursorColName) + if (!cursorCol) { + errors.push( + `Cursor column ${cursorColName} not found in table ${targetSchema}.${sourceTableName}` + ) + } + } + + const tsColName = + tableConfig.timestamp_column || tableConfig.timestampColumn + if ( + tsColName && + (tableConfig as any).explicitTimestampColumn !== false + ) { + const timestampCol = columns.find((col) => col.name === tsColName) + if (!timestampCol) { + errors.push( + `Timestamp column ${tsColName} not found in table ${targetSchema}.${sourceTableName}` + ) + } else if (!this.isTimestampType(timestampCol.type)) { + errors.push( + `Column ${tsColName} in table ${targetSchema}.${sourceTableName} is not a timestamp type (found: ${timestampCol.type})` + ) + } + } + + const idColName = tableConfig.id_column || tableConfig.idColumn + if (idColName && (tableConfig as any).explicitIdColumn !== false) { + const idCol = columns.find((col) => col.name === idColName) + if (!idCol) { + errors.push( + `ID column ${idColName} not found in table ${targetSchema}.${sourceTableName}` + ) + } + } + + return { + valid: errors.length === 0, + errors, + } + } + + private isTimestampType(type: string): boolean { + const normalized = type.toLowerCase().trim() + const timestampTypes = [ + 'timestamp', + 'timestamp with time zone', + 'timestamp without time zone', + 'timestamptz', + 'date', + 'time', + 'time with time zone', + 'time without time zone', + 'datetime', + ] + return ( + timestampTypes.includes(normalized) || + normalized.startsWith('timestamp') + ) + } +} + +export { PostgresSyncSource as PostgresSync } diff --git a/plugins/postgres-sync/meta.json b/plugins/postgres-sync/meta.json new file mode 100644 index 0000000..e80bc36 --- /dev/null +++ b/plugins/postgres-sync/meta.json @@ -0,0 +1,16 @@ +{ + "version": "1.0.0", + "resources": { + "tables": {}, + "secrets": {}, + "variables": { + "dialect": "string", + "schema": "string" + } + }, + "dependencies": { + "tables": {}, + "secrets": {}, + "variables": {} + } +}