diff --git a/.agents/skills/pgpm/references/ci-cd.md b/.agents/skills/pgpm/references/ci-cd.md index 789d8e4ed2..8e9a48c90d 100644 --- a/.agents/skills/pgpm/references/ci-cd.md +++ b/.agents/skills/pgpm/references/ci-cd.md @@ -81,15 +81,13 @@ env: PGPASSWORD: password ``` -For RustFS/S3 testing (uploads, storage): +For RustFS/S3 testing (uploads, storage) — credentials only; endpoint, +provider and region are `storage_module` rows (fixtures set them): ```yaml env: - OBJECT_STORE_ENDPOINT: http://localhost:9000 - AWS_ACCESS_KEY: constructive - AWS_SECRET_KEY: constructive-dev-secret - AWS_REGION: us-east-1 - BUCKET_NAME: test-bucket + STORAGE_ACCESS_KEY_ID: constructive + STORAGE_SECRET_ACCESS_KEY: constructive-dev-secret ``` ## PGPM CLI Caching @@ -421,7 +419,8 @@ strategy: TEST_DATABASE_URL: postgres://postgres:password@localhost:5432/postgres - package: uploads/s3-streamer env: - BUCKET_NAME: test-bucket + STORAGE_ACCESS_KEY_ID: constructive + STORAGE_SECRET_ACCESS_KEY: constructive-dev-secret steps: - name: Test ${{ matrix.package }} diff --git a/.agents/skills/pgpm/references/environment-configuration.md b/.agents/skills/pgpm/references/environment-configuration.md index 9aa0541096..4f34232863 100644 --- a/.agents/skills/pgpm/references/environment-configuration.md +++ b/.agents/skills/pgpm/references/environment-configuration.md @@ -124,16 +124,17 @@ const deployOptions = getDeploymentEnvOptions(); | `SERVER_ORIGIN` | Server origin URL | | `SERVER_STRICT_AUTH` | Strict authentication mode | -### CDN/Storage +### Storage + +Endpoint, provider, region, bucket and public URL prefix are never env: they +are `metaschema_modules_public.storage_module` rows (NULL inherits the platform +database's `platform` plane). Only the credentials are env, and both are +required wherever storage is used: | Variable | Description | |----------|-------------| -| `BUCKET_PROVIDER` | Storage provider (s3, minio, rustfs, gcs) — `minio` is path-style S3-compatible storage (RustFS, MinIO) | -| `BUCKET_NAME` | Bucket name | -| `AWS_REGION` | AWS region | -| `AWS_ACCESS_KEY_ID` | AWS access key | -| `AWS_SECRET_ACCESS_KEY` | AWS secret key | -| `OBJECT_STORE_ENDPOINT` | S3-compatible endpoint URL (RustFS or MinIO; both listen on 9000) | +| `STORAGE_ACCESS_KEY_ID` | Object-store access key | +| `STORAGE_SECRET_ACCESS_KEY` | Object-store secret key | ### Jobs Configuration diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index eda7905997..99c4555c2a 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -355,12 +355,9 @@ jobs: PGPORT: 5432 PGUSER: postgres PGPASSWORD: password - CDN_ENDPOINT: http://localhost:9000 - AWS_ACCESS_KEY: constructive - AWS_SECRET_KEY: constructive-dev-secret - AWS_REGION: us-east-1 - # uploads/s3-streamer reads BUCKET_NAME; harmless for the others. - BUCKET_NAME: test-bucket + # Object-store credentials only; endpoint/provider/region are storage_module rows. + STORAGE_ACCESS_KEY_ID: constructive + STORAGE_SECRET_ACCESS_KEY: constructive-dev-secret # Pin an explicit heap cap: on smaller runners Node's memory-derived # default can land near ~2GB and OOM Jest. NODE_OPTIONS: '--max-old-space-size=4096' diff --git a/CLAUDE.md b/CLAUDE.md index e8b5c156d4..371ed913bf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -115,7 +115,7 @@ Tests require PostgreSQL. Standard PG env vars: - `PGHOST` (default: localhost), `PGPORT` (default: 5432) - `PGUSER` (default: postgres), `PGPASSWORD` (default: password) -For S3/RustFS tests: `OBJECT_STORE_ENDPOINT`, `AWS_ACCESS_KEY`, `AWS_SECRET_KEY`, `AWS_REGION` +For S3/RustFS tests: `STORAGE_ACCESS_KEY_ID`, `STORAGE_SECRET_ACCESS_KEY` (endpoint/provider/region are `storage_module` rows, not env) ## Build System diff --git a/graphile/graphile-bucket-provisioner-plugin/__tests__/plugin.test.ts b/graphile/graphile-bucket-provisioner-plugin/__tests__/plugin.test.ts index 8b7b1d949b..774863f8aa 100644 --- a/graphile/graphile-bucket-provisioner-plugin/__tests__/plugin.test.ts +++ b/graphile/graphile-bucket-provisioner-plugin/__tests__/plugin.test.ts @@ -77,9 +77,6 @@ function createMockPgClient({ entity_table_id: entityField === 'owner_id' ? 'entity-table-uuid' : null, buckets_schema: 'app_public', buckets_table: 'buckets', - endpoint: null, - public_url_prefix: null, - provider: null, allowed_origins: null, entity_schema: entityField === 'owner_id' ? 'app_public' : null, entity_table: entityField === 'owner_id' ? 'accounts' : null, diff --git a/graphile/graphile-bucket-provisioner-plugin/src/plugin.ts b/graphile/graphile-bucket-provisioner-plugin/src/plugin.ts index d66eea264f..119af0f369 100644 --- a/graphile/graphile-bucket-provisioner-plugin/src/plugin.ts +++ b/graphile/graphile-bucket-provisioner-plugin/src/plugin.ts @@ -34,9 +34,6 @@ const ALL_STORAGE_MODULES_QUERY = ` sm.entity_table_id, bs.schema_name AS buckets_schema, bt.name AS buckets_table, - sm.endpoint, - sm.public_url_prefix, - sm.provider, sm.allowed_origins, es.schema_name AS entity_schema, et.name AS entity_table @@ -57,9 +54,6 @@ interface StorageModuleRow { entity_table_id: string | null; buckets_schema: string; buckets_table: string; - endpoint: string | null; - public_url_prefix: string | null; - provider: string | null; allowed_origins: string[] | null; entity_schema?: string | null; entity_table?: string | null; diff --git a/graphile/graphile-presigned-url-plugin/__tests__/custom-key-upload.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/custom-key-upload.test.ts index 7b8d0fe6ea..e62395cb6e 100644 --- a/graphile/graphile-presigned-url-plugin/__tests__/custom-key-upload.test.ts +++ b/graphile/graphile-presigned-url-plugin/__tests__/custom-key-upload.test.ts @@ -51,7 +51,7 @@ const bucket = { } as unknown as BucketConfig; const s3 = { client: { send: jest.fn() }, bucket: 'site-bucket', region: 'us-east-1' } as unknown as S3Config; -const options = { s3 } as unknown as PresignedUrlPluginOptions; +const options: PresignedUrlPluginOptions = { credentials: { accessKeyId: 'test', secretAccessKey: 'test' } }; function fakeTx(existingHash: string, deletable = true) { const queries: Array<{ text: string; values: unknown[] }> = []; diff --git a/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts index 4ecb068bac..7fcae2e9a5 100644 --- a/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts +++ b/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts @@ -8,6 +8,11 @@ * without a server or S3. */ +const mockS3Send = jest.fn(); +jest.mock('@constructive-io/s3-utils', () => ({ + createS3Client: jest.fn(() => ({ send: mockS3Send })), +})); + import { clearFileRefFieldCache } from '../src/file-ref-registry'; import { clearBucketCache, clearStorageModuleCache } from '../src/storage-module-cache'; import type { BucketConfig, PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from '../src/types'; @@ -65,9 +70,11 @@ function storageModuleRow(overrides: Record = {}): Record { expect(target.binding).toBeNull(); expect(target.physicalName).toBe('myapp-default-public-db'); expect(target.s3.bucket).toBe('myapp-default-public-db'); - expect(target.s3.bucket).not.toBe('connection-default'); + expect(target.s3.endpoint).toBe('http://localhost:9000'); + expect(target.s3.region).toBe('us-east-1'); const resolveCall = db.queries.find((q) => /resolve_default_bucket/.test(q.text)); // scope, entity, public_access, and no explicit key: the reserved default tag. @@ -267,8 +268,7 @@ describe('resolveManagedUploadTarget', () => { it('rejects an unreconciled bucket without calling S3 or provisioning', async () => { const { resolveManagedUploadTarget } = await import('../src/managed-upload'); - const send = jest.fn(); - const baseS3 = options().s3 as S3Config; + mockS3Send.mockClear(); const db = fakeDb([ SET_CONFIG, NO_REGISTRY_ROW, @@ -278,10 +278,7 @@ describe('resolveManagedUploadTarget', () => { ]); await expect(resolveManagedUploadTarget({ - options: { - ...options(), - s3: { ...baseS3, client: { send } as any }, - }, + options: options(), withPgClient: db.withPgClient, pgSettings: null, databaseId: DATABASE_ID, @@ -289,7 +286,7 @@ describe('resolveManagedUploadTarget', () => { defaultPublicAccess: true, })).rejects.toThrow('STORAGE_BUCKET_NOT_RECONCILED'); - expect(send).not.toHaveBeenCalled(); + expect(mockS3Send).not.toHaveBeenCalled(); expect(db.queries.some((q) => /UPDATE/.test(q.text))).toBe(false); }); diff --git a/graphile/graphile-presigned-url-plugin/__tests__/physical-bucket.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/physical-bucket.test.ts new file mode 100644 index 0000000000..f8d2bbba63 --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/__tests__/physical-bucket.test.ts @@ -0,0 +1,42 @@ +import { resolveS3ForDatabase } from '../src/physical-bucket'; +import type { PresignedUrlPluginOptions, StorageModuleConfig } from '../src/types'; + +const options = { + credentials: { accessKeyId: 'platform-key', secretAccessKey: 'platform-secret' }, +} as PresignedUrlPluginOptions; + +const config = (overrides: Partial = {}): StorageModuleConfig => + ({ + id: 'sm-1', + scope: 'app', + endpoint: 'https://objects.example.com', + publicUrlPrefix: null, + provider: 'minio', + region: 'us-east-1', + connectionOverrides: [], + ...overrides, + }) as StorageModuleConfig; + +describe('resolveS3ForDatabase', () => { + it('signs against the platform plane connection', () => { + const s3 = resolveS3ForDatabase(options, config(), 'physical-bucket'); + expect(s3).toMatchObject({ + bucket: 'physical-bucket', + region: 'us-east-1', + endpoint: 'https://objects.example.com', + forcePathStyle: true, + }); + }); + + it('refuses a module row that names its own endpoint for the platform credentials', () => { + expect(() => + resolveS3ForDatabase(options, config({ connectionOverrides: ['endpoint', 'region'] }), 'physical-bucket'), + ).toThrow('STORAGE_CONNECTION_OVERRIDE_REFUSED: storage module sm-1 (scope app) sets its own endpoint, region'); + }); + + it('refuses a connection the platform plane has not configured', () => { + expect(() => resolveS3ForDatabase(options, config({ provider: null }), 'physical-bucket')).toThrow( + 'STORAGE_CONNECTION_NOT_CONFIGURED', + ); + }); +}); diff --git a/graphile/graphile-presigned-url-plugin/__tests__/s3-failure.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/s3-failure.test.ts index d9c7485d24..4a21fa9da1 100644 --- a/graphile/graphile-presigned-url-plugin/__tests__/s3-failure.test.ts +++ b/graphile/graphile-presigned-url-plugin/__tests__/s3-failure.test.ts @@ -1,8 +1,8 @@ /** * The presigned lane's diagnosis of a failed S3 call. * - * The case that motivated this: a server whose CDN_ENDPOINT is unset signs - * against the library default (its own loopback), and the transport failure + * The case that motivated this: a server signing against an endpoint it cannot + * reach (its own loopback), and the transport failure * arrives as an `AggregateError` with an empty `message` — so reporting * `err.message` gave the client a blank reason. These assert that the reason is * never blank, that it names the coordinates, and that the original error stays diff --git a/graphile/graphile-presigned-url-plugin/__tests__/s3-signer.integration.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/s3-signer.integration.test.ts index ce95f375ed..4464f52732 100644 --- a/graphile/graphile-presigned-url-plugin/__tests__/s3-signer.integration.test.ts +++ b/graphile/graphile-presigned-url-plugin/__tests__/s3-signer.integration.test.ts @@ -23,10 +23,11 @@ import type { S3Config } from '../src/types'; // --- RustFS config (matches docker-compose.yml + CI env) --- -const OBJECT_STORE_ENDPOINT = process.env.CDN_ENDPOINT || 'http://localhost:9000'; -const AWS_REGION = process.env.AWS_REGION || 'us-east-1'; -const AWS_ACCESS_KEY = process.env.AWS_ACCESS_KEY || 'constructive'; -const AWS_SECRET_KEY = process.env.AWS_SECRET_KEY || 'constructive-dev-secret'; +// The local object store (docker RustFS/MinIO); credentials from the env. +const OBJECT_STORE_ENDPOINT = 'http://localhost:9000'; +const AWS_REGION = 'us-east-1'; +const AWS_ACCESS_KEY = process.env.STORAGE_ACCESS_KEY_ID!; +const AWS_SECRET_KEY = process.env.STORAGE_SECRET_ACCESS_KEY!; const TEST_BUCKET = 'presigned-url-test-bucket'; // --- S3 client + config --- diff --git a/graphile/graphile-presigned-url-plugin/__tests__/storage-file-recorder.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/storage-file-recorder.test.ts index 12e6efb909..3743d04e76 100644 --- a/graphile/graphile-presigned-url-plugin/__tests__/storage-file-recorder.test.ts +++ b/graphile/graphile-presigned-url-plugin/__tests__/storage-file-recorder.test.ts @@ -18,6 +18,8 @@ function storageConfig( endpoint: null, publicUrlPrefix: null, provider: 'minio', + region: 'us-east-1', + connectionOverrides: [], allowedOrigins: null, uploadUrlExpirySeconds: 900, downloadUrlExpirySeconds: 3600, diff --git a/graphile/graphile-presigned-url-plugin/package.json b/graphile/graphile-presigned-url-plugin/package.json index 3e04289728..d9cb620a5c 100644 --- a/graphile/graphile-presigned-url-plugin/package.json +++ b/graphile/graphile-presigned-url-plugin/package.json @@ -1,7 +1,7 @@ { "name": "graphile-presigned-url-plugin", "version": "1.21.1", - "description": "Presigned URL upload plugin for PostGraphile v5 — requestUploadUrl mutation and downloadUrl computed field", + "description": "Presigned URL upload plugin for PostGraphile v5 \u2014 requestUploadUrl mutation and downloadUrl computed field", "author": "Constructive ", "homepage": "https://github.com/constructive-io/constructive", "license": "MIT", @@ -42,6 +42,7 @@ "dependencies": { "@aws-sdk/client-s3": "^3.1052.0", "@aws-sdk/s3-request-presigner": "^3.1052.0", + "@constructive-io/s3-utils": "workspace:^", "@pgpmjs/logger": "workspace:^", "@pgsql/quotes": "^18.2.4", "graphile-plugin-utils": "workspace:^", @@ -59,7 +60,6 @@ "postgraphile": "^5.1.3" }, "devDependencies": { - "@constructive-io/s3-utils": "workspace:^", "@types/node": "^22.19.11", "makage": "^0.8.0" } diff --git a/graphile/graphile-presigned-url-plugin/src/download-url-field.ts b/graphile/graphile-presigned-url-plugin/src/download-url-field.ts index 75ec277323..d6f47f5fce 100644 --- a/graphile/graphile-presigned-url-plugin/src/download-url-field.ts +++ b/graphile/graphile-presigned-url-plugin/src/download-url-field.ts @@ -27,7 +27,7 @@ import type { GraphileConfig } from 'graphile-config'; import { withSystemLaneClient } from 'graphile-plugin-utils'; import { DOWNLOAD_URL_FIELD } from 'graphile-storage-registry'; -import { resolveS3, resolveS3ForDatabase } from './physical-bucket'; +import { resolveS3ForDatabase } from './physical-bucket'; import { withRequestPgClient } from './request-pg-client'; import { generatePresignedGetUrl } from './s3-signer'; import { loadAllStorageModules, resolveStorageConfigFromCodec, storedPhysicalName } from './storage-module-cache'; @@ -110,48 +110,38 @@ export function createDownloadUrlPlugin( return lambda($combined, async ({ key, isPublic, filename, bucketId, withPgClient, pgSettings }: any) => { if (!key) return null; - let s3ForDb = resolveS3(options); - let downloadUrlExpirySeconds = 3600; - try { - if (withPgClient && pgSettings) { - const databaseId = await withRequestPgClient(withPgClient, pgSettings, async (pgClient) => { - const dbResult = await pgClient.query({ - text: `SELECT jwt_private.current_database_id() AS id`, - }); - return (dbResult.rows[0]?.id as string | undefined) ?? null; + if (!withPgClient || !pgSettings) return null; + + const databaseId = await withRequestPgClient(withPgClient, pgSettings, async (pgClient) => { + const dbResult = await pgClient.query({ + text: `SELECT jwt_private.current_database_id() AS id`, + }); + return (dbResult.rows[0]?.id as string | undefined) ?? null; + }); + // Module registration is server config, not user data: + // resolve it in the system lane's bounded role. + const config = databaseId + ? resolveStorageConfigFromCodec( + capturedCodec, + await withSystemLaneClient(withPgClient, (pgClient) => loadAllStorageModules(pgClient, databaseId)), + ) + : null; + const resolved = config && bucketId + ? await withRequestPgClient(withPgClient, pgSettings, async (pgClient) => { + // Look up the stored physical coordinate for scoped S3 resolution + const bucketResult = await pgClient.query({ + text: `SELECT key, physical_name FROM ${config.bucketsQualifiedName} WHERE id = $1 LIMIT 1`, + values: [bucketId], }); - // Module registration is server config, not user data: - // resolve it in the system lane's bounded role. - const config = databaseId - ? resolveStorageConfigFromCodec( - capturedCodec, - await withSystemLaneClient(withPgClient, (pgClient) => loadAllStorageModules(pgClient, databaseId)), - ) - : null; - const resolved = config && bucketId - ? await withRequestPgClient(withPgClient, pgSettings, async (pgClient) => { - // Look up the stored physical coordinate for scoped S3 resolution - const bucketResult = await pgClient.query({ - text: `SELECT key, physical_name FROM ${config.bucketsQualifiedName} WHERE id = $1 LIMIT 1`, - values: [bucketId], - }); - const row = bucketResult.rows[0] as { key: string; physical_name?: string | null } | undefined; - return row ? { config, physicalName: storedPhysicalName(row) } : null; - }) - : null; - if (resolved) { - if (resolved.physicalName === null) { - // No physical bucket was ever provisioned — no object can exist. - return null; - } - downloadUrlExpirySeconds = resolved.config.downloadUrlExpirySeconds; - s3ForDb = resolveS3ForDatabase(options, resolved.config, resolved.physicalName); - } - } - } catch { - // Fall back to global config if lookup fails - } - + const row = bucketResult.rows[0] as { key: string; physical_name?: string | null } | undefined; + return row ? { config, physicalName: storedPhysicalName(row) } : null; + }) + : null; + // No resolvable bucket row, or no physical bucket was ever + // provisioned: no object can exist. + if (!resolved || resolved.physicalName === null) return null; + + const s3ForDb = resolveS3ForDatabase(options, resolved.config, resolved.physicalName); if (isPublic && s3ForDb.publicUrlPrefix) { return `${s3ForDb.publicUrlPrefix}/${s3ForDb.bucket}/${key}`; } @@ -159,7 +149,7 @@ export function createDownloadUrlPlugin( return generatePresignedGetUrl( s3ForDb, key, - downloadUrlExpirySeconds, + resolved.config.downloadUrlExpirySeconds, filename || undefined, ); }); diff --git a/graphile/graphile-presigned-url-plugin/src/index.ts b/graphile/graphile-presigned-url-plugin/src/index.ts index d4c1963c5d..19800d5d6b 100644 --- a/graphile/graphile-presigned-url-plugin/src/index.ts +++ b/graphile/graphile-presigned-url-plugin/src/index.ts @@ -9,18 +9,13 @@ * @example * ```typescript * import { PresignedUrlPreset } from 'graphile-presigned-url-plugin'; - * import { S3Client } from '@aws-sdk/client-s3'; - * - * const s3Client = new S3Client({ region: 'us-east-1' }); - * * const preset = { * extends: [ * PresignedUrlPreset({ - * s3: { - * client: s3Client, - * bucket: 'my-uploads', - * publicUrlPrefix: 'https://cdn.example.com', - * }, + * credentials: () => ({ + * accessKeyId: process.env.STORAGE_ACCESS_KEY_ID!, + * secretAccessKey: process.env.STORAGE_SECRET_ACCESS_KEY!, + * }), * }), * ], * }; @@ -49,9 +44,10 @@ export { } from './managed-upload'; export { assertBucketReconciled, - resolveS3, resolveS3ForDatabase, StorageBucketNotReconciledError, + StorageConnectionNotConfiguredError, + StorageConnectionOverrideError, } from './physical-bucket'; export { createPresignedUrlPlugin,PresignedUrlPlugin } from './plugin'; export { PresignedUrlPreset } from './preset'; @@ -65,6 +61,6 @@ export type { RequestUploadUrlInput, RequestUploadUrlPayload, S3Config, - S3ConfigOrGetter, + StorageCredentials, StorageModuleConfig, } from './types'; diff --git a/graphile/graphile-presigned-url-plugin/src/physical-bucket.ts b/graphile/graphile-presigned-url-plugin/src/physical-bucket.ts index 9778ef73e8..cbbcfea796 100644 --- a/graphile/graphile-presigned-url-plugin/src/physical-bucket.ts +++ b/graphile/graphile-presigned-url-plugin/src/physical-bucket.ts @@ -5,9 +5,20 @@ * A logical bucket belongs to a tenant; a physical bucket is an S3 name. The * mapping is recorded on the bucket row by the storage reconciler, and that * value is the only coordinate anything reads — no name is ever recomputed. + * The connection (endpoint/provider/region) is the storage module's; only the + * credentials come from the plugin options. */ -import type { BucketConfig, PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from './types'; +import type { S3Client } from '@aws-sdk/client-s3'; +import { createS3Client, type StorageProvider } from '@constructive-io/s3-utils'; + +import type { + BucketConfig, + PresignedUrlPluginOptions, + S3Config, + StorageCredentials, + StorageModuleConfig, +} from './types'; export class StorageBucketNotReconciledError extends Error { readonly code = 'STORAGE_BUCKET_NOT_RECONCILED'; @@ -27,46 +38,85 @@ export class StorageBucketNotReconciledError extends Error { } } -/** - * Resolve the plugin's S3 connection (credentials, endpoint, region), memoizing - * a lazy getter on first use. - * - * `s3.bucket` on the result is the deployment's *default* physical bucket. It is - * a connection default only — never a tenant's bucket. Every upload path - * resolves its physical bucket from the tenant's bucket row. - */ -export function resolveS3(options: PresignedUrlPluginOptions): S3Config { - if (typeof options.s3 === 'function') { - const resolved = options.s3(); - options.s3 = resolved; - return resolved; +export class StorageConnectionNotConfiguredError extends Error { + readonly code = 'STORAGE_CONNECTION_NOT_CONFIGURED'; + readonly extensions = { code: 'STORAGE_CONNECTION_NOT_CONFIGURED' }; + + constructor(storageConfig: StorageModuleConfig, missing: string[]) { + super( + `STORAGE_CONNECTION_NOT_CONFIGURED: storage module ${storageConfig.id} (scope ` + + `${storageConfig.scope}) has no ${missing.join(', ')}; set them on ` + + 'the platform database\'s platform storage_module row', + ); + this.name = 'StorageConnectionNotConfiguredError'; + } +} + +export class StorageConnectionOverrideError extends Error { + readonly code = 'STORAGE_CONNECTION_OVERRIDE_REFUSED'; + readonly extensions = { code: 'STORAGE_CONNECTION_OVERRIDE_REFUSED' }; + + constructor(storageConfig: StorageModuleConfig) { + super( + `STORAGE_CONNECTION_OVERRIDE_REFUSED: storage module ${storageConfig.id} (scope ` + + `${storageConfig.scope}) sets its own ${storageConfig.connectionOverrides.join(', ')}; ` + + 'storage credentials only sign for the platform database\'s platform plane, so clear ' + + 'these columns on this row', + ); + this.name = 'StorageConnectionOverrideError'; } - return options.s3; } +function resolveCredentials(options: PresignedUrlPluginOptions): StorageCredentials { + if (typeof options.credentials === 'function') { + options.credentials = options.credentials(); + } + return options.credentials; +} + +/** One S3 client per resolved connection, shared by every bucket on it. */ +const clients = new Map(); /** - * Build the S3 config for a *known* physical bucket. `physicalName` is - * required — callers must resolve the coordinate from the stored row value - * before getting here. No name is ever recomputed. + * Build the S3 config for a *known* physical bucket on the storage module's + * connection. `physicalName` is required — callers must resolve the coordinate + * from the stored row value before getting here. No name is ever recomputed. */ export function resolveS3ForDatabase( options: PresignedUrlPluginOptions, storageConfig: StorageModuleConfig, physicalName: string, ): S3Config { - const globalS3 = resolveS3(options); - const publicUrlPrefix = storageConfig.publicUrlPrefix != null - ? storageConfig.publicUrlPrefix - : globalS3.publicUrlPrefix; + const { endpoint, provider, region, publicUrlPrefix } = storageConfig; + if (storageConfig.connectionOverrides.length > 0) { + throw new StorageConnectionOverrideError(storageConfig); + } + if (!provider || !region) { + throw new StorageConnectionNotConfiguredError(storageConfig, [ + ...(provider ? [] : ['provider']), + ...(region ? [] : ['region']), + ]); + } - if (physicalName === globalS3.bucket && publicUrlPrefix === globalS3.publicUrlPrefix) { - return globalS3; + const cacheKey = JSON.stringify([provider, endpoint, region]); + let client = clients.get(cacheKey); + if (!client) { + const { accessKeyId, secretAccessKey } = resolveCredentials(options); + client = createS3Client({ + provider: provider as StorageProvider, + region, + accessKeyId, + secretAccessKey, + ...(endpoint ? { endpoint } : {}), + }); + clients.set(cacheKey, client); } return { - ...globalS3, + client, bucket: physicalName, + region, + ...(endpoint ? { endpoint, forcePathStyle: provider !== 's3' } : {}), ...(publicUrlPrefix != null ? { publicUrlPrefix } : {}), }; } diff --git a/graphile/graphile-presigned-url-plugin/src/preset.ts b/graphile/graphile-presigned-url-plugin/src/preset.ts index 3ab986721e..7d1c1d3015 100644 --- a/graphile/graphile-presigned-url-plugin/src/preset.ts +++ b/graphile/graphile-presigned-url-plugin/src/preset.ts @@ -18,18 +18,13 @@ import type { PresignedUrlPluginOptions } from './types'; * @example * ```typescript * import { PresignedUrlPreset } from 'graphile-presigned-url-plugin'; - * import { S3Client } from '@aws-sdk/client-s3'; - * - * const s3Client = new S3Client({ region: 'us-east-1' }); - * * const preset = { * extends: [ * PresignedUrlPreset({ - * s3: { - * client: s3Client, - * bucket: 'my-bucket', - * publicUrlPrefix: 'https://cdn.example.com', - * }, + * credentials: () => ({ + * accessKeyId: process.env.STORAGE_ACCESS_KEY_ID!, + * secretAccessKey: process.env.STORAGE_SECRET_ACCESS_KEY!, + * }), * }), * ], * }; diff --git a/graphile/graphile-presigned-url-plugin/src/s3-failure.ts b/graphile/graphile-presigned-url-plugin/src/s3-failure.ts index e41a0fe7b1..58c25015dd 100644 --- a/graphile/graphile-presigned-url-plugin/src/s3-failure.ts +++ b/graphile/graphile-presigned-url-plugin/src/s3-failure.ts @@ -6,9 +6,9 @@ * per-address `errors` hold the `ECONNREFUSED`, and a hung socket arrives as a * bare wrapper around its `cause`. Anything that reports `err.message` verbatim * therefore hands the client a blank reason — which is how a server signing - * against the wrong endpoint (a missing `CDN_ENDPOINT`, so the library default - * `http://localhost:9000`, i.e. the pod's own loopback) presents as an upload - * that fails with nothing to diagnose. + * against the wrong endpoint (a `storage_module.endpoint` naming a host the + * server cannot reach) presents as an upload that fails with nothing to + * diagnose. * * So a failure is described by walking to where the words actually are, and * re-thrown naming the coordinates it was talking to, with the original kept as diff --git a/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts b/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts index 784ffcc909..61d0656563 100644 --- a/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts +++ b/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts @@ -38,6 +38,12 @@ const storageModuleCache = new LRUCache({ * SQL query to resolve ALL storage modules for a database, whatever their * scope. Returns each module with its entity table names so callers can * classify entity-keyed planes and resolve owners. + * + * Endpoint, provider and region are always the platform database's `platform` + * plane — the only object store the deployment's STORAGE_* credentials belong + * to — and `connection_overrides` names any a row sets differently, which + * signing then refuses. A NULL `public_url_prefix` inherits the platform's. + * All in this same (cached) query. */ const ALL_STORAGE_MODULES_QUERY = ` SELECT @@ -49,9 +55,15 @@ const ALL_STORAGE_MODULES_QUERY = ` fs.schema_name AS files_schema, ft.name AS files_table, ps.schema_name AS private_schema, - sm.endpoint, - sm.public_url_prefix, - sm.provider, + psm.endpoint AS endpoint, + coalesce(sm.public_url_prefix, psm.public_url_prefix) AS public_url_prefix, + psm.provider AS provider, + psm.region AS region, + array_remove(ARRAY[ + CASE WHEN sm.endpoint IS DISTINCT FROM psm.endpoint AND sm.endpoint IS NOT NULL THEN 'endpoint' END, + CASE WHEN sm.provider IS DISTINCT FROM psm.provider AND sm.provider IS NOT NULL THEN 'provider' END, + CASE WHEN sm.region IS DISTINCT FROM psm.region AND sm.region IS NOT NULL THEN 'region' END + ], NULL) AS connection_overrides, sm.allowed_origins, sm.upload_url_expiry_seconds, sm.download_url_expiry_seconds, @@ -74,6 +86,10 @@ const ALL_STORAGE_MODULES_QUERY = ` LEFT JOIN metaschema_public.schema ps ON ps.id = sm.private_schema_id LEFT JOIN metaschema_public.table et ON et.id = sm.entity_table_id LEFT JOIN metaschema_public.schema es ON es.id = et.schema_id + LEFT JOIN metaschema_modules_public.storage_module psm + ON psm.scope = 'platform' + AND psm.key = 'default' + AND psm.database_id = (SELECT d.id FROM metaschema_public.database d WHERE d.platform) WHERE sm.database_id = $1 `; @@ -89,6 +105,8 @@ interface StorageModuleRow { endpoint: string | null; public_url_prefix: string | null; provider: string | null; + region: string | null; + connection_overrides: string[] | null; allowed_origins: string[] | null; upload_url_expiry_seconds: number | null; download_url_expiry_seconds: number | null; @@ -128,6 +146,8 @@ function buildConfig(row: StorageModuleRow): StorageModuleConfig { endpoint: row.endpoint, publicUrlPrefix: row.public_url_prefix, provider: row.provider, + region: row.region, + connectionOverrides: row.connection_overrides ?? [], allowedOrigins: row.allowed_origins, uploadUrlExpirySeconds: row.upload_url_expiry_seconds ?? DEFAULT_UPLOAD_URL_EXPIRY_SECONDS, downloadUrlExpirySeconds: row.download_url_expiry_seconds ?? DEFAULT_DOWNLOAD_URL_EXPIRY_SECONDS, diff --git a/graphile/graphile-presigned-url-plugin/src/types.ts b/graphile/graphile-presigned-url-plugin/src/types.ts index 65b477ead2..785517fd89 100644 --- a/graphile/graphile-presigned-url-plugin/src/types.ts +++ b/graphile/graphile-presigned-url-plugin/src/types.ts @@ -48,14 +48,24 @@ export interface StorageModuleConfig { /** Qualified entity table name for ownerId lookups (NULL for app-level) */ entityQualifiedName: string | null; - // --- S3 connection config (NULL in DB = use global env/plugin defaults) --- + // --- Object-store connection: the only source of storage coordinates. --- + // Effective values: a NULL column on this plane inherits the platform + // database's `platform` plane row, resolved in the same cached query. - /** S3-compatible API endpoint URL (per-database override) */ + /** S3-compatible API endpoint URL presigned URLs are signed for (NULL = AWS S3) */ endpoint: string | null; - /** Public URL prefix for generating download URLs (per-database override) */ + /** Public URL prefix for generating download URLs */ publicUrlPrefix: string | null; - /** Storage provider type: 'minio', 's3', 'gcs', etc. (per-database override) */ + /** Storage provider type: 'minio', 'rustfs', 's3', 'gcs', etc. */ provider: string | null; + /** Object-store region */ + region: string | null; + /** + * Coordinates (endpoint/provider/region) this module's row sets differently + * from the platform plane. Signing refuses them: the credentials only + * belong to the platform object store. + */ + connectionOverrides: string[]; /** CORS allowed origins (per-database override, NULL = use global fallback) */ allowedOrigins: string[] | null; @@ -177,7 +187,7 @@ export interface FileProjection { } /** - * S3 configuration for the presigned URL plugin. + * S3 coordinates for one physical bucket, built from a resolved storage module. */ export interface S3Config { /** S3 client instance */ @@ -195,18 +205,21 @@ export interface S3Config { } /** - * S3 configuration or a lazy getter that returns it on first use. - * When a function is provided, it will only be called when the first - * mutation or resolver actually needs the S3 client — avoiding eager - * env-var reads and S3Client creation at module import time. + * Object-store credentials. The only storage input that does not come from a + * `storage_module` row (`STORAGE_ACCESS_KEY_ID` / `STORAGE_SECRET_ACCESS_KEY`). */ -export type S3ConfigOrGetter = S3Config | (() => S3Config); +export interface StorageCredentials { + accessKeyId: string; + secretAccessKey: string; +} /** * Plugin options for the presigned URL plugin. + * + * Only credentials: endpoint, provider, region and public URL prefix are read + * from the resolved storage module. A getter is called on first use, so a + * server that never touches storage never needs the credentials. */ export interface PresignedUrlPluginOptions { - /** S3 configuration (concrete or lazy getter) */ - s3: S3ConfigOrGetter; - + credentials: StorageCredentials | (() => StorageCredentials); } diff --git a/graphile/graphile-settings/__tests__/presigned-url-resolver.test.ts b/graphile/graphile-settings/__tests__/presigned-url-resolver.test.ts index 49ecd6b151..acd768dcf4 100644 --- a/graphile/graphile-settings/__tests__/presigned-url-resolver.test.ts +++ b/graphile/graphile-settings/__tests__/presigned-url-resolver.test.ts @@ -1,78 +1,33 @@ /** - * Unit tests for the connection-default S3 configuration. + * Unit tests for the object-store credentials (the only storage env input). */ -interface CdnOptions { - provider?: string; - bucketName?: string; - awsRegion?: string; - awsAccessKey?: string; - awsSecretKey?: string; - endpoint?: string; - publicUrlPrefix?: string; -} - -async function loadResolverModule(cdn: CdnOptions | undefined) { +async function loadResolverModule(storage: { accessKeyId?: string; secretAccessKey?: string } | undefined) { jest.resetModules(); jest.doMock('@constructive-io/graphql-env', () => ({ - getEnvOptions: jest.fn(() => ({ cdn })), - })); - jest.doMock('@constructive-io/s3-utils', () => ({ - createS3Client: jest.fn(() => ({ send: jest.fn() })), - })); - jest.doMock('@pgpmjs/logger', () => ({ - Logger: jest.fn().mockImplementation(() => ({ info: jest.fn() })), + getEnvOptions: jest.fn(() => ({ storage })), })); return import('../src/presigned-url-resolver'); } -const BASE_CDN: CdnOptions = { - provider: 'minio', - bucketName: 'connection-default', - awsRegion: 'us-east-1', - awsAccessKey: 'access', - awsSecretKey: 'secret', - endpoint: 'http://localhost:9000', - publicUrlPrefix: 'https://cdn.example.com', -}; - -describe('getPresignedUrlS3Config', () => { - it('returns the configured connection-default bucket', async () => { - const { getPresignedUrlS3Config } = await loadResolverModule(BASE_CDN); - - expect(getPresignedUrlS3Config()).toEqual(expect.objectContaining({ - bucket: 'connection-default', - region: 'us-east-1', - endpoint: 'http://localhost:9000', - publicUrlPrefix: 'https://cdn.example.com', - })); - }); - - it('caches the initialized S3 configuration', async () => { - const { getPresignedUrlS3Config } = await loadResolverModule(BASE_CDN); - - expect(getPresignedUrlS3Config()).toBe(getPresignedUrlS3Config()); - }); - - it('requires a CDN bucket name for the connection default', async () => { - const { getPresignedUrlS3Config } = await loadResolverModule({ - ...BASE_CDN, - bucketName: undefined, +describe('getStorageCredentials', () => { + it('returns the dedicated storage credentials', async () => { + const { getStorageCredentials } = await loadResolverModule({ + accessKeyId: 'access', + secretAccessKey: 'secret', }); - expect(() => getPresignedUrlS3Config()).toThrow(/CDN_BUCKET_NAME/); + expect(getStorageCredentials()).toEqual({ accessKeyId: 'access', secretAccessKey: 'secret' }); }); - it('requires CDN configuration and credentials', async () => { - const missingConfig = await loadResolverModule(undefined); - expect(() => missingConfig.getPresignedUrlS3Config()).toThrow(/CDN config not found/); - - const missingCredentials = await loadResolverModule({ - ...BASE_CDN, - awsAccessKey: undefined, - }); - expect(() => missingCredentials.getPresignedUrlS3Config()).toThrow(/S3 credentials/); + it('fails fast naming both env vars when either is missing', async () => { + for (const storage of [undefined, { accessKeyId: 'access' }, { secretAccessKey: 'secret' }]) { + const { getStorageCredentials } = await loadResolverModule(storage); + expect(() => getStorageCredentials()).toThrow( + /STORAGE_ACCESS_KEY_ID and STORAGE_SECRET_ACCESS_KEY/, + ); + } }); }); diff --git a/graphile/graphile-settings/__tests__/upload-resolver.test.ts b/graphile/graphile-settings/__tests__/upload-resolver.test.ts index 4577e714d0..92e95d5c85 100644 --- a/graphile/graphile-settings/__tests__/upload-resolver.test.ts +++ b/graphile/graphile-settings/__tests__/upload-resolver.test.ts @@ -32,9 +32,10 @@ function storageModuleRow(): Record { files_schema: 'storage_public', files_table: 'app_files', private_schema: 'storage_private', - endpoint: null, + endpoint: 'http://localhost:9000', public_url_prefix: 'https://cdn.example.com', provider: 'minio', + region: 'us-east-1', allowed_origins: null, upload_url_expiry_seconds: null, download_url_expiry_seconds: null, @@ -139,15 +140,7 @@ async function loadUploadResolverModule(opts: { detectedContentType: string }) { jest.doMock('@constructive-io/graphql-env', () => ({ getEnvOptions: jest.fn(() => ({ - cdn: { - provider: 'minio', - bucketName: 'myapp', - awsRegion: 'us-east-1', - awsAccessKey: 'test', - awsSecretKey: 'test', - endpoint: 'http://localhost:9000', - publicUrlPrefix: 'https://cdn.example.com', - }, + storage: { accessKeyId: 'test', secretAccessKey: 'test' }, })), })); diff --git a/graphile/graphile-settings/package.json b/graphile/graphile-settings/package.json index f6452798e3..068a9d55c2 100644 --- a/graphile/graphile-settings/package.json +++ b/graphile/graphile-settings/package.json @@ -34,7 +34,6 @@ "@constructive-io/graphql-env": "workspace:^", "@constructive-io/graphql-types": "workspace:^", "@constructive-io/s3-streamer": "workspace:^", - "@constructive-io/s3-utils": "workspace:^", "@constructive-io/upload-names": "workspace:^", "@dataplan/json": "1.0.1", "@dataplan/pg": "1.1.1", @@ -76,6 +75,7 @@ "tamedevil": "0.1.1" }, "devDependencies": { + "@constructive-io/s3-utils": "workspace:^", "@types/cors": "^2.8.17", "@types/express": "^5.0.6", "@types/pg": "^8.20.4", diff --git a/graphile/graphile-settings/src/index.ts b/graphile/graphile-settings/src/index.ts index 9a036a87e6..982e91a00a 100644 --- a/graphile/graphile-settings/src/index.ts +++ b/graphile/graphile-settings/src/index.ts @@ -65,4 +65,4 @@ export * from './presets/index'; export { makePgService }; // Presigned URL utilities -export { getPresignedUrlS3Config } from './presigned-url-resolver'; +export { getStorageCredentials } from './presigned-url-resolver'; diff --git a/graphile/graphile-settings/src/presets/constructive-preset.ts b/graphile/graphile-settings/src/presets/constructive-preset.ts index bf15e4cb24..f99327d7af 100644 --- a/graphile/graphile-settings/src/presets/constructive-preset.ts +++ b/graphile/graphile-settings/src/presets/constructive-preset.ts @@ -25,7 +25,7 @@ import { PgTypeMappingsPreset, RequiredInputPreset } from '../plugins'; -import { getPresignedUrlS3Config } from '../presigned-url-resolver'; +import { getStorageCredentials } from '../presigned-url-resolver'; import { constructiveUploadFieldDefinitions } from '../upload-resolver'; /** @@ -199,7 +199,7 @@ export function createConstructivePreset( if (opts.enablePresignedUploads) { presets.push( PresignedUrlPreset({ - s3: getPresignedUrlS3Config, + credentials: getStorageCredentials, }), BucketProvisionerPreset() ); diff --git a/graphile/graphile-settings/src/presigned-url-resolver.ts b/graphile/graphile-settings/src/presigned-url-resolver.ts index 5e12f141e6..30e9a6b5b2 100644 --- a/graphile/graphile-settings/src/presigned-url-resolver.ts +++ b/graphile/graphile-settings/src/presigned-url-resolver.ts @@ -1,82 +1,22 @@ /** - * Presigned URL resolver for the Constructive presigned URL plugin. + * Object-store credentials for the presigned URL plugin. * - * Reads CDN/S3 configuration from the standard env system - * (getEnvOptions → pgpmDefaults + config files + env vars) and lazily - * initializes an S3Client on first use. - * - * Follows the same lazy-init pattern as upload-resolver.ts. + * Credentials are the only storage input read from the environment + * (`STORAGE_ACCESS_KEY_ID` / `STORAGE_SECRET_ACCESS_KEY`). Endpoint, provider, + * region, bucket and public URL prefix are resolved per request from the + * tenant's `storage_module` row by the plugin itself. */ import { getEnvOptions } from '@constructive-io/graphql-env'; -import { createS3Client } from '@constructive-io/s3-utils'; -import { Logger } from '@pgpmjs/logger'; -import type { S3Config } from 'graphile-presigned-url-plugin'; - -const log = new Logger('presigned-url-resolver'); - -let s3Config: S3Config | null = null; - -/** - * Lazily initialize and return the S3Config for the presigned URL plugin. - * - * Reads CDN config on first call via getEnvOptions() (which already merges - * pgpmDefaults → config file → env vars), creates an S3Client, and caches - * the result. Same CDN config as upload-resolver.ts. - * - * NOTE: The `bucket` field here is only the connection's default and is never - * uploaded to. Every managed upload names its bucket explicitly, resolved from - * the tenant's logical bucket row; there is no environment-global upload - * bucket. - */ -export function getPresignedUrlS3Config(): S3Config { - if (s3Config) return s3Config; - - const { cdn } = getEnvOptions(); - - if (!cdn) { - throw new Error( - '[presigned-url-resolver] CDN config not found. ' + - 'Ensure CDN environment variables (AWS_ACCESS_KEY, AWS_SECRET_KEY, etc.) ' + - 'are set or that pgpmDefaults provides CDN fields.', - ); - } - - const { bucketName, awsRegion, awsAccessKey, awsSecretKey, endpoint, publicUrlPrefix } = cdn; +import type { StorageCredentials } from 'graphile-presigned-url-plugin'; - if (!awsAccessKey || !awsSecretKey) { +export function getStorageCredentials(): StorageCredentials { + const { accessKeyId, secretAccessKey } = getEnvOptions().storage ?? {}; + if (!accessKeyId || !secretAccessKey) { throw new Error( - '[presigned-url-resolver] Missing S3 credentials. ' + - 'Set AWS_ACCESS_KEY and AWS_SECRET_KEY environment variables.', + 'STORAGE_CREDENTIALS_MISSING: object storage requires STORAGE_ACCESS_KEY_ID and ' + + 'STORAGE_SECRET_ACCESS_KEY', ); } - - if (!bucketName) { - throw new Error( - '[presigned-url-resolver] Missing CDN bucket name. ' + - 'Set CDN_BUCKET_NAME environment variable.', - ); - } - - log.info( - `[presigned-url-resolver] Initializing: bucket=${bucketName} endpoint=${endpoint}`, - ); - - const client = createS3Client({ - provider: (cdn.provider || 'minio') as any, - region: awsRegion, - accessKeyId: awsAccessKey, - secretAccessKey: awsSecretKey, - ...(endpoint ? { endpoint } : {}), - }); - - s3Config = { - client, - bucket: bucketName, - region: awsRegion, - publicUrlPrefix, - ...(endpoint ? { endpoint, forcePathStyle: true } : {}), - }; - - return s3Config; + return { accessKeyId, secretAccessKey }; } diff --git a/graphile/graphile-settings/src/upload-resolver.ts b/graphile/graphile-settings/src/upload-resolver.ts index e9781231c1..3574549353 100644 --- a/graphile/graphile-settings/src/upload-resolver.ts +++ b/graphile/graphile-settings/src/upload-resolver.ts @@ -9,26 +9,18 @@ * It used to be a second storage model: stream to `BUCKET_NAME` under a random * key, hand back a URL, record nothing. Objects written that way belonged to no * database, could not be deduplicated, listed, or access-controlled, and storage - * GC could not see that a document still pointed at them. There is no - * environment bucket in this path any more; `cdn.*` supplies S3 credentials and - * an endpoint only. + * GC could not see that a document still pointed at them. Nothing in this path + * comes from the environment but the object-store credentials: the bucket and + * its connection (endpoint/provider/region) are the tenant's storage module. * * Compatibility: `image`/`upload` columns still receive `url` alongside the new * `id`/`key`/`bucket_id`/`size` fields, so existing readers of `photo.url` keep * working while they migrate to `id` + the files row's late-bound `downloadUrl`. * - * ENV VARS (S3 connection only): - * BUCKET_PROVIDER - 'minio' | 's3' (default: 'minio') - * AWS_REGION - AWS region (default: 'us-east-1') - * Defaults come from `pgpmDefaults.cdn` (dev-only values; set these in production). - * AWS_ACCESS_KEY - access key - * AWS_SECRET_KEY - secret key - * CDN_ENDPOINT - S3-compatible endpoint + * ENV VARS: STORAGE_ACCESS_KEY_ID, STORAGE_SECRET_ACCESS_KEY (required). */ -import { getEnvOptions } from '@constructive-io/graphql-env'; import Streamer from '@constructive-io/s3-streamer'; -import { Logger } from '@pgpmjs/logger'; import { createHash, randomUUID } from 'crypto'; import { finalizeStagedUpload, @@ -45,56 +37,16 @@ import type { import { checkTypeAgreement } from 'mime-bytes'; import { Transform } from 'stream'; -import { getPresignedUrlS3Config } from './presigned-url-resolver'; +import { getStorageCredentials } from './presigned-url-resolver'; -const log = new Logger('upload-resolver'); const DEFAULT_IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/svg+xml']; -let streamer: Streamer | null = null; - -/** - * The S3 streamer, built from the CDN connection settings. - * - * Deliberately constructed with no `defaultBucket`: every upload names the - * bucket it resolved, and a default here would be an environment-owned bucket - * standing in for a tenant's. - */ -function getStreamer(): Streamer { - if (streamer) return streamer; - - const { cdn } = getEnvOptions(); - - if (process.env.NODE_ENV === 'production' && (!cdn.awsAccessKey || !cdn.awsSecretKey)) { - log.warn('[upload-resolver] WARNING: CDN credentials not configured in production.'); - } - - const provider = cdn.provider; - log.info(`[upload-resolver] Initializing: provider=${provider}`); - - streamer = new Streamer({ - provider, - awsRegion: cdn.awsRegion, - awsAccessKey: cdn.awsAccessKey, - awsSecretKey: cdn.awsSecretKey, - endpoint: cdn.endpoint, - }); - - return streamer; -} - -/** - * The upload lane's view of the presigned plugin's options: the same S3 - * connection the presigned lane uses, so both transports resolve identical - * coordinates for a bucket. - * - * Built on first upload rather than at import time. - */ let managedOptions: PresignedUrlPluginOptions | null = null; function getManagedOptions(): PresignedUrlPluginOptions { if (!managedOptions) { managedOptions = { - s3: getPresignedUrlS3Config, + credentials: getStorageCredentials, }; } return managedOptions; @@ -213,7 +165,7 @@ async function uploadResolver( ); } - const s3 = getStreamer(); + const s3 = new Streamer({ client: target.s3.client, defaultBucket: target.physicalName }); const { filename } = upload; // Validate before persisting: content type comes from the leading bytes, not diff --git a/graphql/env/__tests__/__snapshots__/merge.test.ts.snap b/graphql/env/__tests__/__snapshots__/merge.test.ts.snap index 796744fd63..a36297bec2 100644 --- a/graphql/env/__tests__/__snapshots__/merge.test.ts.snap +++ b/graphql/env/__tests__/__snapshots__/merge.test.ts.snap @@ -16,15 +16,6 @@ exports[`getEnvOptions merges pgpm defaults, graphql defaults, config, env, and "roleName": "env_role", "routingSchema": "routing_public", }, - "cdn": { - "awsAccessKey": "constructive", - "awsRegion": "us-east-1", - "awsSecretKey": "constructive-dev-secret", - "bucketName": "test-bucket", - "endpoint": "http://localhost:9000", - "provider": "minio", - "publicUrlPrefix": "http://localhost:9000", - }, "db": { "connections": { "admin": { @@ -109,5 +100,6 @@ exports[`getEnvOptions merges pgpm defaults, graphql defaults, config, env, and "port": 587, "secure": false, }, + "storage": {}, } `; diff --git a/graphql/explorer/src/resolvers/uploads.ts b/graphql/explorer/src/resolvers/uploads.ts deleted file mode 100644 index 3004ce2aee..0000000000 --- a/graphql/explorer/src/resolvers/uploads.ts +++ /dev/null @@ -1,98 +0,0 @@ -import Streamer from '@constructive-io/s3-streamer'; -import uploadNames from '@constructive-io/upload-names'; -import type { BucketProvider } from '@pgpmjs/types'; -import { ReadStream } from 'fs'; -import type { GraphQLResolveInfo } from 'graphql'; - -interface UploaderOptions { - bucketName: string; - awsRegion: string; - awsSecretKey: string; - awsAccessKey: string; - endpoint?: string; - provider?: BucketProvider; -} - -interface Upload { - createReadStream: () => NodeJS.ReadableStream; - filename: string; - mimetype: string; - encoding: string; -} - -interface UploadPluginInfo { - tags: { [key: string]: any }; - type: string; -} - -export class UploadHandler { - private streamer: Streamer; - - constructor(private options: UploaderOptions) { - this.streamer = new Streamer({ - defaultBucket: options.bucketName, - awsRegion: options.awsRegion, - awsSecretKey: options.awsSecretKey, - awsAccessKey: options.awsAccessKey, - endpoint: options.endpoint, - provider: options.provider - }); - } - - async handleUpload( - upload: Upload, - _args: any, - _context: any, - info: GraphQLResolveInfo & { uploadPlugin: UploadPluginInfo } - ): Promise { - const { - uploadPlugin: { tags, type } - } = info; - - const readStream = upload.createReadStream() as ReadStream; - const { filename, mimetype } = upload; - - const rand = - Math.random().toString(36).substring(2, 7) + - Math.random().toString(36).substring(2, 7); - const key = rand + '-' + uploadNames(filename); - - const result = await this.streamer.upload({ - readStream, - filename, - key, - bucket: this.options.bucketName - }); - - const url = result.upload.Location; - const { - contentType, - magic: { charset } - } = result; - - const typ = type || tags.type; - - const mim = tags.mime - ? tags.mime.trim().split(',').map((a: string) => a.trim()) - : typ === 'image' - ? ['image/jpg', 'image/jpeg', 'image/png', 'image/svg+xml'] - : []; - - if (mim.length && !mim.includes(contentType)) { - throw new Error(`UPLOAD_MIMETYPE ${mim.join(',')}`); - } - - switch (typ) { - case 'image': - case 'upload': - return { - filename, - mime: contentType, - url - }; - case 'attachment': - default: - return url; - } - } -} diff --git a/graphql/server-test/__fixtures__/seed/db-scope-storage/test-data.sql b/graphql/server-test/__fixtures__/seed/db-scope-storage/test-data.sql index 4b23efaea0..35de14b3fc 100644 --- a/graphql/server-test/__fixtures__/seed/db-scope-storage/test-data.sql +++ b/graphql/server-test/__fixtures__/seed/db-scope-storage/test-data.sql @@ -59,6 +59,24 @@ VALUES ( 'ce554000-0000-4000-8000-000000000001' ) ON CONFLICT (id) DO NOTHING; +-- The deployment's object store: the platform database's platform plane. +-- Every other plane signs against it with the STORAGE_* credentials and may +-- not name an endpoint/provider/region of its own. +INSERT INTO metaschema_public.database (id, owner_id, name, platform) +VALUES ('f1a7f000-0000-4000-8000-000000000001', NULL, 'platform', true) +ON CONFLICT (id) DO NOTHING; + +INSERT INTO metaschema_modules_public.storage_module ( + id, database_id, schema_id, buckets_table_id, files_table_id, + endpoint, provider, region, scope, private_schema_id +) +VALUES ( + 'f1a7f000-0000-4000-8000-000000000002', + 'f1a7f000-0000-4000-8000-000000000001', + 'ce552000-0000-4000-8000-000000000001', 'ce553000-0000-4000-8000-000000000001', 'ce553000-0000-4000-8000-000000000002', + 'http://localhost:9000', 'minio', 'us-east-1', 'platform', 'ce552000-0000-4000-8000-000000000002' +) ON CONFLICT (id) DO NOTHING; + INSERT INTO metaschema_modules_public.storage_module ( id, database_id, @@ -68,6 +86,7 @@ INSERT INTO metaschema_modules_public.storage_module ( endpoint, public_url_prefix, provider, + region, allowed_origins, scope, private_schema_id, @@ -79,9 +98,10 @@ VALUES ( 'ce552000-0000-4000-8000-000000000001', 'ce553000-0000-4000-8000-000000000001', 'ce553000-0000-4000-8000-000000000002', - NULL, -- use global CDN_ENDPOINT - NULL, -- use global CDN_PUBLIC_URL_PREFIX - 'minio', + NULL, + NULL, + NULL, + NULL, ARRAY['*'], 'database', 'ce552000-0000-4000-8000-000000000002', diff --git a/graphql/server-test/__fixtures__/seed/simple-seed-storage/test-data.sql b/graphql/server-test/__fixtures__/seed/simple-seed-storage/test-data.sql index ec3cce190f..ca1a406a9c 100644 --- a/graphql/server-test/__fixtures__/seed/simple-seed-storage/test-data.sql +++ b/graphql/server-test/__fixtures__/seed/simple-seed-storage/test-data.sql @@ -62,6 +62,24 @@ ON CONFLICT (id) DO NOTHING; -- STORAGE MODULE CONFIG -- ===================================================== +-- The deployment's object store: the platform database's platform plane. +-- Every other plane signs against it with the STORAGE_* credentials and may +-- not name an endpoint/provider/region of its own. +INSERT INTO metaschema_public.database (id, owner_id, name, platform) +VALUES ('f1a7f000-0000-4000-8000-000000000001', NULL, 'platform', true) +ON CONFLICT (id) DO NOTHING; + +INSERT INTO metaschema_modules_public.storage_module ( + id, database_id, schema_id, buckets_table_id, files_table_id, + endpoint, provider, region, scope, private_schema_id +) +VALUES ( + 'f1a7f000-0000-4000-8000-000000000002', + 'f1a7f000-0000-4000-8000-000000000001', + '6dbae92a-5450-401b-1ed5-d69e7754940d', 'b0000001-0000-0000-0000-000000000001', 'b0000001-0000-0000-0000-000000000002', + 'http://localhost:9000', 'minio', 'us-east-1', 'platform', '6dbae92a-5450-401b-1ed5-d69e7754940e' +) ON CONFLICT (id) DO NOTHING; + INSERT INTO metaschema_modules_public.storage_module ( id, database_id, @@ -71,6 +89,7 @@ INSERT INTO metaschema_modules_public.storage_module ( endpoint, public_url_prefix, provider, + region, allowed_origins, scope, private_schema_id, @@ -82,9 +101,10 @@ VALUES ( '6dbae92a-5450-401b-1ed5-d69e7754940d', 'b0000001-0000-0000-0000-000000000001', 'b0000001-0000-0000-0000-000000000002', - NULL, -- use global CDN_ENDPOINT - NULL, -- use global CDN_PUBLIC_URL_PREFIX - 'minio', + NULL, + NULL, + NULL, + NULL, ARRAY['*'], 'app', '6dbae92a-5450-401b-1ed5-d69e7754940e', @@ -173,6 +193,7 @@ INSERT INTO metaschema_modules_public.storage_module ( endpoint, public_url_prefix, provider, + region, allowed_origins, scope, private_schema_id, @@ -186,7 +207,8 @@ VALUES ( 'b1b1b1b1-0000-0000-0000-000000000002', NULL, NULL, - 'minio', + NULL, + NULL, ARRAY['*'], 'app', 'a2a2a2a2-b3b3-4c4c-d5d5-e6e6e6e6e6f0', @@ -304,6 +326,7 @@ INSERT INTO metaschema_modules_public.storage_module ( endpoint, public_url_prefix, provider, + region, allowed_origins, scope, private_schema_id, @@ -317,7 +340,8 @@ VALUES ( 'fa33fa33-0000-0000-0000-000000000002', NULL, NULL, - 'minio', + NULL, + NULL, ARRAY['*'], 'app', 'fa22fa22-a3a3-4b4b-c5c5-d6d6d6d6d6d7', diff --git a/graphql/server/package.json b/graphql/server/package.json index ad7588e5af..276a71bf12 100644 --- a/graphql/server/package.json +++ b/graphql/server/package.json @@ -30,8 +30,7 @@ "debug:heap:capture": "node scripts/capture-heap-snapshot.mjs", "lint": "eslint . --fix", "test": "jest --passWithNoTests", - "test:watch": "jest --watch", - "bucket:create": "ts-node src/scripts/create-bucket.ts" + "test:watch": "jest --watch" }, "keywords": [ "server", diff --git a/graphql/server/src/scripts/create-bucket.ts b/graphql/server/src/scripts/create-bucket.ts deleted file mode 100644 index e08fe36e55..0000000000 --- a/graphql/server/src/scripts/create-bucket.ts +++ /dev/null @@ -1,42 +0,0 @@ -// Minimal script to create a bucket in RustFS/S3 using @constructive-io/s3-utils - -import { getEnvOptions } from '@constructive-io/graphql-env'; -import type { StorageProvider } from '@constructive-io/s3-utils'; -import { createS3Bucket,createS3Client } from '@constructive-io/s3-utils'; -import { Logger } from '@pgpmjs/logger'; - -const log = new Logger('create-bucket'); - -(async () => { - try { - const opts = getEnvOptions(); - const { cdn } = opts; - - const provider = cdn.provider as StorageProvider; - const bucket = cdn.bucketName; - const region = cdn.awsRegion; - const accessKey = cdn.awsAccessKey; - const secretKey = cdn.awsSecretKey; - const endpoint = cdn.endpoint; - - const client = createS3Client({ - provider, - region, - accessKeyId: accessKey, - secretAccessKey: secretKey, - ...(endpoint ? { endpoint } : {}), - }); - - const res = await createS3Bucket(client as any, bucket, { provider }); - if (res.success) { - log.success(`${bucket} (provider: ${provider})`); - } else { - log.error(`Failed to create bucket ${bucket}`); - } - - client.destroy(); - } catch (e) { - log.error('error', e); - process.exitCode = 1; - } -})(); diff --git a/graphql/types/src/constructive.ts b/graphql/types/src/constructive.ts index 485a4f4a59..c3d5c3da71 100644 --- a/graphql/types/src/constructive.ts +++ b/graphql/types/src/constructive.ts @@ -1,11 +1,11 @@ import { - CDNOptions, DeploymentOptions, MigrationOptions, pgpmDefaults, PgpmOptions, PgTestConnectionOptions, - ServerOptions} from '@pgpmjs/types'; + ServerOptions, + StorageCredentialOptions} from '@pgpmjs/types'; import deepmerge from 'deepmerge'; import { PgConfig } from 'pg-env'; @@ -48,8 +48,8 @@ export interface ConstructiveOptions extends PgpmOptions, ConstructiveGraphQLOpt features?: GraphileFeatureOptions; /** API configuration options */ api?: ApiOptions; - /** CDN and file storage configuration */ - cdn?: CDNOptions; + /** Object-store credentials (coordinates live in storage_module) */ + storage?: StorageCredentialOptions; /** Module deployment configuration */ deployment?: DeploymentOptions; /** Migration and code generation options */ diff --git a/packages/bucket-provisioner/README.md b/packages/bucket-provisioner/README.md index 5bf6b0581a..d7751ac5e5 100644 --- a/packages/bucket-provisioner/README.md +++ b/packages/bucket-provisioner/README.md @@ -114,8 +114,8 @@ const provisioner = new BucketProvisioner({ connection: { provider: 's3', region: 'us-west-2', - accessKeyId: process.env.AWS_ACCESS_KEY_ID!, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!, + accessKeyId: process.env.STORAGE_ACCESS_KEY_ID!, + secretAccessKey: process.env.STORAGE_SECRET_ACCESS_KEY!, }, allowedOrigins: ['https://app.example.com'], }); diff --git a/packages/bucket-provisioner/__tests__/provisioner.integration.test.ts b/packages/bucket-provisioner/__tests__/provisioner.integration.test.ts index 7ad78dedb1..4cd710a945 100644 --- a/packages/bucket-provisioner/__tests__/provisioner.integration.test.ts +++ b/packages/bucket-provisioner/__tests__/provisioner.integration.test.ts @@ -21,10 +21,11 @@ import { ProvisionerError } from '../src/types'; // --- RustFS config (matches CI env) --- -const OBJECT_STORE_ENDPOINT = process.env.CDN_ENDPOINT || 'http://localhost:9000'; -const AWS_REGION = process.env.AWS_REGION || 'us-east-1'; -const AWS_ACCESS_KEY = process.env.AWS_ACCESS_KEY || 'constructive'; -const AWS_SECRET_KEY = process.env.AWS_SECRET_KEY || 'constructive-dev-secret'; +// The local object store (docker RustFS/MinIO); credentials from the env. +const OBJECT_STORE_ENDPOINT = 'http://localhost:9000'; +const AWS_REGION = 'us-east-1'; +const AWS_ACCESS_KEY = process.env.STORAGE_ACCESS_KEY_ID!; +const AWS_SECRET_KEY = process.env.STORAGE_SECRET_ACCESS_KEY!; const connection: StorageConnectionConfig = { provider: 'minio', diff --git a/pgpm/cli/src/commands/env.ts b/pgpm/cli/src/commands/env.ts index d765687967..5fd0ea1519 100644 --- a/pgpm/cli/src/commands/env.ts +++ b/pgpm/cli/src/commands/env.ts @@ -14,7 +14,7 @@ Database Profiles: --supabase Use Supabase local development profile Additional Services: - --rustfs Include RustFS/S3 environment variables + --rustfs Include the local RustFS object-store credentials Modes: No command Print export statements for shell evaluation @@ -23,7 +23,8 @@ Modes: Options: --help, -h Show this help message --supabase Use Supabase profile instead of default Postgres - --rustfs Include CDN_ENDPOINT, AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION + --rustfs Include STORAGE_ACCESS_KEY_ID, STORAGE_SECRET_ACCESS_KEY + (endpoint/provider/region are storage_module rows, not env) Examples: pgpm env Print default Postgres env exports @@ -47,17 +48,13 @@ const DEFAULT_PROFILE: PgConfig = { }; interface ObjectStoreConfig { - endpoint: string; - accessKey: string; - secretKey: string; - region: string; + accessKeyId: string; + secretAccessKey: string; } const OBJECT_STORE_PROFILE: ObjectStoreConfig = { - endpoint: 'http://localhost:9000', - accessKey: 'constructive', - secretKey: 'constructive-dev-secret', - region: 'us-east-1', + accessKeyId: 'constructive', + secretAccessKey: 'constructive-dev-secret', }; function configToEnvVars(config: PgConfig, objectStore?: ObjectStoreConfig): Record { @@ -70,10 +67,8 @@ function configToEnvVars(config: PgConfig, objectStore?: ObjectStoreConfig): Rec }; if (objectStore) { - vars.CDN_ENDPOINT = objectStore.endpoint; - vars.AWS_ACCESS_KEY = objectStore.accessKey; - vars.AWS_SECRET_KEY = objectStore.secretKey; - vars.AWS_REGION = objectStore.region; + vars.STORAGE_ACCESS_KEY_ID = objectStore.accessKeyId; + vars.STORAGE_SECRET_ACCESS_KEY = objectStore.secretAccessKey; } return vars; diff --git a/pgpm/env/__tests__/__snapshots__/merge.test.ts.snap b/pgpm/env/__tests__/__snapshots__/merge.test.ts.snap index 3d7f12db7e..e26edbf279 100644 --- a/pgpm/env/__tests__/__snapshots__/merge.test.ts.snap +++ b/pgpm/env/__tests__/__snapshots__/merge.test.ts.snap @@ -2,15 +2,6 @@ exports[`getEnvOptions merges defaults, config, env, and overrides 1`] = ` { - "cdn": { - "awsAccessKey": "constructive", - "awsRegion": "us-east-1", - "awsSecretKey": "constructive-dev-secret", - "bucketName": "test-bucket", - "endpoint": "http://localhost:9000", - "provider": "minio", - "publicUrlPrefix": "http://localhost:9000", - }, "db": { "connections": { "admin": { @@ -74,5 +65,6 @@ exports[`getEnvOptions merges defaults, config, env, and overrides 1`] = ` "port": 587, "secure": false, }, + "storage": {}, } `; diff --git a/pgpm/env/__tests__/assert.test.ts b/pgpm/env/__tests__/assert.test.ts index 82e36719e9..e37d9f0698 100644 --- a/pgpm/env/__tests__/assert.test.ts +++ b/pgpm/env/__tests__/assert.test.ts @@ -23,15 +23,7 @@ const safeOpts = (): PgpmOptions => ({ password: 's3cret-pg', database: 'appdb' }, - server: { host: '0.0.0.0' }, - cdn: { - provider: 'minio', - bucketName: 'prod-bucket', - awsAccessKey: 'AKIAREAL', - awsSecretKey: 'realsecret', - endpoint: 'https://s3.example.com', - publicUrlPrefix: 'https://cdn.example.com' - } + server: { host: '0.0.0.0' } }); describe('findUnsafeProductionDefaults', () => { @@ -39,8 +31,6 @@ describe('findUnsafeProductionDefaults', () => { const issues = findUnsafeProductionDefaults(pgpmDefaults); const joined = issues.join('\n'); expect(joined).toContain('pg.password'); - expect(joined).toContain('cdn.awsAccessKey'); - expect(joined).toContain('cdn.awsSecretKey'); expect(joined).toContain('db.connections.app.password'); expect(joined).toContain('pg.host'); // Never leak the value itself (paths mention ".password", but never the @@ -49,7 +39,6 @@ describe('findUnsafeProductionDefaults', () => { expect(joined).not.toContain('admin_password'); expect(joined).not.toContain('constructive-dev-secret'); expect(joined).not.toContain('localhost'); - expect(joined).not.toContain('test-bucket'); }); it('reports no issues when every sensitive field is overridden', () => { diff --git a/pgpm/env/__tests__/merge.test.ts b/pgpm/env/__tests__/merge.test.ts index 8924d0b4ec..00b99af37d 100644 --- a/pgpm/env/__tests__/merge.test.ts +++ b/pgpm/env/__tests__/merge.test.ts @@ -245,12 +245,7 @@ describe('getEnvOptions', () => { PGROOTDATABASE: 'app_root', SERVER_HOST: '0.0.0.0', DB_CONNECTIONS_APP_PASSWORD: 's3cret-app', - DB_CONNECTIONS_ADMIN_PASSWORD: 's3cret-admin', - AWS_ACCESS_KEY: 'AKIAREAL', - AWS_SECRET_KEY: 'realsecret', - CDN_ENDPOINT: 'https://s3.example.com', - CDN_PUBLIC_URL_PREFIX: 'https://cdn.example.com', - BUCKET_NAME: 'prod-bucket' + DB_CONNECTIONS_ADMIN_PASSWORD: 's3cret-admin' }; expect(() => getEnvOptions({}, emptyCwd(), safeEnv)).not.toThrow(); }); diff --git a/pgpm/env/src/assert.ts b/pgpm/env/src/assert.ts index 82fc36efbe..6f17a4b365 100644 --- a/pgpm/env/src/assert.ts +++ b/pgpm/env/src/assert.ts @@ -5,8 +5,7 @@ import { getStrictEnvMode, isProduction } from '12factor-env'; * Production-safety enforcement for the merged PGPM options. * * `pgpmDefaults` bakes in development-only values so local dev and tests work - * out of the box (e.g. `pg.password = 'password'`, `cdn.awsAccessKey = - * 'constructive'`, `pg.host = 'localhost'`). Those are a liability in production: + * out of the box (e.g. `pg.password = 'password'`, `pg.host = 'localhost'`). Those are a liability in production: * a deploy that forgets to set the real value boots on the dev default instead * of failing. `deepmerge` cannot express "dev default, required in prod", so * this is enforced here as an opt-in assertion callers run at startup. @@ -34,15 +33,10 @@ const SENSITIVE_FIELDS: SensitiveField[] = [ { path: 'pg.password', severity: 'secret', envHint: 'PGPASSWORD' }, { path: 'db.connections.app.password', severity: 'secret', envHint: 'DB_CONNECTIONS_APP_PASSWORD' }, { path: 'db.connections.admin.password', severity: 'secret', envHint: 'DB_CONNECTIONS_ADMIN_PASSWORD' }, - { path: 'cdn.awsAccessKey', severity: 'secret', envHint: 'AWS_ACCESS_KEY' }, - { path: 'cdn.awsSecretKey', severity: 'secret', envHint: 'AWS_SECRET_KEY' }, { path: 'pg.host', severity: 'host', envHint: 'PGHOST' }, { path: 'pg.database', severity: 'host', envHint: 'PGDATABASE' }, { path: 'db.rootDb', severity: 'host', envHint: 'PGROOTDATABASE' }, - { path: 'server.host', severity: 'host', envHint: 'SERVER_HOST' }, - { path: 'cdn.endpoint', severity: 'host', envHint: 'CDN_ENDPOINT' }, - { path: 'cdn.publicUrlPrefix', severity: 'host', envHint: 'CDN_PUBLIC_URL_PREFIX' }, - { path: 'cdn.bucketName', severity: 'host', envHint: 'BUCKET_NAME' } + { path: 'server.host', severity: 'host', envHint: 'SERVER_HOST' } ]; const getPath = (obj: unknown, path: string): unknown => diff --git a/pgpm/env/src/env.ts b/pgpm/env/src/env.ts index 5ec9b056a9..668080a439 100644 --- a/pgpm/env/src/env.ts +++ b/pgpm/env/src/env.ts @@ -1,4 +1,4 @@ -import { BucketProvider, DeferredConstraintsMode, PgpmOptions } from '@pgpmjs/types'; +import { DeferredConstraintsMode, PgpmOptions } from '@pgpmjs/types'; import { parseEnvBoolean, parseEnvList, parseEnvNumber } from '12factor-env'; export { parseEnvBoolean, parseEnvList, parseEnvNumber }; @@ -52,15 +52,8 @@ export const getEnvVars = (env: NodeJS.ProcessEnv = process.env): PgpmOptions => PGPASSWORD, PGDATABASE, - BUCKET_PROVIDER, - BUCKET_NAME, - AWS_REGION, - AWS_ACCESS_KEY, - AWS_ACCESS_KEY_ID, - AWS_SECRET_KEY, - AWS_SECRET_ACCESS_KEY, - CDN_ENDPOINT, - CDN_PUBLIC_URL_PREFIX, + STORAGE_ACCESS_KEY_ID, + STORAGE_SECRET_ACCESS_KEY, DEPLOYMENT_USE_TX, DEPLOYMENT_FAST, @@ -142,14 +135,9 @@ export const getEnvVars = (env: NodeJS.ProcessEnv = process.env): PgpmOptions => ...(PGPASSWORD && { password: PGPASSWORD }), ...(PGDATABASE && { database: PGDATABASE }), }, - cdn: { - ...(BUCKET_PROVIDER && { provider: BUCKET_PROVIDER as BucketProvider }), - ...(BUCKET_NAME && { bucketName: BUCKET_NAME }), - ...(AWS_REGION && { awsRegion: AWS_REGION }), - ...((AWS_ACCESS_KEY || AWS_ACCESS_KEY_ID) && { awsAccessKey: AWS_ACCESS_KEY || AWS_ACCESS_KEY_ID }), - ...((AWS_SECRET_KEY || AWS_SECRET_ACCESS_KEY) && { awsSecretKey: AWS_SECRET_KEY || AWS_SECRET_ACCESS_KEY }), - ...(CDN_ENDPOINT && { endpoint: CDN_ENDPOINT }), - ...(CDN_PUBLIC_URL_PREFIX && { publicUrlPrefix: CDN_PUBLIC_URL_PREFIX }), + storage: { + ...(STORAGE_ACCESS_KEY_ID && { accessKeyId: STORAGE_ACCESS_KEY_ID }), + ...(STORAGE_SECRET_ACCESS_KEY && { secretAccessKey: STORAGE_SECRET_ACCESS_KEY }), }, deployment: { ...(DEPLOYMENT_USE_TX && { useTx: parseEnvBoolean(DEPLOYMENT_USE_TX) }), diff --git a/pgpm/types/src/pgpm.ts b/pgpm/types/src/pgpm.ts index 194c9e6e09..3a2fcc0a23 100644 --- a/pgpm/types/src/pgpm.ts +++ b/pgpm/types/src/pgpm.ts @@ -123,28 +123,18 @@ export interface ServerOptions { } /** - * Storage provider type for CDN/bucket operations + * Storage provider type for bucket operations */ export type BucketProvider = 's3' | 'minio' | 'rustfs' | 'gcs'; /** - * CDN and file storage configuration + * Object-store credentials — the only storage input read from the environment + * (`STORAGE_ACCESS_KEY_ID`, `STORAGE_SECRET_ACCESS_KEY`). Endpoint, provider, + * region and public URL prefix are `storage_module` rows, never config. */ -export interface CDNOptions { - /** Storage provider type (s3, minio, rustfs, gcs). 'minio' means path-style S3-compatible (RustFS, MinIO) */ - provider?: BucketProvider; - /** S3 bucket name for file storage */ - bucketName?: string; - /** AWS region for S3 bucket */ - awsRegion?: string; - /** AWS access key for S3 */ - awsAccessKey?: string; - /** AWS secret key for S3 */ - awsSecretKey?: string; - /** S3-compatible API endpoint URL (RustFS, MinIO, R2, DO Spaces, GCS, etc.) */ - endpoint?: string; - /** Public URL prefix for generating download URLs (e.g., CDN domain, S3 public URL) */ - publicUrlPrefix?: string; +export interface StorageCredentialOptions { + accessKeyId?: string; + secretAccessKey?: string; } /** @@ -305,8 +295,8 @@ export interface PgpmOptions { pg?: Partial; /** HTTP server configuration */ server?: ServerOptions; - /** CDN and file storage configuration */ - cdn?: CDNOptions; + /** Object-store credentials (coordinates live in storage_module) */ + storage?: StorageCredentialOptions; /** Module deployment configuration */ deployment?: DeploymentOptions; /** Migration and code generation options */ @@ -385,15 +375,6 @@ export const pgpmDefaults: PgpmOptions = { trustProxy: false, strictAuth: false }, - cdn: { - provider: 'minio', - bucketName: 'test-bucket', - awsRegion: 'us-east-1', - awsAccessKey: 'constructive', - awsSecretKey: 'constructive-dev-secret', - endpoint: 'http://localhost:9000', - publicUrlPrefix: 'http://localhost:9000' - }, deployment: { useTx: true, fast: false, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 504efdd1e5..a08f1007b6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1076,6 +1076,9 @@ importers: '@aws-sdk/s3-request-presigner': specifier: ^3.1052.0 version: 3.1052.0 + '@constructive-io/s3-utils': + specifier: workspace:^ + version: link:../../uploads/s3-utils/dist '@pgpmjs/logger': specifier: workspace:^ version: link:../../pgpm/logger/dist @@ -1116,9 +1119,6 @@ importers: specifier: 5.1.4 version: 5.1.4(103c5e67e025ec5d024bce65778e0145) devDependencies: - '@constructive-io/s3-utils': - specifier: workspace:^ - version: link:../../uploads/s3-utils/dist '@types/node': specifier: ^22.19.11 version: 22.19.15 @@ -1402,9 +1402,6 @@ importers: '@constructive-io/s3-streamer': specifier: workspace:^ version: link:../../uploads/s3-streamer/dist - '@constructive-io/s3-utils': - specifier: workspace:^ - version: link:../../uploads/s3-utils/dist '@constructive-io/upload-names': specifier: workspace:^ version: link:../../uploads/upload-names/dist @@ -1523,6 +1520,9 @@ importers: specifier: 0.1.1 version: 0.1.1 devDependencies: + '@constructive-io/s3-utils': + specifier: workspace:^ + version: link:../../uploads/s3-utils/dist '@types/cors': specifier: ^2.8.17 version: 2.8.19 @@ -4140,9 +4140,6 @@ importers: '@constructive-io/s3-utils': specifier: workspace:^ version: link:../s3-utils/dist - '@pgpmjs/types': - specifier: workspace:^ - version: link:../../pgpm/types/dist devDependencies: '@pgpmjs/env': specifier: workspace:^ diff --git a/uploads/s3-streamer/README.md b/uploads/s3-streamer/README.md index 5101a65cc2..0c30ef8e43 100644 --- a/uploads/s3-streamer/README.md +++ b/uploads/s3-streamer/README.md @@ -35,7 +35,7 @@ Stream uploads to S3 ```js import Streamer from '@constructive-io/s3-streamer'; -const streamer = new Streamer(opts) +const streamer = new Streamer({ client, defaultBucket }) const readStream = createReadStream(filename); const results = await streamer.upload({ readStream, @@ -91,8 +91,9 @@ The upload methods return a detailed payload with upload results and file metada If you don't want to use the `Streamer` class you can use the utils directly: ```js -import { getClient, upload } from '@constructive-io/s3-streamer'; -const client = getClient(opts) +import { createS3Client } from '@constructive-io/s3-utils'; +import { upload } from '@constructive-io/s3-streamer'; +const client = createS3Client(opts) const readStream = createReadStream(filename); const results = await upload({ client, @@ -110,9 +111,12 @@ const results = await upload({ ```js const streamer = new Streamer({ defaultBucket: 'my-bucket', - awsRegion: 'us-east-1', - awsSecretKey: process.env.AWS_SECRET_ACCESS_KEY, - awsAccessKey: process.env.AWS_ACCESS_KEY_ID + client: createS3Client({ + provider: 's3', + region: 'us-east-1', + accessKeyId: process.env.STORAGE_ACCESS_KEY_ID, + secretAccessKey: process.env.STORAGE_SECRET_ACCESS_KEY + }) }); ``` @@ -121,10 +125,13 @@ const streamer = new Streamer({ ```js const streamer = new Streamer({ defaultBucket: 'my-bucket', - awsRegion: 'us-east-1', - awsSecretKey: 'minio-secret', - awsAccessKey: 'minio-access', - minioEndpoint: 'http://localhost:9000' + client: createS3Client({ + provider: 'minio', + region: 'us-east-1', + accessKeyId: process.env.STORAGE_ACCESS_KEY_ID, + secretAccessKey: process.env.STORAGE_SECRET_ACCESS_KEY, + endpoint: 'http://localhost:9000' + }) }); ``` @@ -135,12 +142,9 @@ const streamer = new Streamer({ #### Constructor Options ```typescript -interface StreamerOptions { - awsRegion: string; // AWS region (e.g., 'us-east-1') - awsSecretKey: string; // AWS secret access key - awsAccessKey: string; // AWS access key ID - minioEndpoint?: string; // Optional: MinIO/S3-compatible endpoint - defaultBucket: string; // Default bucket for uploads +{ + client: S3Client; // S3 client for the resolved object store (see createS3Client in @constructive-io/s3-utils) + defaultBucket?: string; // Bucket used when a call names none } ``` @@ -172,14 +176,16 @@ streamer.destroy(); If you prefer functional programming over classes: ```js -import { getClient, upload } from '@constructive-io/s3-streamer'; +import { createS3Client } from '@constructive-io/s3-utils'; +import { upload } from '@constructive-io/s3-streamer'; // Create S3 client -const client = getClient({ - awsRegion: 'us-east-1', - awsSecretKey: process.env.AWS_SECRET_ACCESS_KEY, - awsAccessKey: process.env.AWS_ACCESS_KEY_ID, - minioEndpoint: 'http://localhost:9000' // optional +const client = createS3Client({ + provider: 'minio', + region: 'us-east-1', + accessKeyId: process.env.STORAGE_ACCESS_KEY_ID, + secretAccessKey: process.env.STORAGE_SECRET_ACCESS_KEY, + endpoint: 'http://localhost:9000' // omit for AWS S3 }); // Upload file diff --git a/uploads/s3-streamer/__tests__/uploads.test.ts b/uploads/s3-streamer/__tests__/uploads.test.ts index 7e8fd84d36..1d02266319 100644 --- a/uploads/s3-streamer/__tests__/uploads.test.ts +++ b/uploads/s3-streamer/__tests__/uploads.test.ts @@ -1,35 +1,26 @@ import { S3Client } from '@aws-sdk/client-s3'; -import { createS3Bucket } from '@constructive-io/s3-utils'; -import { getEnvOptions } from '@pgpmjs/env'; +import { createS3Bucket, createS3Client } from '@constructive-io/s3-utils'; import { createReadStream } from 'fs'; import { sync as glob } from 'glob'; import { basename } from 'path'; -import { getClient, Streamer, upload } from '../src'; +import { Streamer, upload } from '../src'; import type { AsyncUploadResult } from '../src/utils'; -// Use Constructive defaults with optional overrides -const config = getEnvOptions({ - cdn: { - bucketName: 'test-bucket' - } -}); - -const { - bucketName: BUCKET_NAME, - awsRegion: AWS_REGION, - awsSecretKey: AWS_SECRET_KEY, - awsAccessKey: AWS_ACCESS_KEY, - endpoint: ENDPOINT -} = config.cdn; +// The local object store (docker RustFS/MinIO); credentials from the env. +const BUCKET_NAME = 'test-bucket'; +const REGION = 'us-east-1'; +const ENDPOINT = 'http://localhost:9000'; +const ACCESS_KEY_ID = process.env.STORAGE_ACCESS_KEY_ID!; +const SECRET_ACCESS_KEY = process.env.STORAGE_SECRET_ACCESS_KEY!; // Initialize S3 client const s3Client = new S3Client({ credentials: { - accessKeyId: AWS_ACCESS_KEY, - secretAccessKey: AWS_SECRET_KEY, + accessKeyId: ACCESS_KEY_ID, + secretAccessKey: SECRET_ACCESS_KEY, }, - region: AWS_REGION, + region: REGION, endpoint: ENDPOINT, forcePathStyle: true }); @@ -62,10 +53,13 @@ describe('uploads', () => { it('upload files via class', async () => { const streamer = new Streamer({ defaultBucket: BUCKET_NAME, - awsRegion: AWS_REGION, - awsSecretKey: AWS_SECRET_KEY, - awsAccessKey: AWS_ACCESS_KEY, - endpoint: ENDPOINT + client: createS3Client({ + provider: 'minio', + region: REGION, + accessKeyId: ACCESS_KEY_ID, + secretAccessKey: SECRET_ACCESS_KEY, + endpoint: ENDPOINT + }) }); try { @@ -94,10 +88,11 @@ describe('uploads', () => { }); it('upload files via functions', async () => { - const client = getClient({ - awsRegion: AWS_REGION, - awsSecretKey: AWS_SECRET_KEY, - awsAccessKey: AWS_ACCESS_KEY, + const client = createS3Client({ + provider: 'minio', + region: REGION, + accessKeyId: ACCESS_KEY_ID, + secretAccessKey: SECRET_ACCESS_KEY, endpoint: ENDPOINT }); diff --git a/uploads/s3-streamer/package.json b/uploads/s3-streamer/package.json index d7d3e487cf..ecd5d59bd5 100644 --- a/uploads/s3-streamer/package.json +++ b/uploads/s3-streamer/package.json @@ -37,8 +37,7 @@ "@aws-sdk/client-s3": "^3.1052.0", "@aws-sdk/lib-storage": "^3.1052.0", "@constructive-io/content-type-stream": "workspace:^", - "@constructive-io/s3-utils": "workspace:^", - "@pgpmjs/types": "workspace:^" + "@constructive-io/s3-utils": "workspace:^" }, "keywords": [ "s3", diff --git a/uploads/s3-streamer/src/index.ts b/uploads/s3-streamer/src/index.ts index 74c9f3389d..3737d9e9cc 100644 --- a/uploads/s3-streamer/src/index.ts +++ b/uploads/s3-streamer/src/index.ts @@ -1,8 +1,6 @@ -import getClient from './s3'; import Streamer from './streamer'; export * from './utils'; -export { getClient }; export { Streamer }; -export default Streamer; \ No newline at end of file +export default Streamer; diff --git a/uploads/s3-streamer/src/s3.ts b/uploads/s3-streamer/src/s3.ts deleted file mode 100644 index 7ec443d77e..0000000000 --- a/uploads/s3-streamer/src/s3.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { S3Client } from '@aws-sdk/client-s3'; -import type { StorageProvider } from '@constructive-io/s3-utils'; -import { createS3Client } from '@constructive-io/s3-utils'; -import type { BucketProvider } from '@pgpmjs/types'; - -interface S3Options { - awsAccessKey: string; - awsSecretKey: string; - awsRegion: string; - endpoint?: string; - provider?: BucketProvider; -} - -export default function getS3(opts: S3Options): S3Client { - return createS3Client({ - provider: (opts.provider as StorageProvider) || 'minio', - region: opts.awsRegion, - accessKeyId: opts.awsAccessKey, - secretAccessKey: opts.awsSecretKey, - ...(opts.endpoint ? { endpoint: opts.endpoint } : {}), - }); -} diff --git a/uploads/s3-streamer/src/streamer.ts b/uploads/s3-streamer/src/streamer.ts index 6597b961e9..dbf77f2279 100644 --- a/uploads/s3-streamer/src/streamer.ts +++ b/uploads/s3-streamer/src/streamer.ts @@ -1,9 +1,7 @@ import { S3Client } from '@aws-sdk/client-s3'; import { streamContentType } from '@constructive-io/content-type-stream'; -import type { BucketProvider } from '@pgpmjs/types'; import type { Readable } from 'stream'; -import getS3 from './s3'; import { type AsyncUploadResult, upload as streamUpload, @@ -11,11 +9,8 @@ import { } from './utils'; interface StreamerOptions { - awsRegion: string; - awsSecretKey: string; - awsAccessKey: string; - endpoint?: string; - provider?: BucketProvider; + /** Client for the object store the caller resolved (see `createS3Client` in `@constructive-io/s3-utils`). */ + client: S3Client; /** * Bucket used when a call does not name one. Optional: a caller that resolves * the bucket per upload (tenant-resolved storage) has no deployment-wide @@ -44,21 +39,8 @@ export class Streamer { private s3: S3Client; private defaultBucket?: string; - constructor({ - awsRegion, - awsSecretKey, - awsAccessKey, - endpoint, - provider, - defaultBucket - }: StreamerOptions) { - this.s3 = getS3({ - awsRegion, - awsSecretKey, - awsAccessKey, - endpoint, - provider - }); + constructor({ client, defaultBucket }: StreamerOptions) { + this.s3 = client; this.defaultBucket = defaultBucket; }