diff --git a/src/Database/Adapter/Limits.php b/src/Database/Adapter/Limits.php index 9300f4fef..9707b4b3d 100644 --- a/src/Database/Adapter/Limits.php +++ b/src/Database/Adapter/Limits.php @@ -3,6 +3,7 @@ namespace Utopia\Database\Adapter; use DateTime; +use Utopia\Database\Database; use Utopia\Query\Schema\ColumnType; /** @@ -34,6 +35,32 @@ public function __construct( public ColumnType $idType, public array $keywords, public array $internalIndexKeys, + public bool $escalatedStringByteCap = true, ) { } + + /** + * Byte capacity for a {@see ColumnType::String} attribute stored outside VARCHAR, matching {@see SQL::getSqlType()}. + * Null when the declared size fits in VARCHAR or when escalated TEXT types have no byte cap (PostgreSQL). + */ + public function stringEscalatedStorageByteLimit(int $size): ?int + { + if ($size <= $this->varchar) { + return null; + } + + if (! $this->escalatedStringByteCap) { + return null; + } + + if ($size > 16777215) { + return Database::MAX_LONGTEXT_BYTES; + } + + if ($size > 65535) { + return Database::MAX_MEDIUMTEXT_BYTES; + } + + return Database::MAX_TEXT_BYTES; + } } diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 1fe80e939..c60e4c46b 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1774,6 +1774,12 @@ public function schema(): PostgresSchema return new PostgresSchema(); } + #[\Override] + protected function limitsEscalatedStringByteCap(): bool + { + return false; + } + #[\Override] protected function getSqlType(ColumnType $type, int $size, bool $signed = true, bool $array = false, bool $required = false): string { diff --git a/src/Database/Adapter/Redis.php b/src/Database/Adapter/Redis.php index 52211398a..92e46720a 100644 --- a/src/Database/Adapter/Redis.php +++ b/src/Database/Adapter/Redis.php @@ -1673,6 +1673,7 @@ public function limits(): Limits idType: ColumnType::Integer, keywords: [], internalIndexKeys: [], + escalatedStringByteCap: false, ); } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 58fbf4d63..ec795b30f 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2610,9 +2610,15 @@ public function limits(): Limits idType: ColumnType::Integer, keywords: $this->getKeywords(), internalIndexKeys: [Storage::INDEX_PRIMARY, Storage::INDEX_CREATED_AT, Storage::INDEX_UPDATED_AT, Storage::INDEX_TENANT_ID], + escalatedStringByteCap: $this->limitsEscalatedStringByteCap(), ); } + protected function limitsEscalatedStringByteCap(): bool + { + return true; + } + #[\Override] public function getCountOfAttributes(Document $collection): int { diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index fd6d7883b..a887e5b10 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -136,6 +136,12 @@ public function __construct(object $pdo) $this->registerUserFunctions(); } + #[\Override] + protected function limitsEscalatedStringByteCap(): bool + { + return $this->emulateMySQL; + } + #[Override] public function reconnect(): void { diff --git a/src/Database/Validator/Structure.php b/src/Database/Validator/Structure.php index c249ce8c6..b68dd73e9 100644 --- a/src/Database/Validator/Structure.php +++ b/src/Database/Validator/Structure.php @@ -325,8 +325,17 @@ protected function checkForInvalidAttributeValues(Document $document, array $str break; case ColumnType::Varchar: + $validators[] = new Text($size, min: 0); + break; + case ColumnType::String: $validators[] = new Text($size, min: 0); + if (! $attribute->array) { + $byteCap = $this->profile->limits->stringEscalatedStorageByteLimit($size); + if ($byteCap !== null) { + $validators[] = new ByteLength($byteCap); + } + } break; case ColumnType::Integer: diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 6aba6bf89..825915476 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -1384,6 +1384,98 @@ public function testTextByteTruncationUpdate(): void } } + public function testStringByteTruncationCreate(): void + { + /** @var Database $database */ + $database = $this->getDatabase(); + + if (!$database->getAdapter()->supports(Capability::DefinedAttributes)) { + $this->expectNotToPerformAssertions(); + return; + } + + $attributeSize = 20000; + $byteCap = $database->profile()->limits->stringEscalatedStorageByteLimit($attributeSize); + + $database->createCollection(Collection::create(id: __FUNCTION__)); + $database->createAttribute(__FUNCTION__, Attribute::string(key: 'cursorJson', size: $attributeSize)); + + $value = \str_repeat('📝', 20000); + $this->assertGreaterThan(65535, \strlen($value)); + + $document = new Document([ + '$id' => 'first', + 'cursorJson' => $value, + '$permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + if ($byteCap !== null) { + try { + $database->createDocument(__FUNCTION__, $document); + $this->fail('Expected StructureException for over-capacity string value'); + } catch (StructureException $e) { + $this->assertStringContainsString((string) $byteCap.' bytes', $e->getMessage()); + } + + return; + } + + $created = $database->createDocument(__FUNCTION__, $document); + $this->assertSame($value, $created->getAttribute('cursorJson')); + } + + public function testStringByteTruncationUpdate(): void + { + /** @var Database $database */ + $database = $this->getDatabase(); + + if (!$database->getAdapter()->supports(Capability::DefinedAttributes)) { + $this->expectNotToPerformAssertions(); + return; + } + + $attributeSize = 20000; + $byteCap = $database->profile()->limits->stringEscalatedStorageByteLimit($attributeSize); + + $database->createCollection(Collection::create(id: __FUNCTION__)); + $database->createAttribute(__FUNCTION__, Attribute::string(key: 'cursorJson', size: $attributeSize)); + + $document = new Document([ + '$id' => 'first', + 'cursorJson' => \str_repeat('a', 16383), + '$permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $created = $database->createDocument(__FUNCTION__, $document); + + $value = \str_repeat('📝', 20000); + $this->assertGreaterThan(65535, \strlen($value)); + + if ($byteCap !== null) { + try { + $database->updateDocument(__FUNCTION__, $created->getId(), $created->setAttribute('cursorJson', $value)); + $this->fail('Expected StructureException for over-capacity string value on update'); + } catch (StructureException $e) { + $this->assertStringContainsString((string) $byteCap.' bytes', $e->getMessage()); + } + + return; + } + + $updated = $database->updateDocument(__FUNCTION__, $created->getId(), $created->setAttribute('cursorJson', $value)); + $this->assertSame($value, $updated->getAttribute('cursorJson')); + } + public function testUpsertDocumentsInc(): void { /** @var Database $database */ diff --git a/tests/unit/Adapter/LimitsStringEscalationTest.php b/tests/unit/Adapter/LimitsStringEscalationTest.php new file mode 100644 index 000000000..94e74a1ec --- /dev/null +++ b/tests/unit/Adapter/LimitsStringEscalationTest.php @@ -0,0 +1,66 @@ +assertNull($limits->stringEscalatedStorageByteLimit(100)); + $this->assertSame(Database::MAX_TEXT_BYTES, $limits->stringEscalatedStorageByteLimit(16382)); + $this->assertSame(Database::MAX_TEXT_BYTES, $limits->stringEscalatedStorageByteLimit(65535)); + $this->assertSame(Database::MAX_MEDIUMTEXT_BYTES, $limits->stringEscalatedStorageByteLimit(65536)); + $this->assertSame(Database::MAX_TEXT_BYTES, $limits->stringEscalatedStorageByteLimit(20000)); + $this->assertSame(Database::MAX_MEDIUMTEXT_BYTES, $limits->stringEscalatedStorageByteLimit(100000)); + $this->assertSame(Database::MAX_LONGTEXT_BYTES, $limits->stringEscalatedStorageByteLimit(20000000)); + } + + public function testStringEscalatedStorageByteLimitDisabled(): void + { + $limits = new Limits( + string: 4294967295, + varchar: 16381, + integer: 0, + bigInteger: 0, + attributes: 0, + indexes: 0, + defaultAttributes: 0, + defaultIndexes: 0, + indexLength: 0, + uidLength: 36, + documentSize: 0, + minDateTime: new \DateTime('0000-01-01'), + maxDateTime: new \DateTime('9999-12-31'), + idType: ColumnType::Integer, + keywords: [], + internalIndexKeys: [], + escalatedStringByteCap: false, + ); + + $this->assertNull($limits->stringEscalatedStorageByteLimit(20000)); + } +} diff --git a/tests/unit/Support/Profiles.php b/tests/unit/Support/Profiles.php index 1f32cd41a..8c9fa6cbf 100644 --- a/tests/unit/Support/Profiles.php +++ b/tests/unit/Support/Profiles.php @@ -35,6 +35,7 @@ public static function of( ?DateTime $minDateTime = null, ?DateTime $maxDateTime = null, array $internalIndexKeys = [], + bool $escalatedStringByteCap = true, ): Profile { return new Profile( new Limits( @@ -54,6 +55,7 @@ public static function of( idType: $idType, keywords: [], internalIndexKeys: $internalIndexKeys, + escalatedStringByteCap: $escalatedStringByteCap, ), $capabilities, $features, diff --git a/tests/unit/Validator/StructureTest.php b/tests/unit/Validator/StructureTest.php index ec065e039..1ff7fb4c3 100644 --- a/tests/unit/Validator/StructureTest.php +++ b/tests/unit/Validator/StructureTest.php @@ -1369,4 +1369,131 @@ public function testTextValidationUsesColumnCapacityNotDeclaredSize(): void $this->assertFalse($validator->isValid(new Document($base + ['text' => \str_repeat('📝', 20000)])), '20,000 emoji are 80,000 bytes and must be rejected by byte length'); $this->assertSame('Invalid document structure: Attribute "text" has invalid type. Value must be a valid string no longer than 65535 bytes', $validator->getDescription()); } + + /** + * A string attribute past the varchar limit is stored as TEXT on MySQL/MariaDB, which is byte-capacity limited. + */ + public function testStringEscalatedStorageByteValidation(): void + { + $collection = new Document([ + '$id' => Id::custom('posts'), + '$collection' => Database::METADATA, + 'name' => 'posts', + 'attributes' => [ + [ + '$id' => 'cursorJson', + 'type' => ColumnType::String->value, + 'format' => '', + 'size' => 20000, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [], + ]); + + $validator = new Structure( + $collection, + Profiles::of( + capabilities: [Capability::DefinedAttributes, Capability::UnsignedBigInt], + string: 4294967295, + varchar: 16381, + ), + ); + + $base = [ + '$collection' => Id::custom('posts'), + '$createdAt' => '2000-04-01T12:00:00.000+00:00', + '$updatedAt' => '2000-04-01T12:00:00.000+00:00', + ]; + + $this->assertTrue($validator->isValid(new Document($base + ['cursorJson' => \str_repeat('a', 20000)]))); + + $this->assertFalse($validator->isValid(new Document($base + ['cursorJson' => \str_repeat('📝', 20000)]))); + $this->assertSame( + 'Invalid document structure: Attribute "cursorJson" has invalid type. Value must be a valid string no longer than 65535 bytes', + $validator->getDescription() + ); + } + + public function testStringEscalatedStorageByteValidationDisabledForPostgres(): void + { + $collection = new Document([ + '$id' => Id::custom('posts'), + '$collection' => Database::METADATA, + 'name' => 'posts', + 'attributes' => [ + [ + '$id' => 'cursorJson', + 'type' => ColumnType::String->value, + 'format' => '', + 'size' => 20000, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [], + ]); + + $validator = new Structure( + $collection, + Profiles::of( + capabilities: [Capability::DefinedAttributes, Capability::UnsignedBigInt], + string: 4294967295, + varchar: 16381, + escalatedStringByteCap: false, + ), + ); + + $base = [ + '$collection' => Id::custom('posts'), + '$createdAt' => '2000-04-01T12:00:00.000+00:00', + '$updatedAt' => '2000-04-01T12:00:00.000+00:00', + ]; + + $this->assertTrue($validator->isValid(new Document($base + ['cursorJson' => \str_repeat('📝', 20000)]))); + } + + public function testStringEscalatedStorageByteValidationSkipsArrayElements(): void + { + $collection = new Document([ + '$id' => Id::custom('posts'), + '$collection' => Database::METADATA, + 'name' => 'posts', + 'attributes' => [ + [ + '$id' => 'tags', + 'type' => ColumnType::String->value, + 'format' => '', + 'size' => 20000, + 'required' => false, + 'signed' => true, + 'array' => true, + 'filters' => [], + ], + ], + 'indexes' => [], + ]); + + $validator = new Structure( + $collection, + Profiles::of( + capabilities: [Capability::DefinedAttributes, Capability::UnsignedBigInt], + string: 4294967295, + varchar: 16381, + ), + ); + + $base = [ + '$collection' => Id::custom('posts'), + '$createdAt' => '2000-04-01T12:00:00.000+00:00', + '$updatedAt' => '2000-04-01T12:00:00.000+00:00', + ]; + + $this->assertTrue($validator->isValid(new Document($base + ['tags' => [\str_repeat('📝', 20000)]]))); + } }