diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index f4a5d2b..b0946c6 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -73,12 +73,22 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJSON(needs.report-tests.outputs.matrix) }} + services: + postgres: + image: ${{ matrix.language == 'csharp' && matrix.category == 'objects' && 'postgres:18.3@sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db' || '' }} + env: + POSTGRES_USER: benchmark + POSTGRES_PASSWORD: benchmark + POSTGRES_DB: benchmark + ports: ['5432:5432'] + options: --health-cmd "pg_isready -U benchmark" --health-interval 5s --health-timeout 5s --health-retries 12 env: LANGUAGE: ${{ matrix.language }} CATEGORY: ${{ matrix.category }} BITS: ${{ matrix.bits }} SIZE: ${{ matrix.size }} RUST_BACKTRACE: ${{ inputs.verbose && 'full' || '0' }} + POSTGRESQL_CONNECTION_STRING: ${{ matrix.language == 'csharp' && matrix.category == 'objects' && 'Host=127.0.0.1;Database=benchmark;Username=benchmark;Password=benchmark' || '' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index 21f2f2c..6d77cd2 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -4,9 +4,9 @@ on: workflow_dispatch: push: branches: [main] - paths: ['csharp/**', 'experiments/**/*.cs', 'experiments/**/*.csproj', 'experiments/system_data_sqlite/**', '.github/workflows/csharp.yml', '.github/actions/**', 'scripts/check_pipeline_status.py', 'scripts/ci_process.py', 'scripts/workflow_yaml.py'] + paths: ['csharp/**', 'experiments/**/*.cs', 'experiments/**/*.csproj', 'experiments/system_data_sqlite/**', 'experiments/postgresql/**', '.github/workflows/csharp.yml', '.github/actions/**', 'scripts/check_pipeline_status.py', 'scripts/ci_process.py', 'scripts/workflow_yaml.py'] pull_request: - paths: ['csharp/**', 'experiments/**/*.cs', 'experiments/**/*.csproj', 'experiments/system_data_sqlite/**', '.github/workflows/csharp.yml', '.github/actions/**', 'scripts/check_pipeline_status.py', 'scripts/ci_process.py', 'scripts/workflow_yaml.py'] + paths: ['csharp/**', 'experiments/**/*.cs', 'experiments/**/*.csproj', 'experiments/system_data_sqlite/**', 'experiments/postgresql/**', '.github/workflows/csharp.yml', '.github/actions/**', 'scripts/check_pipeline_status.py', 'scripts/ci_process.py', 'scripts/workflow_yaml.py'] permissions: contents: read @@ -50,9 +50,40 @@ jobs: - name: Check SQLite provider reports run: python ../experiments/system_data_sqlite/check_reports.py + postgresql: + name: PostgreSQL integration + runs-on: ubuntu-24.04 + timeout-minutes: 15 + services: + postgres: + image: postgres:18.3@sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db + env: + POSTGRES_USER: benchmark + POSTGRES_PASSWORD: benchmark + POSTGRES_DB: benchmark + ports: ['5432:5432'] + options: --health-cmd "pg_isready -U benchmark" --health-interval 5s --health-timeout 5s --health-retries 12 + env: + POSTGRESQL_CONNECTION_STRING: Host=127.0.0.1;Database=benchmark;Username=benchmark;Password=benchmark + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + persist-credentials: false + - uses: ./.github/actions/prepare-ci # zizmor: ignore[self-repository] actionlint 1.7.12 lacks $/ syntax + - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6 + with: + global-json-file: csharp/global.json + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: '3.13' + - run: dotnet build -c Release -warnaserror + - run: dotnet test --project SQLiteVSDoublets.Tests -c Release --no-build -- --filter-class '*PostgreSQLTests' --timeout 5m + - run: python ../experiments/postgresql/check_reports.py + pipeline-status: name: Pipeline Status - needs: [test] + needs: [test, postgresql] if: ${{ !cancelled() }} runs-on: ubuntu-24.04 timeout-minutes: 5 diff --git a/README.md b/README.md index 288ccaa..e28ae66 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,28 @@ rows compare with Microsoft.Data.Sqlite of the same durability. Published tables from older runs include only the providers measured in those runs; the next Benchmarks run adds the new rows. +C# object benchmarks also support `PostgreSQL_EFCore` using +[Npgsql.EntityFrameworkCore.PostgreSQL](https://www.npgsql.org/efcore/). +Set `POSTGRESQL_CONNECTION_STRING` to include it in the default object comparison; +embedded benchmarks work without a PostgreSQL server. Explicitly selecting it +without configuration fails with a setup message. Both id widths use the same +deterministic posts, scattered point reads, validated operations, warm-ups and +transaction boundaries as the other object stores. EF saves one post at a time, +clears tracking after each save, reads without tracking and deletes by id. +Each warm-up and repetition creates and drops its own generated schema; the +database and other schemas remain intact. The configured user needs permission +to connect and create schemas in a benchmark database. + +PostgreSQL is a server database and EF Core adds ORM and network costs. Its row +shows absolute timings without an embedded-storage speed ratio. `server_bytes` +measures the benchmark table, indexes and TOAST storage after creation, shown in +a separate server-relations column; it excludes shared WAL, server memory and +local client files. JSON and generated tables record PostgreSQL, Npgsql EF Core +and EF Core versions. Existing published measurements stay intact. CI adds a +pinned PostgreSQL service to each C# object benchmark job so all compared +variants still run on the same machine. Set `POSTGRESQL_VERBOSE=1` to log EF +diagnostics to stderr; logging is off by default and sensitive values are disabled. + Every repetition runs on a fresh store in an empty directory, after discarded warm-up repetitions on up to 10,000 records that run for at least a second, so the .NET JIT has already optimized the code. Each operation is one @@ -95,6 +117,23 @@ dotnet run -c Release --project csharp/SQLiteVSDoublets -- \ python3 scripts/benchmark_report.py results ``` +Run a bounded PostgreSQL comparison: + +```bash +docker run -d --name benchmark-postgres -p 127.0.0.1:5432:5432 \ + -e POSTGRES_USER=benchmark -e POSTGRES_PASSWORD=benchmark \ + -e POSTGRES_DB=benchmark postgres:18.3 +export POSTGRESQL_CONNECTION_STRING='Host=127.0.0.1;Database=benchmark;Username=benchmark;Password=benchmark' +dotnet run -c Release --project csharp/SQLiteVSDoublets -- \ + objects 64 1000 --repetitions 3 \ + --variants SQLite_Memory,SQLite_File,PostgreSQL_EFCore \ + --output results/objects-csharp-64-1000.json +python3 scripts/benchmark_report.py results +python3 experiments/postgresql/check_reports.py +docker stop benchmark-postgres +docker rm benchmark-postgres +``` + Notes: - The split stores of doublets 0.5.0 (Rust) lose a link that is updated to diff --git a/README.ru.md b/README.ru.md index fd108c7..2d5a202 100644 --- a/README.ru.md +++ b/README.ru.md @@ -53,6 +53,44 @@ Microsoft.Data.Sqlite. Оба провайдера используют один надёжности хранения. Опубликованные таблицы старых запусков содержат только измеренные тогда провайдеры; следующий запуск Benchmarks добавит новые строки. +Тесты объектов на C# также поддерживают `PostgreSQL_EFCore` через +[Npgsql.EntityFrameworkCore.PostgreSQL](https://www.npgsql.org/efcore/). +Переменная `POSTGRESQL_CONNECTION_STRING` добавляет этот вариант к сравнению +объектов по умолчанию. Без неё встроенные хранилища работают как прежде, +а явный выбор PostgreSQL завершается сообщением о настройке. Оба размера +идентификаторов используют те же данные, прогрев, проверяемые операции, +порядок точечных чтений и транзакции. EF сохраняет записи по одной, очищает +трекер после сохранения, читает без отслеживания и удаляет по id. Каждый +прогрев и повтор создаёт и удаляет собственную схему со случайным именем; +другие схемы и сама база сохраняются. Пользователю нужны права подключения +и создания схем в базе для тестирования. + +PostgreSQL работает как сервер, а EF Core добавляет расходы ORM и сети. +Строка показывает абсолютное время без коэффициента относительно встроенного +SQLite. `server_bytes` — размер таблицы, индексов и TOAST после создания, +показанный в отдельном столбце; общие WAL, память сервера и локальные файлы +клиента в него не входят. JSON и таблицы содержат версии PostgreSQL, провайдера +Npgsql EF Core и EF Core. Опубликованные измерения сохраняются. CI запускает +PostgreSQL в том же задании, что и остальные варианты объектов C#. +`POSTGRESQL_VERBOSE=1` включает диагностику EF в stderr; по умолчанию она +выключена, чувствительные значения не записываются. + +Пример с небольшим объёмом данных: + +```bash +docker run -d --name benchmark-postgres -p 127.0.0.1:5432:5432 \ + -e POSTGRES_USER=benchmark -e POSTGRES_PASSWORD=benchmark \ + -e POSTGRES_DB=benchmark postgres:18.3 +export POSTGRESQL_CONNECTION_STRING='Host=127.0.0.1;Database=benchmark;Username=benchmark;Password=benchmark' +dotnet run -c Release --project csharp/SQLiteVSDoublets -- \ + objects 64 1000 --repetitions 3 \ + --variants SQLite_Memory,SQLite_File,PostgreSQL_EFCore \ + --output results/objects-csharp-64-1000.json +python3 experiments/postgresql/check_reports.py +docker stop benchmark-postgres +docker rm benchmark-postgres +``` + Каждый повтор выполняется на новом хранилище в пустой папке после отбрасываемых прогревочных повторов на не более чем 10 000 записей, которые длятся не меньше секунды, чтобы JIT .NET уже оптимизировал код. Каждая diff --git a/csharp/SQLiteVSDoublets.Tests/PostgreSQLTests.cs b/csharp/SQLiteVSDoublets.Tests/PostgreSQLTests.cs new file mode 100644 index 0000000..4f318a7 --- /dev/null +++ b/csharp/SQLiteVSDoublets.Tests/PostgreSQLTests.cs @@ -0,0 +1,101 @@ +using Npgsql; +using Xunit; +using static Comparisons.SQLiteVSDoublets.Dataset; + +namespace Comparisons.SQLiteVSDoublets.Tests; + +public class PostgreSQLTests +{ + public static bool IsConfigured => !string.IsNullOrEmpty( + Environment.GetEnvironmentVariable(PostgreSQLBlogPosts.ConnectionStringVariable)); + + [Fact] + public void PostgreSQLUsesTheObjectsHarness() + { + Assert.Contains("PostgreSQL_EFCore", Harness.ObjectsVariants); + Assert.DoesNotContain("PostgreSQL_EFCore", Harness.LinksVariants); + } + + [Theory(Skip = "Set POSTGRESQL_CONNECTION_STRING for PostgreSQL integration tests.", SkipUnless = nameof(IsConfigured))] + [InlineData(false), InlineData(true)] + public void EveryIdWidthPassesValidatedRepetitions(bool wide) + { + var directory = Directory.CreateTempSubdirectory().FullName; + try + { + var measured = wide + ? Harness.MeasureObjects("PostgreSQL_EFCore", 100, 2, directory) + : Harness.MeasureObjects("PostgreSQL_EFCore", 100, 2, directory); + Assert.Null(measured.FileBytes); + Assert.True(measured.ServerBytes > 0); + Assert.Equal(new[] { "create", "read_all", "read_by_id", "delete" }, measured.Operations.Select(operation => operation.Operation)); + Assert.All(measured.Operations, operation => Assert.Equal(2, operation.Samples.Count)); + Assert.Empty(Directory.GetFileSystemEntries(directory)); + } + finally + { + Directory.Delete(directory, true); + } + } + + [Fact(Skip = "Set POSTGRESQL_CONNECTION_STRING for PostgreSQL integration tests.", SkipUnless = nameof(IsConfigured))] + public void PostsRoundTripWithoutAUniqueTitleConstraintOrTrackingCache() + { + using var posts = new PostgreSQLBlogPosts(PostgreSQLBlogPosts.ConnectionString()); + var inputs = new[] { BlogPost(1), new BlogPost("", "", 0), new BlogPost("Ünïcödé 🌍 ✓", "a", 1_600_000_000), BlogPost(1) }; + var ids = posts.Transaction(() => inputs.Select(posts.Create).ToArray()); + Assert.Equal(inputs, ids.Select(posts.Get)); + var stored = new List<(ulong, BlogPost)>(); + posts.Each((id, post) => stored.Add((id, post))); + Assert.Equal(ids.Zip(inputs), stored.OrderBy(entry => entry.Item1)); + + using var connection = new NpgsqlConnection(PostgreSQLBlogPosts.ConnectionString()); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = $"UPDATE \"{posts.SchemaName}\".blog_posts SET title = 'changed on server' WHERE id = $1"; + command.Parameters.AddWithValue(checked((long)ids[0])); + command.ExecuteNonQuery(); + Assert.Equal("changed on server", posts.Get(ids[0])!.Title); + foreach (var id in ids) + { + posts.Delete(id); + } + Assert.Equal(0UL, posts.Count()); + Assert.Null(posts.Get(ids[0])); + } + + [Fact(Skip = "Set POSTGRESQL_CONNECTION_STRING for PostgreSQL integration tests.", SkipUnless = nameof(IsConfigured))] + public void RollbackLeavesTheStoreReusable() + { + using var posts = new PostgreSQLBlogPosts(PostgreSQLBlogPosts.ConnectionString()); + Assert.Throws(() => posts.Transaction(() => + { + posts.Create(BlogPost(1)); + throw new InvalidOperationException("roll back the inserted post"); + })); + Assert.Equal(0UL, posts.Count()); + var id = posts.Transaction(() => posts.Create(BlogPost(2))); + Assert.Equal(BlogPost(2), posts.Get(id)); + Assert.Equal(1UL, posts.Count()); + } + + [Fact(Skip = "Set POSTGRESQL_CONNECTION_STRING for PostgreSQL integration tests.", SkipUnless = nameof(IsConfigured))] + public void StoresAreIsolatedAndDisposeOnlyTheirOwnSchema() + { + using var first = new PostgreSQLBlogPosts(PostgreSQLBlogPosts.ConnectionString()); + using var second = new PostgreSQLBlogPosts(PostgreSQLBlogPosts.ConnectionString()); + Assert.NotEqual(first.SchemaName, second.SchemaName); + var id = second.Create(BlogPost(2)); + first.Create(BlogPost(1)); + var schema = first.SchemaName; + first.Dispose(); + first.Dispose(); + Assert.Equal(BlogPost(2), second.Get(id)); + using var connection = new NpgsqlConnection(PostgreSQLBlogPosts.ConnectionString()); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = "SELECT COUNT(*) FROM pg_namespace WHERE nspname = $1"; + command.Parameters.AddWithValue(schema); + Assert.Equal(0L, command.ExecuteScalar()); + } +} diff --git a/csharp/SQLiteVSDoublets.Tests/StoragesTests.cs b/csharp/SQLiteVSDoublets.Tests/StoragesTests.cs index 4a85878..249bb16 100644 --- a/csharp/SQLiteVSDoublets.Tests/StoragesTests.cs +++ b/csharp/SQLiteVSDoublets.Tests/StoragesTests.cs @@ -29,7 +29,7 @@ public void ScatteredVisitsEveryIdOnce(ulong n) => public static TheoryData LinksVariants => [.. Harness.LinksVariants]; - public static TheoryData ObjectsVariants => [.. Harness.ObjectsVariants]; + public static TheoryData ObjectsVariants => [.. Harness.ObjectsVariants.Where(variant => variant != "PostgreSQL_EFCore")]; [Theory, InlineData("SystemDataSQLite_Memory"), InlineData("SystemDataSQLite_File")] public void SystemDataSQLiteVariantsAreRegistered(string variant) diff --git a/csharp/SQLiteVSDoublets/Harness.cs b/csharp/SQLiteVSDoublets/Harness.cs index 98cc653..9b6c553 100644 --- a/csharp/SQLiteVSDoublets/Harness.cs +++ b/csharp/SQLiteVSDoublets/Harness.cs @@ -31,6 +31,7 @@ public static class Harness public static readonly string[] ObjectsVariants = [ + "PostgreSQL_EFCore", "SQLite_Memory", "SQLite_File", "SystemDataSQLite_Memory", @@ -225,6 +226,8 @@ private static void Check(bool condition, string message) public sealed record Measurement(string Variant, ulong? FileBytes, List<(string Operation, List Samples)> Operations) { + public ulong? ServerBytes { get; init; } + public JsonObject ToJson() { var operations = new JsonObject(); @@ -241,7 +244,12 @@ public JsonObject ToJson() ["samples_ns"] = new JsonArray(samples.Select(sample => (JsonNode)sample).ToArray()), }; } - return new JsonObject { ["variant"] = Variant, ["file_bytes"] = FileBytes, ["operations"] = operations }; + var result = new JsonObject { ["variant"] = Variant, ["file_bytes"] = FileBytes, ["operations"] = operations }; + if (ServerBytes is not null) + { + result["server_bytes"] = ServerBytes; + } + return result; } } @@ -258,7 +266,8 @@ private static Measurement Measure( int repetitions, string directory, Func open, - Func> lifecycle) + Func> lifecycle, + Func? serverBytes = null) where TStorage : IDisposable { (List<(string, TimeSpan)>, ulong) Run(ulong size) @@ -269,11 +278,15 @@ private static Measurement Measure( GC.WaitForPendingFinalizers(); ulong fileBytes = 0; List<(string, TimeSpan)> timings; - using (var storage = open(workspace)) + try { - timings = lifecycle(storage, size, () => fileBytes = DirectoryBytes(workspace)); + using var storage = open(workspace); + timings = lifecycle(storage, size, () => fileBytes = serverBytes?.Invoke(storage) ?? DirectoryBytes(workspace)); + } + finally + { + Directory.Delete(workspace, true); } - Directory.Delete(workspace, true); return (timings, fileBytes); } @@ -288,7 +301,9 @@ private static Measurement Measure( { var (timings, fileBytes) = Run(n); Console.Error.WriteLine($"{variant} #{repetition}: {Summary(timings, n)}"); - measurement = measurement with { FileBytes = fileBytes > 0 ? fileBytes : null }; + measurement = serverBytes is null + ? measurement with { FileBytes = fileBytes > 0 ? fileBytes : null } + : measurement with { ServerBytes = fileBytes }; foreach (var (operation, elapsed) in timings) { var nsPerOperation = elapsed.TotalNanoseconds / n; @@ -346,6 +361,12 @@ public static Measurement MeasureLinks(string variant, ulong n, int repetitio public static Measurement MeasureObjects(string variant, ulong n, int repetitions, string directory) where T : struct, IBinaryInteger, IUnsignedNumber, IMinMaxValue { + if (variant == "PostgreSQL_EFCore") + { + return Measure(variant, n, repetitions, directory, + _ => new PostgreSQLBlogPosts(PostgreSQLBlogPosts.ConnectionString()), + ObjectsLifecycle, storage => storage.ServerBytes()); + } var cached = variant.EndsWith("_Cached"); Func> open = variant.Replace("_Uncached", "").Replace("_Cached", "") switch { diff --git a/csharp/SQLiteVSDoublets/PostgreSQLBlogPosts.cs b/csharp/SQLiteVSDoublets/PostgreSQLBlogPosts.cs new file mode 100644 index 0000000..67fddd0 --- /dev/null +++ b/csharp/SQLiteVSDoublets/PostgreSQLBlogPosts.cs @@ -0,0 +1,167 @@ +using System.Numerics; +using Microsoft.EntityFrameworkCore; +using Npgsql; + +namespace Comparisons.SQLiteVSDoublets; + +/// EF Core blog posts in a fresh, owned schema on a configured PostgreSQL server. +public sealed class PostgreSQLBlogPosts : IBlogPostsStorage where T : struct, IBinaryInteger +{ + public static string ConnectionStringVariable => "POSTGRESQL_CONNECTION_STRING"; + private readonly NpgsqlConnection _connection; + private readonly PostsContext _context; + private bool _disposed; + + public string SchemaName { get; } = $"benchmark_{Guid.NewGuid():N}"; + public string ServerVersion => _connection.ServerVersion; + + public static string ConnectionString() => + Environment.GetEnvironmentVariable(ConnectionStringVariable) is { Length: > 0 } value + ? value + : throw new InvalidOperationException($"Set {ConnectionStringVariable} to run PostgreSQL_EFCore."); + + public PostgreSQLBlogPosts(string connectionString) + { + _connection = new NpgsqlConnection(connectionString); + var options = new DbContextOptionsBuilder().UseNpgsql(_connection); + if (Environment.GetEnvironmentVariable("POSTGRESQL_VERBOSE") == "1") + { + options.LogTo(Console.Error.WriteLine); + } + _context = new PostsContext(options.Options); + try + { + _connection.Open(); + // Identifiers contain only a fixed prefix and a generated hexadecimal GUID. + Execute($"CREATE SCHEMA \"{SchemaName}\""); + Execute($"SET search_path TO \"{SchemaName}\""); + _context.Database.ExecuteSqlRaw(_context.Database.GenerateCreateScript()); + } + catch + { + Dispose(); + throw; + } + } + + public T Create(BlogPost post) + { + var row = new PostRow + { + Id = 0, // EF treats zero as unset and replaces it with the server-generated identity. + Title = post.Title, + Content = post.Content, + PublicationDate = checked((long)post.PublicationDate), + }; + _context.Posts.Add(row); + _context.SaveChanges(); + // Point reads must hit the server, and the tracker must not grow with the dataset. + _context.ChangeTracker.Clear(); + return T.CreateChecked(row.Id); + } + + public BlogPost? Get(T id) + { + var key = long.CreateChecked(id); + var row = _context.Posts.AsNoTracking().SingleOrDefault(post => post.Id == key); + return row is null ? null : Post(row); + } + + public void Each(Action visit) + { + foreach (var row in _context.Posts.AsNoTracking()) + { + visit(T.CreateChecked(row.Id), Post(row)); + } + } + + public void Delete(T id) + { + var key = long.CreateChecked(id); + _context.Posts.Where(post => post.Id == key).ExecuteDelete(); + } + + public ulong Count() => checked((ulong)_context.Posts.LongCount()); + + public TResult Transaction(Func work) + { + using var transaction = _context.Database.BeginTransaction(); + try + { + var result = work(); + transaction.Commit(); + return result; + } + finally + { + _context.ChangeTracker.Clear(); + } + } + + /// Server table, index and TOAST bytes; excludes shared WAL and server memory. + public ulong ServerBytes() + { + using var command = _connection.CreateCommand(); + command.CommandText = """ + SELECT COALESCE(SUM(pg_total_relation_size(c.oid)), 0)::bigint + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relkind = 'r' + """; + command.Parameters.AddWithValue(SchemaName); + return checked((ulong)(long)command.ExecuteScalar()!); + } + + private void Execute(string sql) + { + using var command = _connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } + + public void Dispose() + { + if (_disposed) + { + return; + } + _disposed = true; + try + { + if (_connection.State == System.Data.ConnectionState.Open) + { + Execute($"DROP SCHEMA IF EXISTS \"{SchemaName}\" CASCADE"); + } + } + finally + { + _context.Dispose(); + _connection.Dispose(); + } + } + + private static BlogPost Post(PostRow row) => new(row.Title, row.Content, checked((ulong)row.PublicationDate)); + + private sealed class PostRow + { + public long Id { get; set; } + public string Title { get; set; } = ""; + public string Content { get; set; } = ""; + public long PublicationDate { get; set; } + } + + private sealed class PostsContext(DbContextOptions options) : DbContext(options) + { + public DbSet Posts => Set(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + var post = modelBuilder.Entity(); + post.ToTable("blog_posts"); + post.HasKey(row => row.Id); + post.Property(row => row.Id).HasColumnName("id"); + post.Property(row => row.Title).HasColumnName("title"); + post.Property(row => row.Content).HasColumnName("content"); + post.Property(row => row.PublicationDate).HasColumnName("publication_date"); + } + } +} diff --git a/csharp/SQLiteVSDoublets/Program.cs b/csharp/SQLiteVSDoublets/Program.cs index d2a3254..ddad2b8 100644 --- a/csharp/SQLiteVSDoublets/Program.cs +++ b/csharp/SQLiteVSDoublets/Program.cs @@ -24,7 +24,8 @@ var allVariants = category == "links" ? Harness.LinksVariants : Harness.ObjectsVariants; var variants = options.TryGetValue("variants", out var names) ? names.Split(',').Select(name => allVariants.Contains(name) ? name : throw new ArgumentException(name)).ToArray() - : allVariants; + : allVariants.Where(name => name != "PostgreSQL_EFCore" || + !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(PostgreSQLBlogPosts.ConnectionStringVariable))).ToArray(); var directory = options.GetValueOrDefault("directory") ?? Path.GetTempPath(); var measurements = variants.Select(variant => (category, bits) switch @@ -61,6 +62,18 @@ }, ["results"] = new JsonArray(measurements.Select(measurement => (JsonNode)measurement.ToJson()).ToArray()), }; +if (variants.Contains("PostgreSQL_EFCore")) +{ + using var connection = new Npgsql.NpgsqlConnection(PostgreSQLBlogPosts.ConnectionString()); + connection.Open(); + report["postgresql"] = new JsonObject + { + ["provider"] = "Npgsql.EntityFrameworkCore.PostgreSQL", + ["provider_version"] = typeof(Npgsql.EntityFrameworkCore.PostgreSQL.Infrastructure.NpgsqlDbContextOptionsBuilder).Assembly.GetName().Version!.ToString(), + ["ef_core_version"] = typeof(Microsoft.EntityFrameworkCore.DbContext).Assembly.GetName().Version!.ToString(), + ["server_version"] = connection.ServerVersion, + }; +} var json = report.ToJsonString(new JsonSerializerOptions { WriteIndented = true }); if (options.TryGetValue("output", out var output)) { diff --git a/csharp/SQLiteVSDoublets/SQLiteVSDoublets.csproj b/csharp/SQLiteVSDoublets/SQLiteVSDoublets.csproj index 7aa2830..22929aa 100644 --- a/csharp/SQLiteVSDoublets/SQLiteVSDoublets.csproj +++ b/csharp/SQLiteVSDoublets/SQLiteVSDoublets.csproj @@ -13,6 +13,7 @@ + diff --git a/dev/log/issues/33/pulls/99/ANALYSIS.md b/dev/log/issues/33/pulls/99/ANALYSIS.md new file mode 100644 index 0000000..43bfe40 --- /dev/null +++ b/dev/log/issues/33/pulls/99/ANALYSIS.md @@ -0,0 +1,96 @@ +# PostgreSQL comparison investigation + +Issue #33 requests Npgsql.EntityFrameworkCore.PostgreSQL. The PR review asks +that it fit the current benchmark architecture and preserve its value. + +## Historical CI and merge conflicts + +The branch head at the start was `e32623e`, committed on 2025-09-14. +CI run `17704198070`, created at 2025-09-14T00:48:35Z, matches that SHA and +failed on Linux, Windows and macOS. Earlier failed run `17704191247` matches +implementation commit `4d3d2c6`. Downloading both logs returns HTTP 410; +the available Linux annotation says only “Process completed with exit code 1.” +The historical build diagnostics cannot be recovered from those annotations. + +Current main moved C# into `csharp/` and replaced the old BenchmarkDotNet +project with a deterministic, checksum-validating harness. The draft still +modified deleted root files, producing modify/delete conflicts. Merge commit +`a59755e` preserves history and adopts the maintained harness. Its Release +build succeeds with zero warnings and errors before adding the new provider. +The PostgreSQL comparison is implemented in that harness, rather than reviving +the obsolete root project or downgrading the current .NET 10 target. + +## Reproduction before implementation + +`PostgreSQLUsesTheObjectsHarness` failed because `PostgreSQL_EFCore` was absent +from `Harness.ObjectsVariants`. + +Three report regressions failed before implementation: + +- `baseline("PostgreSQL_EFCore")` returned `SQLite_Memory`. +- A PostgreSQL row was labeled “3× slower” against embedded SQLite memory, + and its server relation size was not displayed. +- PostgreSQL results without server/provider versions were accepted. + +These tests now pass. The artifact regression also requires PostgreSQL in +each C# object benchmark cell, so a missing measurement fails publication. + +## Architecture and scope + +The provider implements `IBlogPostsStorage` and uses `ObjectsLifecycle` for +create, read all, scattered reads by id and delete with 32-bit and 64-bit ids. +All operations use the shared deterministic posts and checksum validation. +One transaction covers each timed operation, with per-record EF saves, +untracked reads and deletes by id. Clearing tracking after each save prevents +the read benchmark from returning cached entities and bounds tracker growth. + +Each warm-up/repetition owns a unique schema. Creation and schema teardown +happen outside timing. Teardown drops only that generated schema, including +on failed lifecycles. No caller database or shared table is deleted. No unique +title constraint or generated date default changes the shared workload. +The date remains integer Unix seconds, including zero, as in the SQLite store. + +The provider is an optional local object variant enabled through +`POSTGRESQL_CONNECTION_STRING`; explicit selection without configuration +fails. Existing embedded variants and links benchmarks remain available. +CI supplies a pinned PostgreSQL service beside all C# object variants, with +another job for PostgreSQL integration and configuration/report tests. + +Server timings include EF and local client/server communication, so the report +does not assign an embedded SQLite durability baseline to PostgreSQL. +`server_bytes` is measured separately from `file_bytes`: table, indexes and +TOAST through `pg_total_relation_size`, excluding shared WAL and server memory. +Provider, EF Core and server versions are reported without connection strings. + +The implementation follows the [Npgsql EF Core documentation](https://www.npgsql.org/efcore/) +and [PostgreSQL size functions](https://www.postgresql.org/docs/current/functions-admin.html). + +## Local verification + +- Release build with warnings as errors and C# whitespace checks pass. +- 43 C# tests pass with PostgreSQL 16.15: both id widths, repeated validated + lifecycles, Unicode/empty strings, duplicate titles, date zero, fresh server + reads after an external update, rollback/reuse and schema isolation/cleanup. +- The executable PostgreSQL report probe passes for both id widths, checks + metadata and server bytes, preserves all 12 embedded variants without server + configuration and rejects explicit unconfigured selection. +- 59 Python tests and six evidence integrity tests pass. +- Rust formatting, Clippy with warnings denied and all nine Rust tests pass. +- The existing SQLite executable probe passes all four provider report cells. +- Ruff, mypy, Bandit, Markdown, workflow policies, actionlint/ShellCheck, + both zizmor profiles and the direct/transitive NuGet audit pass. + +Docker cannot start containers in this workspace's cgroup configuration; +PostgreSQL was extracted from Ubuntu packages into a temporary directory and +run as the workspace user with 64 MiB shared buffers and 20 connections. +Hosted integration uses the pinned PostgreSQL 18.3 service. The bounded tests +establish correctness and report integration, not a general performance claim. + +## Hosted static analysis + +Codacy on `522ee59` reported S2339 for the public environment-variable constant +and S3459 for the EF-generated identity property. The configuration name now +uses a static read-only property. New rows explicitly start with the unset +identity value zero, which EF replaces with the generated key after saving. +The PostgreSQL integration tests and executable report probe verify that ids +are generated and round-trip correctly with these changes. diff --git a/experiments/postgresql/check_reports.py b/experiments/postgresql/check_reports.py new file mode 100644 index 0000000..1632781 --- /dev/null +++ b/experiments/postgresql/check_reports.py @@ -0,0 +1,84 @@ +"""Check bounded PostgreSQL object benchmarks, metadata and explicit configuration.""" + +import json +import os +import subprocess # nosec B404 # fixed executable and argument lists, without a shell. +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "scripts")) +from benchmark_report import validate # noqa: E402 +from ci_process import executable # noqa: E402 + +BENCHMARK = ROOT / "csharp/SQLiteVSDoublets/bin/Release/net10.0/sqlite-vs-doublets.dll" +VARIABLE = "POSTGRESQL_CONNECTION_STRING" +VARIANTS = "SQLite_Memory,SQLite_File,PostgreSQL_EFCore" + + +class PostgreSQLReports(unittest.TestCase): + def run_benchmark(self, directory, bits, variants=None, configured=True): + arguments = [ + executable("dotnet"), + str(BENCHMARK), + "objects", + str(bits), + "100", + "--repetitions", + "2", + "--directory", + directory, + ] + if variants: + arguments += ["--variants", variants] + environment = os.environ.copy() + if not configured: + environment.pop(VARIABLE, None) + return subprocess.run( # nosec B603 # fixed benchmark, bounded inputs and separate arguments. + arguments, capture_output=True, text=True, env=environment, check=False + ) + + def test_both_id_widths_report_server_bytes_and_versions(self): + self.assertTrue(os.environ.get(VARIABLE), f"Set {VARIABLE} for integration tests") + with tempfile.TemporaryDirectory() as directory: + for bits in (32, 64): + with self.subTest(bits=bits): + result = self.run_benchmark(directory, bits, VARIANTS) + self.assertEqual(result.returncode, 0, result.stderr) + data = json.loads(result.stdout) + validate(data, "PostgreSQL benchmark") + postgres = data["postgresql"] + self.assertEqual(postgres["provider"], "Npgsql.EntityFrameworkCore.PostgreSQL") + self.assertTrue(all(postgres.values())) + self.assertNotIn("Password", result.stdout) + measured = next(row for row in data["results"] if row["variant"] == "PostgreSQL_EFCore") + self.assertIsNone(measured["file_bytes"]) + self.assertGreater(measured["server_bytes"], 0) + self.assertEqual( + set(measured["operations"]), {"create", "read_all", "read_by_id", "delete"} + ) + for operation in measured["operations"].values(): + self.assertEqual(len(operation["samples_ns"]), 2) + self.assertFalse(list(Path(directory).iterdir())) + + def test_unconfigured_default_preserves_embedded_variants(self): + with tempfile.TemporaryDirectory() as directory: + result = self.run_benchmark(directory, 32, configured=False) + self.assertEqual(result.returncode, 0, result.stderr) + data = json.loads(result.stdout) + self.assertNotIn("postgresql", data) + self.assertEqual(len(data["results"]), 12) + self.assertNotIn("PostgreSQL_EFCore", [row["variant"] for row in data["results"]]) + + def test_explicit_unconfigured_postgresql_fails(self): + with tempfile.TemporaryDirectory() as directory: + result = self.run_benchmark(directory, 32, "PostgreSQL_EFCore", configured=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn(f"Set {VARIABLE}", result.stderr) + self.assertFalse(list(Path(directory).iterdir())) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/benchmark_ci.py b/scripts/benchmark_ci.py index da086b2..4ed8867 100644 --- a/scripts/benchmark_ci.py +++ b/scripts/benchmark_ci.py @@ -29,6 +29,8 @@ def expected_variants(category, language): variants = VARIANTS[category] if language in ("C#", "csharp"): variants = variants | {"SystemDataSQLite_Memory", "SystemDataSQLite_File"} + if category == "objects": + variants = variants | {"PostgreSQL_EFCore"} return variants diff --git a/scripts/benchmark_report.py b/scripts/benchmark_report.py index 3f0cfe5..7508196 100644 --- a/scripts/benchmark_report.py +++ b/scripts/benchmark_report.py @@ -71,6 +71,7 @@ def russian_plural(count, one, few, many): }, "storage": "Storage", "file_size": "File size", + "server_size": "Server relations", "faster": "{ratio}× faster", "slower": "{ratio}× slower", "same": "≈ same", @@ -105,6 +106,7 @@ def russian_plural(count, one, few, many): }, "storage": "Хранилище", "file_size": "Размер файлов", + "server_size": "Размер на сервере", "faster": "в {ratio}× быстрее", "slower": "в {ratio}× медленнее", "same": "≈ так же", @@ -140,6 +142,7 @@ def validate(report, path): try: validate_metadata(report) validate_variants(report["results"]) + validate_postgresql(report) for result in report["results"]: if set(result["operations"]) != set(OPERATIONS[report["category"]]): raise ValueError("missing or unexpected operations") @@ -168,6 +171,23 @@ def validate_variants(results): raise ValueError("missing SQLite baseline") +def validate_postgresql(report): + """Require PostgreSQL provenance and keep server bytes separate from local files.""" + postgres = [result for result in report["results"] if result["variant"] == "PostgreSQL_EFCore"] + if not postgres: + return + metadata = report.get("postgresql", {}) + if ( + report["language"] != "C#" + or report["category"] != "objects" + or metadata.get("provider") != "Npgsql.EntityFrameworkCore.PostgreSQL" + or not all(metadata.get(key) for key in ("provider_version", "ef_core_version", "server_version")) + or postgres[0].get("file_bytes") is not None + or not positive_integer(postgres[0].get("server_bytes")) + ): + raise ValueError("missing or invalid PostgreSQL provenance or server size") + + def positive_integer(value): return isinstance(value, int) and not isinstance(value, bool) and value > 0 @@ -189,7 +209,7 @@ def validate_measurement(measurement): def baseline(variant): """Compare with Microsoft.Data.Sqlite of the same durability, including the System.Data.SQLite provider.""" - if variant.startswith("SQLite"): + if variant.startswith("SQLite") or variant == "PostgreSQL_EFCore": return None if variant.startswith("SystemDataSQLite_"): return variant.replace("SystemDataSQLite_", "SQLite_", 1) @@ -245,10 +265,12 @@ def comparison(measured, reference, text): def table(report, text): operations = OPERATIONS[report["category"]] by_variant = {result["variant"]: result for result in report["results"]} + has_server = "PostgreSQL_EFCore" in by_variant header = [ text["storage"], *(text["operations"][operation] for operation in operations), text["file_size"], + *([text["server_size"]] if has_server else []), ] lines = ["| " + " | ".join(header) + " |", "| --- |" + " ---: |" * (len(header) - 1)] for variant, result in by_variant.items(): @@ -261,18 +283,28 @@ def table(report, text): cell += f" ({comparison(measured, reference['operations'][operation], text)})" cells.append(cell) cells.append(file_size(result["file_bytes"])) + if has_server: + cells.append(file_size(result.get("server_bytes"))) lines.append("| " + " | ".join(cells) + " |") return "\n".join(lines) def provenance(report, text): run = report.get("run_url") - return text["provenance"].format( + value = text["provenance"].format( repetitions=text["repetitions"](report["repetitions"]), sqlite=report["sqlite_version"], machine=report.get("machine") or text["machine"], source=text["run"].format(url=run, date=report.get("date")) if run else text["local"], ) + if "postgresql" in report: + postgres = report["postgresql"] + value += ( + f" PostgreSQL {postgres['server_version']}; " + f"Npgsql.EntityFrameworkCore.PostgreSQL {postgres['provider_version']}; " + f"EF Core {postgres['ef_core_version']}." + ) + return value def chart_name(category, language, bits): diff --git a/scripts/test_benchmark_ci.py b/scripts/test_benchmark_ci.py index 09df74c..a5a187b 100644 --- a/scripts/test_benchmark_ci.py +++ b/scripts/test_benchmark_ci.py @@ -17,6 +17,16 @@ def complete_sample(category, language, bits, size, **extra): dict(copy.deepcopy(measured), variant=variant) for variant in sorted(ci.expected_variants(category, language)) ] + if category == "objects" and language == "C#": + data["postgresql"] = { + "provider": "Npgsql.EntityFrameworkCore.PostgreSQL", + "provider_version": "10.0.3", + "ef_core_version": "10.0.4", + "server_version": "18.3", + } + next(result for result in data["results"] if result["variant"] == "PostgreSQL_EFCore")[ + "server_bytes" + ] = 8192 return data @@ -52,6 +62,14 @@ def test_missing_storage_variant_fails_even_with_valid_baselines(self): with self.assertRaisesRegex(ValueError, "variants"): ci.verify(self.root, [1000], self.sha) + def test_missing_postgresql_measurement_fails_the_csharp_objects_plan(self): + path = self.root / "objects-csharp-32-1000.json" + data = json.loads(path.read_text()) + data["results"] = [result for result in data["results"] if result["variant"] != "PostgreSQL_EFCore"] + path.write_text(json.dumps(data), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "variants"): + ci.verify(self.root, [1000], self.sha) + def test_stale_result_is_not_a_substitute_for_a_failed_job(self): path = next(self.root.glob("*.json")) data = json.loads(path.read_text()) @@ -87,7 +105,8 @@ def test_csharp_preserves_both_sqlite_providers(self): for category in ("links", "objects"): self.assertEqual( ci.expected_variants(category, "C#") - ci.expected_variants(category, "Rust"), - {"SystemDataSQLite_Memory", "SystemDataSQLite_File"}, + {"SystemDataSQLite_Memory", "SystemDataSQLite_File"} + | ({"PostgreSQL_EFCore"} if category == "objects" else set()), ) def test_default_matrix_preserves_twenty_tables(self): diff --git a/scripts/test_postgresql_report.py b/scripts/test_postgresql_report.py new file mode 100644 index 0000000..1c76863 --- /dev/null +++ b/scripts/test_postgresql_report.py @@ -0,0 +1,31 @@ +"""Keep PostgreSQL measurements distinct from embedded storage measurements.""" + +import unittest + +import benchmark_report as report +from test_benchmark_report import results, sample + + +class PostgreSQLReportTests(unittest.TestCase): + def test_server_provider_has_no_embedded_durability_baseline(self): + self.assertIsNone(report.baseline("PostgreSQL_EFCore")) + + def test_server_size_is_shown_separately_from_local_files(self): + data = sample("objects", "C#", 32, 1000) + data["results"] += results("objects", {"PostgreSQL_EFCore": 3000})["results"] + data["results"][-1]["server_bytes"] = 2**20 + table = report.table(data, report.TEXT["en"]) + self.assertIn("Server relations", table) + row = next(line for line in table.splitlines() if "PostgreSQL EFCore" in line) + self.assertIn("1.0 MiB", row) + self.assertNotIn("slower", row) + + def test_postgresql_results_require_server_and_provider_versions(self): + data = sample("objects", "C#", 32, 1000) + data["results"] += results("objects", {"PostgreSQL_EFCore": 3000})["results"] + with self.assertRaisesRegex(ValueError, "PostgreSQL"): + report.validate(data, "postgresql.json") + + +if __name__ == "__main__": + unittest.main()