Benchmark System.Data.SQLite alongside Microsoft.Data.Sqlite and Doublets

.github/workflows/csharp.yml

@@ -3,9 +3,9 @@ name: C#
 on:
   push:
     branches: [main]
-    paths: ['csharp/**', '.github/workflows/csharp.yml']
+    paths: ['csharp/**', 'experiments/system_data_sqlite/**', '.github/workflows/csharp.yml']
   pull_request:
-    paths: ['csharp/**', '.github/workflows/csharp.yml']
+    paths: ['csharp/**', 'experiments/system_data_sqlite/**', '.github/workflows/csharp.yml']
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -31,3 +31,5 @@ jobs:
           global-json-file: csharp/global.json
       - run: dotnet build -c Release -warnaserror
       - run: dotnet test --project SQLiteVSDoublets.Tests -c Release --no-build
+      - name: Check SQLite provider reports
+        run: python ../experiments/system_data_sqlite/check_reports.py

README.md

@@ -18,7 +18,8 @@ Rust ([doublets](https://crates.io/crates/doublets) and
 [rusqlite](https://crates.io/crates/rusqlite) with bundled SQLite) and C#
 ([Platform.Data.Doublets](https://www.nuget.org/packages/Platform.Data.Doublets),
 [Platform.Data.Doublets.Sequences](https://www.nuget.org/packages/Platform.Data.Doublets.Sequences)
-and [Microsoft.Data.Sqlite](https://www.nuget.org/packages/Microsoft.Data.Sqlite))
+and the providers [Microsoft.Data.Sqlite](https://www.nuget.org/packages/Microsoft.Data.Sqlite)
+and [System.Data.SQLite](https://www.nuget.org/packages/System.Data.SQLite))
 run the same workloads on the same deterministic data, with 32 bit
 (`u32`/`uint`) and 64 bit (`u64`/`ulong`) ids:
 
@@ -42,6 +43,16 @@ with index trees) and split (separate data and index arrays), each volatile
 with SQLite of the same durability: volatile with `SQLite Memory`,
 non-volatile with `SQLite File`.
 
+C# also runs `SystemDataSQLite_Memory` and `SystemDataSQLite_File` with
+System.Data.SQLite 2.0.4. `SQLite_Memory` and `SQLite_File` use
+Microsoft.Data.Sqlite. Both providers use the same schemas, parameterized
+commands, immediate transactions and native `e_sqlite3` library supplied by
+Microsoft.Data.Sqlite, on Linux, macOS and Windows. The JSON reports record
+each provider and engine version in `sqlite_providers`. The System.Data.SQLite
+rows compare with Microsoft.Data.Sqlite of the same durability. Published
+tables from older runs include only the providers measured in those runs;
+the next Benchmarks run adds the new rows.
+
 Every repetition runs on a fresh store in an empty directory, after discarded
 warm-up repetitions on up to 10,000 records that run for at least a second, so
 the .NET JIT has already optimized the code. Each operation is one
@@ -76,6 +87,10 @@ cargo run --release --manifest-path rust/Cargo.toml -- \
   links 64 100000 --output results/links-rust-64-100000.json
 dotnet run -c Release --project csharp/SQLiteVSDoublets -- \
   objects 32 1000 --variants SQLite_File,Doublets_Split_NonVolatile_Cached
+dotnet run -c Release --project csharp/SQLiteVSDoublets -- \
+  links 64 1000 --repetitions 3 \
+  --variants SQLite_Memory,SystemDataSQLite_Memory \
+  --output results/links-csharp-64-1000.json
 # print the tables, add --readme README.md --charts docs/benchmarks to update
 python3 scripts/benchmark_report.py results
 ```

README.ru.md

@@ -18,7 +18,8 @@ Rust ([doublets](https://crates.io/crates/doublets) и
 [rusqlite](https://crates.io/crates/rusqlite) со встроенной SQLite) и C#
 ([Platform.Data.Doublets](https://www.nuget.org/packages/Platform.Data.Doublets),
 [Platform.Data.Doublets.Sequences](https://www.nuget.org/packages/Platform.Data.Doublets.Sequences)
-и [Microsoft.Data.Sqlite](https://www.nuget.org/packages/Microsoft.Data.Sqlite))
+и провайдеры [Microsoft.Data.Sqlite](https://www.nuget.org/packages/Microsoft.Data.Sqlite)
+и [System.Data.SQLite](https://www.nuget.org/packages/System.Data.SQLite))
 выполняют одинаковую нагрузку на одинаковых детерминированных данных
 с 32-битными (`u32`/`uint`) и 64-битными (`u64`/`ulong`) идентификаторами:
 
@@ -42,6 +43,16 @@ Rust ([doublets](https://crates.io/crates/doublets) и
 файлах). Дуплеты сравниваются с SQLite той же надёжности хранения:
 энергозависимые с `SQLite Memory`, энергонезависимые с `SQLite File`.
 
+C# также проверяет `SystemDataSQLite_Memory` и `SystemDataSQLite_File`
+с System.Data.SQLite 2.0.4. `SQLite_Memory` и `SQLite_File` используют
+Microsoft.Data.Sqlite. Оба провайдера используют одинаковые схемы,
+параметризованные команды, немедленные транзакции и нативную библиотеку
+`e_sqlite3` из Microsoft.Data.Sqlite на Linux, macOS и Windows. JSON-отчёты
+содержат версии каждого провайдера и движка в `sqlite_providers`.
+Строки System.Data.SQLite сравниваются с Microsoft.Data.Sqlite той же
+надёжности хранения. Опубликованные таблицы старых запусков содержат только
+измеренные тогда провайдеры; следующий запуск Benchmarks добавит новые строки.
+
 Каждый повтор выполняется на новом хранилище в пустой папке после
 отбрасываемых прогревочных повторов на не более чем 10 000 записей, которые
 длятся не меньше секунды, чтобы JIT .NET уже оптимизировал код. Каждая
@@ -77,6 +88,10 @@ cargo run --release --manifest-path rust/Cargo.toml -- \
   links 64 100000 --output results/links-rust-64-100000.json
 dotnet run -c Release --project csharp/SQLiteVSDoublets -- \
   objects 32 1000 --variants SQLite_File,Doublets_Split_NonVolatile_Cached
+dotnet run -c Release --project csharp/SQLiteVSDoublets -- \
+  links 64 1000 --repetitions 3 \
+  --variants SQLite_Memory,SystemDataSQLite_Memory \
+  --output results/links-csharp-64-1000.json
 # печатает таблицы, для обновления этого файла добавьте
 # --readme README.ru.md --charts docs/benchmarks
 python3 scripts/benchmark_report.py results

csharp/SQLiteVSDoublets.Tests/StoragesTests.cs

@@ -31,6 +31,13 @@ public void ScatteredVisitsEveryIdOnce(ulong n) =>
 
     public static TheoryData<string> ObjectsVariants => [.. Harness.ObjectsVariants];
 
+    [Theory, InlineData("SystemDataSQLite_Memory"), InlineData("SystemDataSQLite_File")]
+    public void SystemDataSQLiteVariantsAreRegistered(string variant)
+    {
+        Assert.Contains(variant, Harness.LinksVariants);
+        Assert.Contains(variant, Harness.ObjectsVariants);
+    }
+
     [Theory, MemberData(nameof(LinksVariants))]
     public void EveryLinksVariantPassesTheLifecycle(string variant)
     {
@@ -103,9 +110,31 @@ public void EveryObjectsStorageReturnsTheStoredPosts(bool cached)
     {
         RoundTrip(SQLiteBlogPosts<uint>.InMemory());
         RoundTrip(SQLiteBlogPosts<ulong>.InMemory());
+        RoundTrip(SQLiteBlogPosts<uint>.InMemory(SQLiteProvider.SystemDataSQLite));
+        RoundTrip(SQLiteBlogPosts<ulong>.InMemory(SQLiteProvider.SystemDataSQLite));
         RoundTrip(new DoubletsBlogPosts<uint>(Harness.United<uint>(new HeapResizableDirectMemory(), external: true), cached));
         RoundTrip(new DoubletsBlogPosts<ulong>(Harness.United<ulong>(new HeapResizableDirectMemory(), external: true), cached));
         RoundTrip(new DoubletsBlogPosts<uint>(Harness.Split<uint>(new HeapResizableDirectMemory(), new HeapResizableDirectMemory(), external: true), cached));
         RoundTrip(new DoubletsBlogPosts<ulong>(Harness.Split<ulong>(new HeapResizableDirectMemory(), new HeapResizableDirectMemory(), external: true), cached));
     }
+
+    [Theory, InlineData(SQLiteProvider.MicrosoftDataSqlite), InlineData(SQLiteProvider.SystemDataSQLite)]
+    public void SQLiteCommandsCanBeReusedAfterRollback(SQLiteProvider provider)
+    {
+        using var links = SQLiteLinks<ulong>.InMemory(provider);
+        Assert.Throws<InvalidOperationException>(() => links.Transaction<int>(() =>
+        {
+            links.Create(1, 2);
+            throw new InvalidOperationException("roll back the inserted link");
+        }));
+        Assert.Equal(0UL, links.Count());
+        Assert.Null(links.Search(1, 2));
+        var id = links.Transaction(() => links.Create(3, 4));
+        Assert.Equal(new Link<ulong>(id, 3, 4), links.Get(id));
+        Assert.Equal(1UL, links.Count());
+    }
+
+    [Fact]
+    public void BothSQLiteProvidersUseTheSameNativeEngine() =>
+        Assert.Equal(SQLiteStorage.Version(SQLiteProvider.MicrosoftDataSqlite), SQLiteStorage.Version(SQLiteProvider.SystemDataSQLite));
 }

csharp/SQLiteVSDoublets/Harness.cs

@@ -21,6 +21,8 @@ public static class Harness
     [
         "SQLite_Memory",
         "SQLite_File",
+        "SystemDataSQLite_Memory",
+        "SystemDataSQLite_File",
         "Doublets_United_Volatile",
         "Doublets_United_NonVolatile",
         "Doublets_Split_Volatile",
@@ -31,6 +33,8 @@ public static class Harness
     [
         "SQLite_Memory",
         "SQLite_File",
+        "SystemDataSQLite_Memory",
+        "SystemDataSQLite_File",
         "Doublets_United_Volatile_Cached",
         "Doublets_United_Volatile_Uncached",
         "Doublets_United_NonVolatile_Cached",
@@ -328,6 +332,8 @@ public static Measurement MeasureLinks<T>(string variant, ulong n, int repetitio
         {
             "SQLite_Memory" => _ => SQLiteLinks<T>.InMemory(),
             "SQLite_File" => dir => SQLiteLinks<T>.Open(File(dir, "links.db")),
+            "SystemDataSQLite_Memory" => _ => SQLiteLinks<T>.InMemory(SQLiteProvider.SystemDataSQLite),
+            "SystemDataSQLite_File" => dir => SQLiteLinks<T>.Open(File(dir, "links.db"), SQLiteProvider.SystemDataSQLite),
             "Doublets_United_Volatile" => _ => new DoubletsLinks<T>(United<T>(new HeapResizableDirectMemory(), external: false)),
             "Doublets_United_NonVolatile" => dir => new DoubletsLinks<T>(United<T>(Mapped(dir, "links.links"), external: false)),
             "Doublets_Split_Volatile" => _ => new DoubletsLinks<T>(Split<T>(new HeapResizableDirectMemory(), new HeapResizableDirectMemory(), external: false)),
@@ -345,6 +351,8 @@ public static Measurement MeasureObjects<T>(string variant, ulong n, int repetit
         {
             "SQLite_Memory" => _ => SQLiteBlogPosts<T>.InMemory(),
             "SQLite_File" => dir => SQLiteBlogPosts<T>.Open(File(dir, "blog_posts.db")),
+            "SystemDataSQLite_Memory" => _ => SQLiteBlogPosts<T>.InMemory(SQLiteProvider.SystemDataSQLite),
+            "SystemDataSQLite_File" => dir => SQLiteBlogPosts<T>.Open(File(dir, "blog_posts.db"), SQLiteProvider.SystemDataSQLite),
             "Doublets_United_Volatile" => _ => new DoubletsBlogPosts<T>(United<T>(new HeapResizableDirectMemory(), external: true), cached),
             "Doublets_United_NonVolatile" => dir => new DoubletsBlogPosts<T>(United<T>(Mapped(dir, "links.links"), external: true), cached),
             "Doublets_Split_Volatile" => _ => new DoubletsBlogPosts<T>(Split<T>(new HeapResizableDirectMemory(), new HeapResizableDirectMemory(), external: true), cached),

csharp/SQLiteVSDoublets/Links.cs

@@ -1,5 +1,6 @@
+using System.Data;
+using System.Data.Common;
 using System.Numerics;
-using Microsoft.Data.Sqlite;
 using Platform.Data;
 using Platform.Data.Doublets;
 
@@ -24,28 +25,32 @@ public interface ILinksStorage<T> : IDisposable where T : struct
 
 public sealed class SQLiteLinks<T> : SQLiteStorage, ILinksStorage<T> where T : struct, IBinaryInteger<T>
 {
-    private static readonly (string, SqliteType) IdParameter = ("$id", SqliteType.Integer);
-    private static readonly (string, SqliteType) FromParameter = ("$from", SqliteType.Integer);
-    private static readonly (string, SqliteType) ToParameter = ("$to", SqliteType.Integer);
+    private static readonly (string, DbType) IdParameter = ("$id", DbType.Int64);
+    private static readonly (string, DbType) FromParameter = ("$from", DbType.Int64);
+    private static readonly (string, DbType) ToParameter = ("$to", DbType.Int64);
 
-    private readonly SqliteCommand _create;
-    private readonly SqliteCommand _update;
-    private readonly SqliteCommand _delete;
-    private readonly SqliteCommand _get;
-    private readonly SqliteCommand _search;
-    private readonly SqliteCommand _each;
-    private readonly SqliteCommand _eachWithFrom;
-    private readonly SqliteCommand _eachWithTo;
+    private readonly DbCommand _create;
+    private readonly DbCommand _update;
+    private readonly DbCommand _delete;
+    private readonly DbCommand _get;
+    private readonly DbCommand _search;
+    private readonly DbCommand _each;
+    private readonly DbCommand _eachWithFrom;
+    private readonly DbCommand _eachWithTo;
 
-    public static SQLiteLinks<T> Open(string path) => new(path);
+    public static SQLiteLinks<T> Open(string path) => Open(path, SQLiteProvider.MicrosoftDataSqlite);
 
-    public static SQLiteLinks<T> InMemory() => new(":memory:");
+    public static SQLiteLinks<T> Open(string path, SQLiteProvider provider) => new(path, provider);
 
-    private SQLiteLinks(string path) : base(path, "links", """
+    public static SQLiteLinks<T> InMemory() => InMemory(SQLiteProvider.MicrosoftDataSqlite);
+
+    public static SQLiteLinks<T> InMemory(SQLiteProvider provider) => new(":memory:", provider);
+
+    private SQLiteLinks(string path, SQLiteProvider provider) : base(path, "links", """
         CREATE TABLE links (id INTEGER PRIMARY KEY, "from" INTEGER NOT NULL, "to" INTEGER NOT NULL);
         CREATE INDEX links_from_to ON links ("from", "to");
         CREATE INDEX links_to_from ON links ("to", "from");
-        """)
+        """, provider)
     {
         _create = Command("""INSERT INTO links ("from", "to") VALUES ($from, $to)""", FromParameter, ToParameter);
         _update = Command("""UPDATE links SET "from" = $from, "to" = $to WHERE id = $id""", IdParameter, FromParameter, ToParameter);
@@ -59,7 +64,7 @@ private SQLiteLinks(string path) : base(path, "links", """
 
     private static T Id(long sql) => T.CreateChecked(sql);
 
-    private static SqliteCommand With(SqliteCommand command, params ReadOnlySpan<T> values)
+    private static DbCommand With(DbCommand command, params ReadOnlySpan<T> values)
     {
         for (var i = 0; i < values.Length; i++)
         {
@@ -92,7 +97,7 @@ public T Create(T from, T to)
 
     public void EachWithTo(T to, Action<Link<T>> visit) => Read(With(_eachWithTo, to), visit);
 
-    private static void Read(SqliteCommand command, Action<Link<T>> visit)
+    private static void Read(DbCommand command, Action<Link<T>> visit)
     {
         using var reader = command.ExecuteReader();
         while (reader.Read())

csharp/SQLiteVSDoublets/Objects.cs

@@ -1,5 +1,6 @@
+using System.Data;
+using System.Data.Common;
 using System.Numerics;
-using Microsoft.Data.Sqlite;
 using Platform.Collections.Stacks;
 using Platform.Converters;
 using Platform.Data;
@@ -26,27 +27,31 @@ public interface IBlogPostsStorage<T> : IDisposable where T : struct
 
 public sealed class SQLiteBlogPosts<T> : SQLiteStorage, IBlogPostsStorage<T> where T : struct, IBinaryInteger<T>
 {
-    private readonly SqliteCommand _create;
-    private readonly SqliteCommand _get;
-    private readonly SqliteCommand _each;
-    private readonly SqliteCommand _delete;
+    private readonly DbCommand _create;
+    private readonly DbCommand _get;
+    private readonly DbCommand _each;
+    private readonly DbCommand _delete;
 
-    public static SQLiteBlogPosts<T> Open(string path) => new(path);
+    public static SQLiteBlogPosts<T> Open(string path) => Open(path, SQLiteProvider.MicrosoftDataSqlite);
 
-    public static SQLiteBlogPosts<T> InMemory() => new(":memory:");
+    public static SQLiteBlogPosts<T> Open(string path, SQLiteProvider provider) => new(path, provider);
 
-    private SQLiteBlogPosts(string path) : base(path, "blog_posts",
-        "CREATE TABLE blog_posts (id INTEGER PRIMARY KEY, title TEXT NOT NULL, content TEXT NOT NULL, publication_date INTEGER NOT NULL)")
+    public static SQLiteBlogPosts<T> InMemory() => InMemory(SQLiteProvider.MicrosoftDataSqlite);
+
+    public static SQLiteBlogPosts<T> InMemory(SQLiteProvider provider) => new(":memory:", provider);
+
+    private SQLiteBlogPosts(string path, SQLiteProvider provider) : base(path, "blog_posts",
+        "CREATE TABLE blog_posts (id INTEGER PRIMARY KEY, title TEXT NOT NULL, content TEXT NOT NULL, publication_date INTEGER NOT NULL)", provider)
     {
         _create = Command(
             "INSERT INTO blog_posts (title, content, publication_date) VALUES ($title, $content, $publication_date)",
-            ("$title", SqliteType.Text), ("$content", SqliteType.Text), ("$publication_date", SqliteType.Integer));
-        _get = Command("SELECT title, content, publication_date FROM blog_posts WHERE id = $id", ("$id", SqliteType.Integer));
+            ("$title", DbType.String), ("$content", DbType.String), ("$publication_date", DbType.Int64));
+        _get = Command("SELECT title, content, publication_date FROM blog_posts WHERE id = $id", ("$id", DbType.Int64));
         _each = Command("SELECT title, content, publication_date, id FROM blog_posts");
-        _delete = Command("DELETE FROM blog_posts WHERE id = $id", ("$id", SqliteType.Integer));
+        _delete = Command("DELETE FROM blog_posts WHERE id = $id", ("$id", DbType.Int64));
     }
 
-    private static BlogPost BlogPost(SqliteDataReader reader) =>
+    private static BlogPost BlogPost(DbDataReader reader) =>
         new(reader.GetString(0), reader.GetString(1), (ulong)reader.GetInt64(2));
 
     public T Create(BlogPost post)

csharp/SQLiteVSDoublets/Program.cs

@@ -7,7 +7,6 @@
 using System.Text.Json;
 using System.Text.Json.Nodes;
 using Comparisons.SQLiteVSDoublets;
-using Microsoft.Data.Sqlite;
 
 const ulong LinksWork = 3_000_000;
 const ulong ObjectsWork = 500_000;
@@ -37,10 +36,6 @@
     _ => throw new ArgumentException("category must be links or objects and bits must be 32 or 64"),
 }).ToList();
 
-using var connection = new SqliteConnection("Data Source=:memory:");
-connection.Open();
-using var version = connection.CreateCommand();
-version.CommandText = "SELECT sqlite_version()";
 var report = new JsonObject
 {
     ["language"] = "C#",
@@ -50,7 +45,20 @@
     ["repetitions"] = repetitions,
     ["warm_up_size"] = Math.Min(size, Harness.WarmUpSize),
     ["warm_up_seconds"] = Harness.WarmUpTime.TotalSeconds,
-    ["sqlite_version"] = (string)version.ExecuteScalar()!,
+    ["sqlite_version"] = SQLiteStorage.Version(SQLiteProvider.MicrosoftDataSqlite),
+    ["sqlite_providers"] = new JsonObject
+    {
+        ["Microsoft.Data.Sqlite"] = new JsonObject
+        {
+            ["provider_version"] = typeof(Microsoft.Data.Sqlite.SqliteConnection).Assembly.GetName().Version!.ToString(),
+            ["sqlite_version"] = SQLiteStorage.Version(SQLiteProvider.MicrosoftDataSqlite),
+        },
+        ["System.Data.SQLite"] = new JsonObject
+        {
+            ["provider_version"] = typeof(System.Data.SQLite.SQLiteConnection).Assembly.GetName().Version!.ToString(),
+            ["sqlite_version"] = SQLiteStorage.Version(SQLiteProvider.SystemDataSQLite),
+        },
+    },
     ["results"] = new JsonArray(measurements.Select(measurement => (JsonNode)measurement.ToJson()).ToArray()),
 };
 var json = report.ToJsonString(new JsonSerializerOptions { WriteIndented = true });