From 0a2b9b294268c59ae50e6bf07e4d378de7c8965a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:57:27 +0000 Subject: [PATCH 01/85] Bump the analyzers group with 2 updates Bumps Meziantou.Analyzer from 3.0.123 to 3.0.137 Bumps SonarAnalyzer.CSharp from 10.29.0.143774 to 10.31.0.145097 --- updated-dependencies: - dependency-name: Meziantou.Analyzer dependency-version: 3.0.137 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: analyzers - dependency-name: SonarAnalyzer.CSharp dependency-version: 10.31.0.145097 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: analyzers ... Signed-off-by: dependabot[bot] --- Directory.Build.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 8397e4a..edf140d 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -19,11 +19,11 @@ $(DefineConstants.Replace("TRACE;", "")) - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all From f5d11646092017a7947ae0cfceb5cc291176b37c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:59:43 +0000 Subject: [PATCH 02/85] Bump Microsoft.Testing.Extensions.TrxReport from 2.3.2 to 2.3.3 --- updated-dependencies: - dependency-name: Microsoft.Testing.Extensions.TrxReport dependency-version: 2.3.3 dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: Microsoft.Testing.Extensions.TrxReport dependency-version: 2.3.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .../FixedWidthParser.Generator.Tests.csproj | 2 +- tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj b/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj index 5a19001..1c1eba6 100644 --- a/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj +++ b/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj @@ -14,7 +14,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj index d090f90..486945c 100644 --- a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj +++ b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj @@ -22,7 +22,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all From fe496cdd9e16634674dee35de15ebe767424720c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:04:25 +0000 Subject: [PATCH 03/85] Bump CsCheck from 4.7.0 to 4.9.1 --- updated-dependencies: - dependency-name: CsCheck dependency-version: 4.9.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj index 486945c..cb033b0 100644 --- a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj +++ b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj @@ -19,7 +19,7 @@ - + From 45a74ef26e96a890680570c1b673f1256acc8129 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:07:59 +0000 Subject: [PATCH 04/85] Bump Microsoft.Testing.Extensions.CodeCoverage from 18.9.0 to 18.12.0 --- updated-dependencies: - dependency-name: Microsoft.Testing.Extensions.CodeCoverage dependency-version: 18.12.0 dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: Microsoft.Testing.Extensions.CodeCoverage dependency-version: 18.12.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .../FixedWidthParser.Generator.Tests.csproj | 2 +- tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj b/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj index 1c1eba6..094ab7c 100644 --- a/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj +++ b/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj @@ -13,7 +13,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj index 486945c..1cd3afd 100644 --- a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj +++ b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj @@ -21,7 +21,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive From 70b58f809810a6677840eef92faf14c978ef87e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:05:30 +0000 Subject: [PATCH 05/85] Bump the test group with 2 updates Bumps xunit.runner.visualstudio from 3.1.5 to 4.0.1 Bumps xunit.v3.mtp-v2 from 3.2.2 to 4.0.2 --- updated-dependencies: - dependency-name: xunit.runner.visualstudio dependency-version: 4.0.1 dependency-type: direct:production update-type: version-update:semver-major dependency-group: test - dependency-name: xunit.runner.visualstudio dependency-version: 4.0.1 dependency-type: direct:production update-type: version-update:semver-major dependency-group: test - dependency-name: xunit.v3.mtp-v2 dependency-version: 4.0.2 dependency-type: direct:production update-type: version-update:semver-major dependency-group: test - dependency-name: xunit.v3.mtp-v2 dependency-version: 4.0.2 dependency-type: direct:production update-type: version-update:semver-major dependency-group: test ... Signed-off-by: dependabot[bot] --- .../FixedWidthParser.Generator.Tests.csproj | 4 ++-- tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj b/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj index 1c1eba6..582e960 100644 --- a/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj +++ b/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj @@ -15,11 +15,11 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + diff --git a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj index 486945c..2841229 100644 --- a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj +++ b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj @@ -23,11 +23,11 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + From 17c0baac9e5fe4e0700817f03ffb9b73508f8b57 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:14:41 +0000 Subject: [PATCH 06/85] Bump System.IO.Pipelines from 10.0.10 to 10.0.13 --- updated-dependencies: - dependency-name: System.IO.Pipelines dependency-version: 10.0.13 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- src/FixedWidthParser/FixedWidthParser.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FixedWidthParser/FixedWidthParser.csproj b/src/FixedWidthParser/FixedWidthParser.csproj index 6669b4a..d63184e 100644 --- a/src/FixedWidthParser/FixedWidthParser.csproj +++ b/src/FixedWidthParser/FixedWidthParser.csproj @@ -49,7 +49,7 @@ - + Date: Thu, 15 Oct 2026 14:38:06 -0300 Subject: [PATCH 07/85] feat!: read and write bytes only, configured through options 2.0 keeps one parse path. The char parser, FixedWidthUtf8, FixedWidthByteReader, Utf8FixedWidthParser, IUtf8FixedWidthModel and IUtf8FixedWidthConverter are gone: there is one parser, one reader, one writer and one generated TryParse/TryFormat, all over bytes. Column positions are bytes; text columns, dates parsed with a format and types that only parse from characters are decoded with an ASCII-compatible encoding, UTF-8 by default. The writer formats straight to the stream, batched into one pooled buffer, and never splits a multi-byte character when it truncates. The package targets net10.0 only. Reads take FixedWidthReaderOptions (format provider, encoding, string pool, buffer size, MaxLineLength, OnParseError) and the writer takes FixedWidthWriterOptions (format provider, encoding, new line). Generated code and IFixedWidthConverter receive a FixedWidthContext by in reference, so a later setting arrives as a property instead of a new parameter. A line that fails to parse throws FixedWidthParseException, a FormatException that names the line, the column, its position and its raw text, or says how short the line is. The reader learns the column by parsing the failed line again with a failure sink, so lines that parse pay nothing. OnParseError can skip the line instead. MaxLineLength bounds the line buffer on stream, async and pipe reads. A source without line breaks used to double the buffer until memory ran out. The generator targets netstandard2.0, reports FWP012 for a non-ASCII Padding or TrimChar, and its tests now compile the generated code. The library tracks its public API (PublicAPI.*.txt), validates the package against 1.1.1 with the deliberate breaks listed in CompatibilitySuppressions.xml, and builds with warnings as errors. --- .github/workflows/benchmark-pages.yml | 4 +- .github/workflows/benchmark-pr.yml | 4 +- .../FixedWidthModelGenerator.cs | 772 +++++++----------- .../FixedWidthParser.Generator.csproj | 4 +- .../CompatibilitySuppressions.xml | 505 ++++++++++++ src/FixedWidthParser/Delegates.cs | 11 +- src/FixedWidthParser/FixedWidth.cs | 126 +-- src/FixedWidthParser/FixedWidthContext.cs | 65 ++ .../FixedWidthParseException.cs | 60 ++ src/FixedWidthParser/FixedWidthParser.csproj | 31 +- .../FixedWidthReaderOptions.cs | 62 ++ src/FixedWidthParser/FixedWidthRuntime.cs | 282 +++++-- src/FixedWidthParser/FixedWidthUtf8.cs | 109 --- .../FixedWidthWriterOptions.cs | 21 + .../Formatters/ColumnFormatOptions.cs | 20 +- .../Formatters/ColumnFormatters.cs | 59 ++ .../Formatters/ConverterColumnFormatter.cs | 16 - .../Formatters/IColumnFormatter.cs | 7 - .../Formatters/NullableColumnFormatter.cs | 26 - .../SpanFormattableColumnFormatter.cs | 14 - .../Formatters/StringColumnFormatter.cs | 14 - src/FixedWidthParser/IFixedWidthModel.cs | 36 +- src/FixedWidthParser/IUtf8FixedWidthModel.cs | 30 - .../Parsers/FixedWidthParser.cs | 36 +- src/FixedWidthParser/Parsers/ParserBuilder.cs | 40 +- .../Parsers/Utf8FixedWidthParser.cs | 81 -- .../Processors/ColumnParser.cs | 19 +- .../Processors/ColumnParserFactory.cs | 272 +++--- .../Processors/ColumnParserFactoryShared.cs | 74 -- .../Processors/CultureHelpers.cs | 193 +---- .../Processors/IFixedWidthConverter.cs | 27 +- .../Processors/Utf8ColumnParser.cs | 26 - .../Processors/Utf8ColumnParserFactory.cs | 219 ----- src/FixedWidthParser/PublicAPI.Shipped.txt | 1 + src/FixedWidthParser/PublicAPI.Unshipped.txt | 173 ++++ .../Readers/AsyncRecordEnumerator.cs | 95 +++ .../Readers/AsyncRecordEnumeratorCore.cs | 126 --- .../FixedWidthAsyncRecordEnumerable.cs | 52 +- .../Readers/FixedWidthByteReader.cs | 101 --- .../Readers/FixedWidthPipeRecordEnumerable.cs | 37 + .../Readers/FixedWidthReader.cs | 85 +- .../Readers/FixedWidthRecordEnumerable.cs | 51 +- ...ratedUtf8FixedWidthPipeRecordEnumerable.cs | 45 - src/FixedWidthParser/Readers/ILineParser.cs | 57 +- src/FixedWidthParser/Readers/ISource.cs | 18 - .../Readers/IUtf8LineParser.cs | 47 -- .../Readers/LineBufferState.cs | 131 +-- src/FixedWidthParser/Readers/ParseFailure.cs | 78 ++ ...meratorCore.cs => PipeRecordEnumerator.cs} | 119 ++- .../Readers/RecordEnumerator.cs | 93 +++ .../Readers/RecordEnumeratorCore.cs | 107 --- src/FixedWidthParser/Readers/StreamSource.cs | 29 - .../Readers/TextReaderSource.cs | 101 --- .../Utf8FixedWidthAsyncRecordEnumerable.cs | 73 -- .../Utf8FixedWidthPipeRecordEnumerable.cs | 48 -- .../Readers/Utf8FixedWidthRecordEnumerable.cs | 82 -- src/FixedWidthParser/Utf8FixedWidthRuntime.cs | 127 --- .../Writers/FixedWidthWriter.cs | 367 ++++----- .../Benchmarks/Perf/AsyncReaderBenchmarks.cs | 55 +- tests/Benchmarks/Perf/ByteReaderBenchmarks.cs | 108 --- .../Perf/ByteReaderPoolingBenchmarks.cs | 101 --- .../Perf/ByteReaderStreamBenchmarks.cs | 74 -- tests/Benchmarks/Perf/ComparisonBenchmarks.cs | 28 +- .../Perf/ComparisonWriterBenchmarks.cs | 7 +- tests/Benchmarks/Perf/ParserBenchmarks.cs | 14 +- tests/Benchmarks/Perf/PipeReaderBenchmarks.cs | 54 +- tests/Benchmarks/Perf/ReaderBenchmarks.cs | 56 +- .../Perf/ReaderPoolingBenchmarks.cs | 72 ++ .../Perf/ReaderStartupBenchmarks.cs | 11 +- .../Perf/SourceGenParserBenchmarks.cs | 14 +- tests/Benchmarks/Perf/WriterBenchmarks.cs | 77 +- tests/Benchmarks/SampleData.cs | 25 + .../FixedWidthModelGeneratorTests.cs | 167 ++-- .../FixedWidthParser.Generator.Tests.csproj | 2 +- .../AllocationGuardTests.cs | 157 +--- .../ByteReaderStreamTests.cs | 342 -------- .../FixedWidthParser.Tests/ByteReaderTests.cs | 373 --------- .../ColumnValidationTests.cs | 20 +- .../ConverterAttributeTests.cs | 60 +- .../CorrectnessTests.cs | 164 +--- tests/FixedWidthParser.Tests/CoverageTests.cs | 93 +-- tests/FixedWidthParser.Tests/CultureTests.cs | 216 ++--- .../FixedWidthParser.Tests.csproj | 5 +- .../FixedWidthParser.Tests/GeneratedModels.cs | 80 +- .../GeneratedReaderTests.cs | 254 ++---- .../GeneratedUtf8ReaderTests.cs | 328 -------- .../GeneratedWriterTests.cs | 138 ++-- .../NullableColumnTests.cs | 165 +--- .../ParserConstructionTests.cs | 61 -- tests/FixedWidthParser.Tests/ParserTests.cs | 58 +- .../FixedWidthParser.Tests/PipeReaderTests.cs | 40 +- .../PropertyRoundTripTests.cs | 30 +- .../ReaderOptionsTests.cs | 197 +++++ .../ReaderStreamTests.cs | 136 --- tests/FixedWidthParser.Tests/ReaderTests.cs | 228 ++++-- .../FixedWidthParser.Tests/RefStructTests.cs | 27 - .../FixedWidthParser.Tests/RoundTripTests.cs | 4 +- .../SourceGenParityTests.cs | 136 ++- tests/FixedWidthParser.Tests/TestHelpers.cs | 37 +- tests/FixedWidthParser.Tests/TestModels.cs | 120 ++- tests/FixedWidthParser.Tests/TrimCharTests.cs | 113 +-- .../Utf8SourceGenParityTests.cs | 164 ---- .../WriterAsyncTests.cs | 61 +- .../WriterEncodingTests.cs | 96 +++ 104 files changed, 3857 insertions(+), 6419 deletions(-) create mode 100644 src/FixedWidthParser/CompatibilitySuppressions.xml create mode 100644 src/FixedWidthParser/FixedWidthContext.cs create mode 100644 src/FixedWidthParser/FixedWidthParseException.cs create mode 100644 src/FixedWidthParser/FixedWidthReaderOptions.cs delete mode 100644 src/FixedWidthParser/FixedWidthUtf8.cs create mode 100644 src/FixedWidthParser/FixedWidthWriterOptions.cs create mode 100644 src/FixedWidthParser/Formatters/ColumnFormatters.cs delete mode 100644 src/FixedWidthParser/Formatters/ConverterColumnFormatter.cs delete mode 100644 src/FixedWidthParser/Formatters/IColumnFormatter.cs delete mode 100644 src/FixedWidthParser/Formatters/NullableColumnFormatter.cs delete mode 100644 src/FixedWidthParser/Formatters/SpanFormattableColumnFormatter.cs delete mode 100644 src/FixedWidthParser/Formatters/StringColumnFormatter.cs delete mode 100644 src/FixedWidthParser/IUtf8FixedWidthModel.cs delete mode 100644 src/FixedWidthParser/Parsers/Utf8FixedWidthParser.cs delete mode 100644 src/FixedWidthParser/Processors/ColumnParserFactoryShared.cs delete mode 100644 src/FixedWidthParser/Processors/Utf8ColumnParser.cs delete mode 100644 src/FixedWidthParser/Processors/Utf8ColumnParserFactory.cs create mode 100644 src/FixedWidthParser/PublicAPI.Shipped.txt create mode 100644 src/FixedWidthParser/PublicAPI.Unshipped.txt create mode 100644 src/FixedWidthParser/Readers/AsyncRecordEnumerator.cs delete mode 100644 src/FixedWidthParser/Readers/AsyncRecordEnumeratorCore.cs delete mode 100644 src/FixedWidthParser/Readers/FixedWidthByteReader.cs create mode 100644 src/FixedWidthParser/Readers/FixedWidthPipeRecordEnumerable.cs delete mode 100644 src/FixedWidthParser/Readers/GeneratedUtf8FixedWidthPipeRecordEnumerable.cs delete mode 100644 src/FixedWidthParser/Readers/ISource.cs delete mode 100644 src/FixedWidthParser/Readers/IUtf8LineParser.cs create mode 100644 src/FixedWidthParser/Readers/ParseFailure.cs rename src/FixedWidthParser/Readers/{Utf8PipeRecordEnumeratorCore.cs => PipeRecordEnumerator.cs} (64%) create mode 100644 src/FixedWidthParser/Readers/RecordEnumerator.cs delete mode 100644 src/FixedWidthParser/Readers/RecordEnumeratorCore.cs delete mode 100644 src/FixedWidthParser/Readers/StreamSource.cs delete mode 100644 src/FixedWidthParser/Readers/TextReaderSource.cs delete mode 100644 src/FixedWidthParser/Readers/Utf8FixedWidthAsyncRecordEnumerable.cs delete mode 100644 src/FixedWidthParser/Readers/Utf8FixedWidthPipeRecordEnumerable.cs delete mode 100644 src/FixedWidthParser/Readers/Utf8FixedWidthRecordEnumerable.cs delete mode 100644 src/FixedWidthParser/Utf8FixedWidthRuntime.cs delete mode 100644 tests/Benchmarks/Perf/ByteReaderBenchmarks.cs delete mode 100644 tests/Benchmarks/Perf/ByteReaderPoolingBenchmarks.cs delete mode 100644 tests/Benchmarks/Perf/ByteReaderStreamBenchmarks.cs create mode 100644 tests/Benchmarks/Perf/ReaderPoolingBenchmarks.cs create mode 100644 tests/Benchmarks/SampleData.cs delete mode 100644 tests/FixedWidthParser.Tests/ByteReaderStreamTests.cs delete mode 100644 tests/FixedWidthParser.Tests/ByteReaderTests.cs delete mode 100644 tests/FixedWidthParser.Tests/GeneratedUtf8ReaderTests.cs delete mode 100644 tests/FixedWidthParser.Tests/ParserConstructionTests.cs create mode 100644 tests/FixedWidthParser.Tests/ReaderOptionsTests.cs delete mode 100644 tests/FixedWidthParser.Tests/ReaderStreamTests.cs delete mode 100644 tests/FixedWidthParser.Tests/RefStructTests.cs delete mode 100644 tests/FixedWidthParser.Tests/Utf8SourceGenParityTests.cs create mode 100644 tests/FixedWidthParser.Tests/WriterEncodingTests.cs diff --git a/.github/workflows/benchmark-pages.yml b/.github/workflows/benchmark-pages.yml index a04099e..0e96c30 100644 --- a/.github/workflows/benchmark-pages.yml +++ b/.github/workflows/benchmark-pages.yml @@ -41,9 +41,7 @@ jobs: Benchmarks.Perf.AsyncReaderBenchmarks.* - name: byte filter: >- - Benchmarks.Perf.ByteReaderBenchmarks.* - Benchmarks.Perf.ByteReaderPoolingBenchmarks.* - Benchmarks.Perf.ByteReaderStreamBenchmarks.* + Benchmarks.Perf.ReaderPoolingBenchmarks.* Benchmarks.Perf.PipeReaderBenchmarks.* - name: writer filter: Benchmarks.Perf.WriterBenchmarks.* diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index 04835fe..4492318 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -39,9 +39,7 @@ jobs: Benchmarks.Perf.AsyncReaderBenchmarks.* - name: byte filter: >- - Benchmarks.Perf.ByteReaderBenchmarks.* - Benchmarks.Perf.ByteReaderPoolingBenchmarks.* - Benchmarks.Perf.ByteReaderStreamBenchmarks.* + Benchmarks.Perf.ReaderPoolingBenchmarks.* Benchmarks.Perf.PipeReaderBenchmarks.* - name: writer filter: Benchmarks.Perf.WriterBenchmarks.* diff --git a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs index d6c14dd..efb79db 100644 --- a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs +++ b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs @@ -10,18 +10,18 @@ namespace FixedWidthParser.Generator { /// - /// Emits a reflection-free TryParse for every type that declares - /// FixedWidthParser.IFixedWidthModel<TSelf>, mirroring the runtime parser's semantics. + /// Emits a reflection-free TryParse and TryFormat over bytes for every + /// type that declares FixedWidthParser.IFixedWidthModel<TSelf>, + /// mirroring the runtime parser's and writer's semantics. /// [Generator(LanguageNames.CSharp)] public sealed class FixedWidthModelGenerator : IIncrementalGenerator { private const string MarkerMetadataName = "FixedWidthParser.IFixedWidthModel`1"; - private const string Utf8MarkerMetadataName = "FixedWidthParser.IUtf8FixedWidthModel`1"; private const string ColumnAttributeMetadataName = "FixedWidthParser.Attributes.FixedColumnAttribute"; private const string ConverterInterfaceMetadataName = "FixedWidthParser.Processors.IFixedWidthConverter`1"; - private const string Utf8ConverterInterfaceMetadataName = "FixedWidthParser.Processors.IUtf8FixedWidthConverter`1"; private const string SpanFormattableMetadataName = "System.ISpanFormattable"; + private const string RuntimeFqn = "global::FixedWidthParser.FixedWidthRuntime"; private static readonly DiagnosticDescriptor MustBePartial = new( "FWP001", "Fixed-width model must be partial", @@ -35,12 +35,7 @@ public sealed class FixedWidthModelGenerator : IIncrementalGenerator private static readonly DiagnosticDescriptor UnsupportedColumnType = new( "FWP003", "Unsupported fixed-width column type", - "Column '{0}' has type '{1}', which is not string, double, float or ISpanParsable; generated parsing cannot handle it", - "FixedWidthParser", DiagnosticSeverity.Error, isEnabledByDefault: true); - - private static readonly DiagnosticDescriptor UnsupportedUtf8ColumnType = new( - "FWP007", "Unsupported UTF-8 fixed-width column type", - "Column '{0}' has type '{1}', which is not string, double, float or IUtf8SpanParsable; generated UTF-8 parsing cannot handle it", + "Column '{0}' has type '{1}', which is not string, double, float, IUtf8SpanParsable or ISpanParsable; set a Converter for it", "FixedWidthParser", DiagnosticSeverity.Error, isEnabledByDefault: true); private static readonly DiagnosticDescriptor NegativeStart = new( @@ -63,14 +58,9 @@ public sealed class FixedWidthModelGenerator : IIncrementalGenerator "Converter '{0}' for column '{1}' does not implement IFixedWidthConverter<{2}>", "FixedWidthParser", DiagnosticSeverity.Error, isEnabledByDefault: true); - private static readonly DiagnosticDescriptor UnsupportedUtf8ConverterType = new( - "FWP009", "Converter does not implement IUtf8FixedWidthConverter", - "Converter '{0}' for column '{1}' does not implement IUtf8FixedWidthConverter<{2}>", - "FixedWidthParser", DiagnosticSeverity.Error, isEnabledByDefault: true); - private static readonly DiagnosticDescriptor UnsupportedWriteColumnType = new( "FWP010", "Unsupported fixed-width column type for writing", - "Column '{0}' has type '{1}', which is not string or ISpanFormattable; generated writing cannot handle it", + "Column '{0}' has type '{1}', which is not string or ISpanFormattable; set a Converter for it", "FixedWidthParser", DiagnosticSeverity.Error, isEnabledByDefault: true); private static readonly DiagnosticDescriptor InvalidOverflowBehavior = new( @@ -78,6 +68,11 @@ public sealed class FixedWidthModelGenerator : IIncrementalGenerator "Column '{0}' in '{1}' has type '{2}' and specifies OverflowBehavior.Truncate, which is only supported for string columns", "FixedWidthParser", DiagnosticSeverity.Error, isEnabledByDefault: true); + private static readonly DiagnosticDescriptor NonAsciiCharacter = new( + "FWP012", "Padding and TrimChar must be ASCII", + "Column '{0}' in '{1}' sets {2} to '{3}', which is not an ASCII character; column positions are bytes, so it must be one byte in every supported encoding", + "FixedWidthParser", DiagnosticSeverity.Error, isEnabledByDefault: true); + public void Initialize(IncrementalGeneratorInitializationContext context) { var models = context.SyntaxProvider.CreateSyntaxProvider( @@ -87,6 +82,7 @@ public void Initialize(IncrementalGeneratorInitializationContext context) context.RegisterSourceOutput(models, static (spc, model) => Emit(spc, model!)); } + private static ModelInfo? Extract(GeneratorSyntaxContext ctx, System.Threading.CancellationToken ct) { var declaration = (TypeDeclarationSyntax)ctx.Node; @@ -98,24 +94,12 @@ public void Initialize(IncrementalGeneratorInitializationContext context) var compilation = ctx.SemanticModel.Compilation; var marker = compilation.GetTypeByMetadataName(MarkerMetadataName); - var utf8Marker = compilation.GetTypeByMetadataName(Utf8MarkerMetadataName); var columnAttribute = compilation.GetTypeByMetadataName(ColumnAttributeMetadataName); - if (columnAttribute is null || (marker is null && utf8Marker is null)) + if (columnAttribute is null || marker is null || !ImplementsMarker(symbol, marker)) { return null; } - // Implements IFixedWidthModel (char) and/or IUtf8FixedWidthModel (byte)? - bool implementsChar = ImplementsMarker(symbol, marker); - bool implementsUtf8 = ImplementsMarker(symbol, utf8Marker); - if (!implementsChar && !implementsUtf8) - { - return null; - } - - bool isPartial = declaration.Modifiers.Any(m => m.IsKind(SyntaxKind.PartialKeyword)); - bool isNested = symbol.ContainingType is not null; - var columns = ImmutableArray.CreateBuilder(); foreach (var member in symbol.GetMembers()) { @@ -180,15 +164,21 @@ public void Initialize(IncrementalGeneratorInitializationContext context) } } - var (kind, parsableTypeFqn, converterFqn, isNullable, isCharParsable, isUtf8Parsable) = Classify(memberType, converterType, format, compilation); - var (writeKind, isCharFormattable) = ClassifyWrite(memberType, converterType, compilation); - // Mirrors FixedWidthWriter.DetermineOverflowBehavior: an explicit Overflow wins; otherwise - // string truncates and everything else throws. - int resolvedOverflow = overflow != 0 ? overflow : (writeKind == WriteKind.String ? 1 : 2); + var underlying = UnwrapNullable(memberType); + bool isNullable = !SymbolEqualityComparer.Default.Equals(underlying, memberType); + string? converterFqn = converterType?.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); + var kind = Classify(underlying, converterType, format, compilation); + var writeKind = ClassifyWrite(underlying, converterType, compilation); + // Mirrors FixedWidthWriter: an explicit Overflow wins; otherwise string truncates and everything else throws. + int resolvedOverflow = overflow; + if (resolvedOverflow == 0) + { + resolvedOverflow = writeKind == WriteKind.String ? 1 : 2; + } columns.Add(new ColumnInfo( - member.Name, start, length, kind, parsableTypeFqn, converterFqn, isNullable, isCharParsable, isUtf8Parsable, - writeKind, isCharFormattable, alignment, padding, format, resolvedOverflow, trimChar, trimMode)); + member.Name, start, length, kind, underlying.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), converterFqn, + isNullable, writeKind, alignment, padding, format, resolvedOverflow, trimChar, trimMode)); } string? ns = symbol.ContainingNamespace.IsGlobalNamespace @@ -201,102 +191,69 @@ public void Initialize(IncrementalGeneratorInitializationContext context) symbol.Name, symbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), new EquatableArray(columns.ToImmutable()), - isPartial, - isNested, - implementsChar, - implementsUtf8, + declaration.Modifiers.Any(m => m.IsKind(SyntaxKind.PartialKeyword)), + symbol.ContainingType is not null, declaration.Identifier.GetLocation()); } - // Implements I…Model for the given (open) marker interface? - private static bool ImplementsMarker(INamedTypeSymbol symbol, INamedTypeSymbol? marker) + // Implements IFixedWidthModel? + private static bool ImplementsMarker(INamedTypeSymbol symbol, INamedTypeSymbol marker) { - return marker is not null && symbol.AllInterfaces.Any(i => + return symbol.AllInterfaces.Any(i => SymbolEqualityComparer.Default.Equals(i.OriginalDefinition, marker) && i.TypeArguments.Length == 1 && SymbolEqualityComparer.Default.Equals(i.TypeArguments[0], symbol)); } - private static (ColumnKind Kind, string TypeFqn, string? ConverterFqn, bool IsNullable, bool IsCharParsable, bool IsUtf8Parsable) Classify( - ITypeSymbol type, ITypeSymbol? converterType, string? format, Compilation compilation) + // Mirrors ColumnParserFactory: converter → date/time with Format → string → double/float → + // IUtf8SpanParsable → ISpanParsable over the decoded text. + private static ColumnKind Classify(ITypeSymbol type, ITypeSymbol? converterType, string? format, Compilation compilation) { - // T? (Nullable): classify the underlying T (including any converter, which targets T, - // not T?) and flag IsNullable so Emit/codegen wrap it with the blank-is-null check. - if (type is INamedTypeSymbol { OriginalDefinition.SpecialType: SpecialType.System_Nullable_T } named) + if (converterType is not null) { - var underlying = Classify(named.TypeArguments[0], converterType, format, compilation); - return (underlying.Kind, underlying.TypeFqn, underlying.ConverterFqn, true, underlying.IsCharParsable, underlying.IsUtf8Parsable); + return ImplementsGeneric(converterType, type, compilation, ConverterInterfaceMetadataName) ? ColumnKind.Converter : ColumnKind.BadConverter; } string fqn = type.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); - - // FixedColumnAttribute.Converter takes priority over the built-in primitive/parsable - // handling, for both string and non-string column types. - if (converterType is not null) + if (format is not null && ExactHelper(fqn) is not null) { - string converterFqn = converterType.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); - bool convertsChar = ImplementsConverter(converterType, type, compilation, ConverterInterfaceMetadataName); - bool convertsUtf8 = ImplementsConverter(converterType, type, compilation, Utf8ConverterInterfaceMetadataName); - var convKind = convertsChar || convertsUtf8 ? ColumnKind.Converter : ColumnKind.Unsupported; - return (convKind, fqn, converterFqn, false, convertsChar, convertsUtf8); + return ColumnKind.Exact; } switch (type.SpecialType) { case SpecialType.System_String: - return (ColumnKind.String, "string", null, false, true, true); + return ColumnKind.String; case SpecialType.System_Double: - return (ColumnKind.Double, "double", null, false, true, true); + return ColumnKind.Double; case SpecialType.System_Single: - return (ColumnKind.Float, "float", null, false, true, true); + return ColumnKind.Float; default: break; } - bool isCharParsable = ImplementsParsable(type, compilation, "System.ISpanParsable`1"); - bool isUtf8Parsable = ImplementsParsable(type, compilation, "System.IUtf8SpanParsable`1"); - - bool isDateTimeType = IsDateTimeType(fqn); - if (isDateTimeType && format is not null) + if (ImplementsGeneric(type, type, compilation, "System.IUtf8SpanParsable`1")) { - isCharParsable = true; - isUtf8Parsable = true; + return ColumnKind.Utf8Parsable; } - - // SpanParsable when at least one path can handle it; otherwise unsupported by every path. - var kind = isCharParsable || isUtf8Parsable ? ColumnKind.SpanParsable : ColumnKind.Unsupported; - return (kind, fqn, null, false, isCharParsable, isUtf8Parsable); - } - - private static bool IsDateTimeType(string fqn) - { - return fqn is "global::System.DateTime" or "System.DateTime" - or "global::System.DateOnly" or "System.DateOnly" - or "global::System.TimeOnly" or "System.TimeOnly" - or "global::System.DateTimeOffset" or "System.DateTimeOffset"; + return ImplementsGeneric(type, type, compilation, "System.ISpanParsable`1") ? ColumnKind.DecodedParsable : ColumnKind.Unsupported; } - // Classification for the write side: mirrors FixedWidthWriter.CreateFormatter's own resolution - // (converter → string → ISpanFormattable fallback), but write has no "primitive" concept — - // double/float are ISpanFormattable like everything else, no special-casing. - private static (WriteKind Kind, bool IsCharFormattable) ClassifyWrite(ITypeSymbol type, ITypeSymbol? converterType, Compilation compilation) + // Mirrors FixedWidthWriter: converter → string → ISpanFormattable. + private static WriteKind ClassifyWrite(ITypeSymbol type, ITypeSymbol? converterType, Compilation compilation) { - var underlying = UnwrapNullable(type); - if (converterType is not null) { - bool convertsChar = ImplementsConverter(converterType, underlying, compilation, ConverterInterfaceMetadataName); - bool convertsUtf8 = ImplementsConverter(converterType, underlying, compilation, Utf8ConverterInterfaceMetadataName); - return (convertsChar || convertsUtf8 ? WriteKind.Converter : WriteKind.Unsupported, false); + return ImplementsGeneric(converterType, type, compilation, ConverterInterfaceMetadataName) ? WriteKind.Converter : WriteKind.Unsupported; } - - if (underlying.SpecialType == SpecialType.System_String) + if (type.SpecialType == SpecialType.System_String) { - return (WriteKind.String, true); + return WriteKind.String; } - - bool isCharFormattable = ImplementsInterface(underlying, compilation, SpanFormattableMetadataName); - return (isCharFormattable ? WriteKind.Formattable : WriteKind.Unsupported, isCharFormattable); + var formattable = compilation.GetTypeByMetadataName(SpanFormattableMetadataName); + return formattable is not null && type.AllInterfaces.Any(i => SymbolEqualityComparer.Default.Equals(i, formattable)) + ? WriteKind.Formattable + : WriteKind.Unsupported; } private static ITypeSymbol UnwrapNullable(ITypeSymbol type) @@ -306,31 +263,26 @@ private static ITypeSymbol UnwrapNullable(ITypeSymbol type) : type; } - // Implements the given (non-generic) BCL interface metadata name? - private static bool ImplementsInterface(ITypeSymbol type, Compilation compilation, string interfaceMetadataName) + // Does `type` implement the open interface `interfaceMetadataName` closed over `argument`? + private static bool ImplementsGeneric(ITypeSymbol type, ITypeSymbol argument, Compilation compilation, string interfaceMetadataName) { var iface = compilation.GetTypeByMetadataName(interfaceMetadataName); - return iface is not null && type.AllInterfaces.Any(i => SymbolEqualityComparer.Default.Equals(i, iface)); - } - - // Implements I…SpanParsable for the given (open) BCL interface metadata name? - private static bool ImplementsParsable(ITypeSymbol type, Compilation compilation, string interfaceMetadataName) - { - var parsable = compilation.GetTypeByMetadataName(interfaceMetadataName); - return parsable is not null && type.AllInterfaces.Any(i => - SymbolEqualityComparer.Default.Equals(i.OriginalDefinition, parsable) + return iface is not null && type.AllInterfaces.Any(i => + SymbolEqualityComparer.Default.Equals(i.OriginalDefinition, iface) && i.TypeArguments.Length == 1 - && SymbolEqualityComparer.Default.Equals(i.TypeArguments[0], type)); + && SymbolEqualityComparer.Default.Equals(i.TypeArguments[0], argument)); } - // Implements I…Converter (the given open interface) for the column's converter type? - private static bool ImplementsConverter(ITypeSymbol converterType, ITypeSymbol columnType, Compilation compilation, string interfaceMetadataName) + private static string? ExactHelper(string fqn) { - var iface = compilation.GetTypeByMetadataName(interfaceMetadataName); - return iface is not null && converterType.AllInterfaces.Any(i => - SymbolEqualityComparer.Default.Equals(i.OriginalDefinition, iface) - && i.TypeArguments.Length == 1 - && SymbolEqualityComparer.Default.Equals(i.TypeArguments[0], columnType)); + return fqn switch + { + "global::System.DateTime" => "TryDateTimeExact", + "global::System.DateOnly" => "TryDateOnlyExact", + "global::System.TimeOnly" => "TryTimeOnlyExact", + "global::System.DateTimeOffset" => "TryDateTimeOffsetExact", + _ => null, + }; } private static string BuildTypeKeywords(INamedTypeSymbol symbol) @@ -369,122 +321,103 @@ private static void Emit(SourceProductionContext spc, ModelInfo model) return; } - // Structural errors (geometry) block ALL generated methods (parse char/byte + write). - // Per-column-type errors block only the affected path: FWP003 the char parse method, - // FWP007 the byte parse method, FWP010/FWP008 the write method. - bool structuralError = false; - bool charTypeError = false; - bool utf8TypeError = false; - bool writeTypeError = false; + // The interface needs both methods, so any column error blocks the whole emission. + bool error = false; var columns = model.Columns.AsImmutableArray(); foreach (var column in columns) { - bool isPrimitive = column.Kind is ColumnKind.String or ColumnKind.Double or ColumnKind.Float; - if (!isPrimitive) - { - if (model.ImplementsChar && !column.IsCharParsable) - { - spc.ReportDiagnostic(column.ConverterFqn is not null - ? Diagnostic.Create(UnsupportedConverterType, model.Location, column.ConverterFqn, column.Name, column.TypeFqn) - : Diagnostic.Create(UnsupportedColumnType, model.Location, column.Name, column.TypeFqn)); - charTypeError = true; - } - if (model.ImplementsUtf8 && !column.IsUtf8Parsable) - { - spc.ReportDiagnostic(column.ConverterFqn is not null - ? Diagnostic.Create(UnsupportedUtf8ConverterType, model.Location, column.ConverterFqn, column.Name, column.TypeFqn) - : Diagnostic.Create(UnsupportedUtf8ColumnType, model.Location, column.Name, column.TypeFqn)); - utf8TypeError = true; - } - } - if (model.ImplementsChar) - { - bool writeOk = column.WKind switch - { - WriteKind.String => true, - WriteKind.Converter => column.IsCharParsable, // IFixedWidthConverter is symmetric read/write - WriteKind.Formattable => column.IsCharFormattable, - _ => false - }; - if (!writeOk) - { - spc.ReportDiagnostic(column.ConverterFqn is not null - ? Diagnostic.Create(UnsupportedConverterType, model.Location, column.ConverterFqn, column.Name, column.TypeFqn) - : Diagnostic.Create(UnsupportedWriteColumnType, model.Location, column.Name, column.TypeFqn)); - writeTypeError = true; - } - if (column.Overflow == 1 && column.WKind != WriteKind.String) - { - spc.ReportDiagnostic(Diagnostic.Create(InvalidOverflowBehavior, model.Location, column.Name, model.TypeName, column.TypeFqn)); - writeTypeError = true; - } - } - if (column.Start < 0) - { - spc.ReportDiagnostic(Diagnostic.Create(NegativeStart, model.Location, column.Name, model.TypeName, column.Start)); - structuralError = true; - } - if (column.Length < 1) - { - spc.ReportDiagnostic(Diagnostic.Create(InvalidLength, model.Location, column.Name, model.TypeName, column.Length)); - structuralError = true; - } + error |= ReportColumnErrors(spc, model, column); } - - if (columns.Length > 1) + error |= ReportOverlaps(spc, model, columns); + if (error) { - var sorted = columns.ToArray(); - Array.Sort(sorted, static (a, b) => - a.Start != b.Start ? a.Start.CompareTo(b.Start) : a.Length.CompareTo(b.Length)); + return; + } - var farthest = sorted[0]; - int maxEnd = farthest.Start + farthest.Length; - foreach (ref readonly var current in sorted.AsSpan(1)) - { - if (current.Start < maxEnd) - { - spc.ReportDiagnostic(Diagnostic.Create( - OverlappingColumns, - model.Location, - model.TypeName, - farthest.Name, - farthest.Start, - farthest.Start + farthest.Length, - current.Name, - current.Start, - current.Start + current.Length)); - structuralError = true; - } + spc.AddSource(model.FullyQualifiedName.Replace("global::", string.Empty) + ".FixedWidth.g.cs", BuildSource(model)); + } - int end = current.Start + current.Length; - if (end > maxEnd) + private static bool ReportColumnErrors(SourceProductionContext spc, ModelInfo model, ColumnInfo column) + { + bool error = false; + void Report(DiagnosticDescriptor descriptor, params object?[] args) + { + spc.ReportDiagnostic(Diagnostic.Create(descriptor, model.Location, args)); + error = true; + } + + switch (column.Kind) + { + case ColumnKind.BadConverter: + Report(UnsupportedConverterType, column.ConverterFqn, column.Name, column.TypeFqn); + break; + case ColumnKind.Unsupported: + Report(UnsupportedColumnType, column.Name, column.TypeFqn); + break; + default: + if (column.WKind == WriteKind.Unsupported) { - maxEnd = end; - farthest = current; + Report(UnsupportedWriteColumnType, column.Name, column.TypeFqn); } - } + break; } - if (structuralError) + if (column.Overflow == 1 && column.WKind != WriteKind.String) { - return; + Report(InvalidOverflowBehavior, column.Name, model.TypeName, column.TypeFqn); + } + if (column.Padding > '\x7F') + { + Report(NonAsciiCharacter, column.Name, model.TypeName, "Padding", column.Padding); + } + if (column.TrimChar > '\x7F') + { + Report(NonAsciiCharacter, column.Name, model.TypeName, "TrimChar", column.TrimChar); + } + if (column.Start < 0) + { + Report(NegativeStart, column.Name, model.TypeName, column.Start); } + if (column.Length < 1) + { + Report(InvalidLength, column.Name, model.TypeName, column.Length); + } + return error; + } - bool emitChar = model.ImplementsChar && !charTypeError; - bool emitUtf8 = model.ImplementsUtf8 && !utf8TypeError; - bool emitWrite = model.ImplementsChar && !writeTypeError; - if (!emitChar && !emitUtf8 && !emitWrite) + private static bool ReportOverlaps(SourceProductionContext spc, ModelInfo model, ImmutableArray columns) + { + if (columns.Length < 2) { - return; + return false; } - string hint = model.FullyQualifiedName.Replace("global::", string.Empty) + ".FixedWidth.g.cs"; - spc.AddSource(hint, BuildSource(model, emitChar, emitUtf8, emitWrite)); - } + bool error = false; + var sorted = columns.ToArray(); + Array.Sort(sorted, static (a, b) => a.Start != b.Start ? a.Start.CompareTo(b.Start) : a.Length.CompareTo(b.Length)); + var farthest = sorted[0]; + int maxEnd = farthest.Start + farthest.Length; + foreach (ref readonly var current in sorted.AsSpan(1)) + { + if (current.Start < maxEnd) + { + spc.ReportDiagnostic(Diagnostic.Create( + OverlappingColumns, model.Location, model.TypeName, + farthest.Name, farthest.Start, farthest.Start + farthest.Length, + current.Name, current.Start, current.Start + current.Length)); + error = true; + } - private const string CharRuntimeFqn = "global::FixedWidthParser.FixedWidthRuntime"; - private const string Utf8RuntimeFqn = "global::FixedWidthParser.Utf8FixedWidthRuntime"; + int end = current.Start + current.Length; + if (end > maxEnd) + { + maxEnd = end; + farthest = current; + } + } + return error; + } - private static string BuildSource(ModelInfo model, bool emitChar, bool emitUtf8, bool emitWrite) + private static string BuildSource(ModelInfo model) { var sb = new StringBuilder(); sb.AppendLine("// "); @@ -506,39 +439,11 @@ private static string BuildSource(ModelInfo model, bool emitChar, bool emitUtf8, sb.Append(indent).AppendLine("{"); string body = indent + " "; - bool wroteConverterFields = AppendConverterFields(sb, body, model); - bool wroteOptionsFields = emitWrite && AppendWriteOptionsFields(sb, body, model); - bool wroteTrimFields = emitUtf8 && AppendUtf8TrimFields(sb, body, model); - if (wroteConverterFields || wroteOptionsFields || wroteTrimFields) - { - sb.AppendLine(); - } - - // TryParse (char / byte) and TryFormat (char) coexist in one partial; each static abstract - // interface member binds implicitly to its matching span element type. - bool wroteMethod = false; - if (emitChar) - { - AppendMethod(sb, body, model, "char", CharRuntimeFqn); - wroteMethod = true; - } - if (emitUtf8) - { - if (wroteMethod) - { - sb.AppendLine(); - } - AppendMethod(sb, body, model, "byte", Utf8RuntimeFqn); - wroteMethod = true; - } - if (emitWrite) - { - if (wroteMethod) - { - sb.AppendLine(); - } - AppendWriteMethod(sb, body, model); - } + AppendFields(sb, body, model); + sb.AppendLine(); + AppendParseMethod(sb, body, model); + sb.AppendLine(); + AppendFormatMethod(sb, body, model); sb.Append(indent).AppendLine("}"); if (model.Namespace is not null) @@ -548,312 +453,227 @@ private static string BuildSource(ModelInfo model, bool emitChar, bool emitUtf8, return sb.ToString(); } - // Emits one static readonly field per converter-backed column, so the converter instance is - // created once (not per parse call) and shared across both TryParse overloads. Returns whether - // any field was written. - private static bool AppendConverterFields(StringBuilder sb, string body, ModelInfo model) + // One converter instance per converter column (created once, shared by parse and format) and + // one ColumnFormatOptions per column, resolved at compile time. + private static void AppendFields(StringBuilder sb, string body, ModelInfo model) { var columns = model.Columns.AsImmutableArray(); - bool any = false; for (int i = 0; i < columns.Length; i++) { - if (columns[i].Kind != ColumnKind.Converter) + var c = columns[i]; + if (c.Kind == ColumnKind.Converter) { - continue; + sb.Append(body).Append("private static readonly ").Append(c.ConverterFqn) + .Append(" __converter").Append(i).Append(" = new ").Append(c.ConverterFqn).AppendLine("();"); } - sb.Append(body).Append("private static readonly ").Append(columns[i].ConverterFqn) - .Append(" __converter").Append(i).Append(" = new ").Append(columns[i].ConverterFqn).AppendLine("();"); - any = true; + sb.Append(body).Append("private static readonly global::FixedWidthParser.Formatters.ColumnFormatOptions __options").Append(i) + .Append(" = new(").Append(AlignmentLiteral(c.Alignment)).Append(", ").Append(SymbolDisplay.FormatLiteral(c.Padding, true)) + .Append(", ").Append(c.Format is null ? "null" : SymbolDisplay.FormatLiteral(c.Format, true)) + .Append(", ").Append(OverflowLiteral(c.Overflow)).AppendLine(");"); + } + } + + private static int LineLength(ModelInfo model) + { + int length = 0; + foreach (var c in model.Columns.AsImmutableArray()) + { + length = Math.Max(length, c.Start + c.Length); } - return any; + return length; } - // Emits one static readonly byte field per column whose TrimChar isn't the default space, - // converting the char to its single-byte ASCII representation once (at type-init) instead of - // on every parse call. A non-ASCII TrimChar throws there with a clear message, mirroring how - // CultureHelpers rejects a non-ASCII decimal separator on the byte path. - private static bool AppendUtf8TrimFields(StringBuilder sb, string body, ModelInfo model) + private static void AppendParseMethod(StringBuilder sb, string body, ModelInfo model) { var columns = model.Columns.AsImmutableArray(); - bool any = false; + sb.Append(body) + .AppendLine("[global::System.Runtime.CompilerServices.MethodImpl(global::System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]"); + sb.Append(body) + .Append("public static bool TryParse(global::System.ReadOnlySpan line, in global::FixedWidthParser.FixedWidthContext context, out ") + .Append(model.FullyQualifiedName).AppendLine(" model)"); + sb.Append(body).AppendLine("{"); + + string stmt = body + " "; + int lineLength = LineLength(model); + if (lineLength > 0) + { + sb.Append(stmt).Append("if (line.Length < ").Append(lineLength).Append(") { context.ReportShortLine(").Append(lineLength) + .AppendLine("); model = default!; return false; }"); + } + for (int i = 0; i < columns.Length; i++) { - if (columns[i].TrimChar == ' ') + var c = columns[i]; + string col = "line.Slice(" + c.Start + ", " + c.Length + ")"; + if (!c.IsNullable) { + AppendColumnParse(sb, stmt, c, col, i, "__v" + i); continue; } - sb.Append(body).Append("private static readonly byte __trim").Append(i) - .Append(" = global::FixedWidthParser.Utf8FixedWidthRuntime.ToAsciiByte(") - .Append(SymbolDisplay.FormatLiteral(columns[i].TrimChar, true)).Append(", ") - .Append(SymbolDisplay.FormatLiteral(columns[i].Name, true)).AppendLine(");"); - any = true; + + // T?: a blank column assigns null without invoking the underlying parser. + sb.Append(stmt).Append(c.TypeFqn).Append("? __v").Append(i).AppendLine(";"); + sb.Append(stmt).Append("if (").Append(RuntimeFqn).Append(".IsBlank(").Append(col).Append(", ").Append(ByteLiteral(c.TrimChar)).AppendLine("))"); + sb.Append(stmt).AppendLine("{"); + sb.Append(stmt).Append(" __v").Append(i).AppendLine(" = null;"); + sb.Append(stmt).AppendLine("}"); + sb.Append(stmt).AppendLine("else"); + sb.Append(stmt).AppendLine("{"); + AppendColumnParse(sb, stmt + " ", c, col, i, "__u" + i); + sb.Append(stmt).Append(" __v").Append(i).Append(" = __u").Append(i).AppendLine(";"); + sb.Append(stmt).AppendLine("}"); } - return any; - } - // Emits one static readonly ColumnFormatOptions field per column (alignment/padding/format/ - // overflow, resolved at Extract time), shared by the write method's per-column formatting calls. - private static bool AppendWriteOptionsFields(StringBuilder sb, string body, ModelInfo model) - { - var columns = model.Columns.AsImmutableArray(); + sb.Append(stmt).Append("model = new ").Append(model.FullyQualifiedName).AppendLine(); + sb.Append(stmt).AppendLine("{"); for (int i = 0; i < columns.Length; i++) { - var c = columns[i]; - sb.Append(body).Append("private static readonly global::FixedWidthParser.Formatters.ColumnFormatOptions __options").Append(i) - .Append(" = new(").Append(AlignmentLiteral(c.Alignment)).Append(", ").Append(SymbolDisplay.FormatLiteral(c.Padding, true)) - .Append(", ").Append(c.Format is null ? "null" : SymbolDisplay.FormatLiteral(c.Format, true)) - .Append(", ").Append(OverflowLiteral(c.Overflow)).AppendLine(");"); + sb.Append(stmt).Append(" ").Append(columns[i].Name).Append(" = __v").Append(i).AppendLine(","); } - return columns.Length > 0; + sb.Append(stmt).AppendLine("};"); + sb.Append(stmt).AppendLine("return true;"); + sb.Append(body).AppendLine("}"); } - private static string AlignmentLiteral(int value) + // Emits the kind-specific parse statement for one column into a local named localName. + private static void AppendColumnParse(StringBuilder sb, string stmt, ColumnInfo c, string col, int i, string localName) { - return value == 1 - ? "global::FixedWidthParser.Attributes.Alignment.Right" - : "global::FixedWidthParser.Attributes.Alignment.Left"; - } + string trimArg = TrimArgSuffix(c); + string fail = ")) { context.ReportFailedColumn(" + SymbolDisplay.FormatLiteral(c.Name, true) + ", " + c.Start + ", " + c.Length + + "); model = default!; return false; }"; - private static string OverflowLiteral(int resolvedValue) - { - return resolvedValue switch + switch (c.Kind) { - 1 => "global::FixedWidthParser.Attributes.OverflowBehavior.Truncate", - 2 => "global::FixedWidthParser.Attributes.OverflowBehavior.Throw", - _ => "global::FixedWidthParser.Attributes.OverflowBehavior.Default", - }; + case ColumnKind.Exact: + sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append('.').Append(ExactHelper(c.TypeFqn)).Append('(').Append(col) + .Append(", ").Append(SymbolDisplay.FormatLiteral(c.Format!, true)).Append(", context, out var ").Append(localName).Append(trimArg).AppendLine(fail); + break; + case ColumnKind.String: + sb.Append(stmt).Append("string ").Append(localName) + .Append(" = ").Append(RuntimeFqn).Append(".String(").Append(col).Append(", context").Append(trimArg).AppendLine(");"); + break; + case ColumnKind.Double: + sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append(".TryDouble(").Append(col) + .Append(", context, out double ").Append(localName).Append(trimArg).AppendLine(fail); + break; + case ColumnKind.Float: + sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append(".TryFloat(").Append(col) + .Append(", context, out float ").Append(localName).Append(trimArg).AppendLine(fail); + break; + case ColumnKind.Utf8Parsable: + case ColumnKind.DecodedParsable: + sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append(c.Kind == ColumnKind.Utf8Parsable ? ".TryParse<" : ".TryParseDecoded<") + .Append(c.TypeFqn).Append(">(").Append(col).Append(", context, out ").Append(c.TypeFqn).Append(' ').Append(localName).Append(trimArg).AppendLine(fail); + break; + case ColumnKind.Converter: + sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append(".TryConvert<").Append(c.TypeFqn).Append(", ").Append(c.ConverterFqn) + .Append(">(").Append(col).Append(", context, __converter").Append(i).Append(", out ").Append(c.TypeFqn) + .Append(' ').Append(localName).Append(trimArg).AppendLine(fail); + break; + default: + // Unsupported kinds were reported in Emit, which then skips generation. + break; + } } - private static void AppendWriteMethod(StringBuilder sb, string body, ModelInfo model) + private static void AppendFormatMethod(StringBuilder sb, string body, ModelInfo model) { var columns = model.Columns.AsImmutableArray(); - int lineLength = 0; - foreach (var c in columns) - { - int end = c.Start + c.Length; - if (end > lineLength) - { - lineLength = end; - } - } - + int lineLength = LineLength(model); sb.Append(body) .AppendLine("[global::System.Runtime.CompilerServices.MethodImpl(global::System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]"); sb.Append(body) .Append("public static bool TryFormat(in ").Append(model.FullyQualifiedName) - .Append(" model, global::System.Span destination, global::System.IFormatProvider? formatProvider, out int charsWritten)").AppendLine(); + .AppendLine(" model, global::System.Span destination, in global::FixedWidthParser.FixedWidthContext context, out int bytesWritten)"); sb.Append(body).AppendLine("{"); string stmt = body + " "; - sb.Append(stmt).Append("if (destination.Length < ").Append(lineLength).AppendLine(") { charsWritten = 0; return false; }"); + sb.Append(stmt).Append("if (destination.Length < ").Append(lineLength).AppendLine(") { bytesWritten = 0; return false; }"); if (columns.Length > 0) { sb.Append(stmt).Append("var __line = destination[..").Append(lineLength).AppendLine("];"); - sb.Append(stmt).AppendLine("__line.Fill(' ');"); + sb.Append(stmt).AppendLine("__line.Fill((byte)' ');"); for (int i = 0; i < columns.Length; i++) { - var c = columns[i]; - string slice = "__line.Slice(" + c.Start + ", " + c.Length + ")"; - AppendColumnWrite(sb, stmt, c, slice, i); + AppendColumnFormat(sb, stmt, columns[i], i); } } - sb.Append(stmt).Append("charsWritten = ").Append(lineLength).AppendLine(";"); + sb.Append(stmt).Append("bytesWritten = ").Append(lineLength).AppendLine(";"); sb.Append(stmt).AppendLine("return true;"); - sb.Append(body).AppendLine("}"); } - // Writes one column: a nullable (T?) column fills its slice with the padding character when - // null, otherwise formats exactly as the non-nullable case would (via the temp local pattern - // variable from the `is { }` check). - private static void AppendColumnWrite(StringBuilder sb, string stmt, ColumnInfo c, string slice, int i) + // A nullable (T?) column fills its slice with its trim character when null. + private static void AppendColumnFormat(StringBuilder sb, string stmt, ColumnInfo c, int i) { + string slice = "__line.Slice(" + c.Start + ", " + c.Length + ")"; if (!c.IsNullable) { - AppendColumnKindWrite(sb, stmt, c, slice, i, "model." + c.Name); + AppendColumnKindFormat(sb, stmt, c, slice, i, "model." + c.Name); return; } sb.Append(stmt).Append("if (model.").Append(c.Name).Append(" is { } __u").Append(i).AppendLine(")"); sb.Append(stmt).AppendLine("{"); - AppendColumnKindWrite(sb, stmt + " ", c, slice, i, "__u" + i); + AppendColumnKindFormat(sb, stmt + " ", c, slice, i, "__u" + i); sb.Append(stmt).AppendLine("}"); sb.Append(stmt).AppendLine("else"); sb.Append(stmt).AppendLine("{"); - char fillChar = c.TrimChar != ' ' ? c.TrimChar : ' '; - sb.Append(stmt).Append(" ").Append(slice).Append(".Fill(").Append(SymbolDisplay.FormatLiteral(fillChar, true)).AppendLine(");"); + sb.Append(stmt).Append(" ").Append(slice).Append(".Fill(").Append(ByteLiteral(c.TrimChar)).AppendLine(");"); sb.Append(stmt).AppendLine("}"); } - private static void AppendColumnKindWrite(StringBuilder sb, string stmt, ColumnInfo c, string slice, int i, string valueExpr) + private static void AppendColumnKindFormat(StringBuilder sb, string stmt, ColumnInfo c, string slice, int i, string valueExpr) { + string tail = ", __options" + i + ", " + SymbolDisplay.FormatLiteral(c.Name, true) + ");"; switch (c.WKind) { case WriteKind.String: - sb.Append(stmt).Append("global::FixedWidthParser.FixedWidthRuntime.FormatString(").Append(valueExpr).Append(", ").Append(slice) - .Append(", __options").Append(i).Append(", \"").Append(c.Name).AppendLine("\");"); + sb.Append(stmt).Append(RuntimeFqn).Append(".FormatString(").Append(valueExpr).Append(", ").Append(slice).Append(", context").AppendLine(tail); break; case WriteKind.Formattable: - sb.Append(stmt).Append("global::FixedWidthParser.FixedWidthRuntime.FormatValue(").Append(valueExpr).Append(", ").Append(slice) - .Append(", formatProvider, __options").Append(i).Append(", \"").Append(c.Name).AppendLine("\");"); + sb.Append(stmt).Append(RuntimeFqn).Append(".FormatValue(").Append(valueExpr).Append(", ").Append(slice).Append(", context").AppendLine(tail); break; case WriteKind.Converter: - sb.Append(stmt).Append("global::FixedWidthParser.FixedWidthRuntime.FormatConvert(").Append(valueExpr).Append(", ").Append(slice) - .Append(", formatProvider, __converter").Append(i).Append(", __options").Append(i).Append(", \"").Append(c.Name).AppendLine("\");"); + sb.Append(stmt).Append(RuntimeFqn).Append(".FormatConvert(").Append(valueExpr).Append(", ").Append(slice) + .Append(", context, __converter").Append(i).AppendLine(tail); break; default: - // Should not happen, already filtered in Emit with a diagnostic. + // Unsupported kinds were reported in Emit, which then skips generation. break; } } - private static void AppendMethod(StringBuilder sb, string body, ModelInfo model, string elementType, string runtimeFqn) + // Trailing trim arguments for a runtime parse call, omitted for the default (space, trailing). + private static string TrimArgSuffix(ColumnInfo c) { - var columns = model.Columns.AsImmutableArray(); - sb.Append(body) - .AppendLine("[global::System.Runtime.CompilerServices.MethodImpl(global::System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]"); - sb.Append(body) - .Append("public static bool TryParse(global::System.ReadOnlySpan<").Append(elementType) - .Append("> line, global::System.IFormatProvider? formatProvider, global::CommunityToolkit.HighPerformance.Buffers.StringPool? stringPool, out ") - .Append(model.FullyQualifiedName).AppendLine(" model)"); - sb.Append(body).AppendLine("{"); - - string stmt = body + " "; - if (columns.Length > 0) - { - int maxEnd = 0; - foreach (var v in columns) - { - int end = v.Start + v.Length; - if (end > maxEnd) - { - maxEnd = end; - } - } - - sb.Append(stmt).Append("if (line.Length < ").Append(maxEnd).AppendLine(") { model = default!; return false; }"); - AppendParseAndReturn(sb, stmt, model, runtimeFqn, elementType, static c => "line.Slice(" + c.Start + ", " + c.Length + ")"); - } - else + if (c.TrimChar == ' ' && c.TrimMode == 0) { - AppendParseAndReturn(sb, stmt, model, runtimeFqn, elementType, static _ => "default"); + return string.Empty; } - - sb.Append(body).AppendLine("}"); + return ", " + ByteLiteral(c.TrimChar) + ", " + TrimModeLiteral(c.TrimMode); } - private static void AppendParseAndReturn( - StringBuilder sb, - string stmt, - ModelInfo model, - string runtimeFqn, - string elementType, - Func columnExpression) + private static string ByteLiteral(char value) { - var columns = model.Columns.AsImmutableArray(); - for (int i = 0; i < columns.Length; i++) - { - var c = columns[i]; - string col = columnExpression(c); - - if (!c.IsNullable) - { - AppendColumnKindParse(sb, stmt, c, col, runtimeFqn, elementType, i, "__v" + i); - continue; - } - - // T?: a blank (trimmed-empty) column assigns null without invoking the underlying - // parser; otherwise the underlying T parses into a temp local exactly as it would for - // a non-nullable column, then is boxed into the T? local. - string trimCall = elementType == "char" - ? "global::FixedWidthParser.FixedWidthRuntime.TrimColumn(" + col + ", " + SymbolDisplay.FormatLiteral(c.TrimChar, true) + ", global::FixedWidthParser.Attributes.TrimMode.Both).Trim(' ').IsEmpty" - : "global::FixedWidthParser.Utf8FixedWidthRuntime.TrimColumn(" + col + ", " + (c.TrimChar == ' ' ? "(byte)' '" : "__trim" + i) + ", global::FixedWidthParser.Attributes.TrimMode.Both).Trim((byte)' ').IsEmpty"; - sb.Append(stmt).Append(c.TypeFqn).Append("? __v").Append(i).AppendLine(";"); - sb.Append(stmt).Append("if (").Append(trimCall).AppendLine(")"); - sb.Append(stmt).AppendLine("{"); - sb.Append(stmt).Append(" __v").Append(i).AppendLine(" = null;"); - sb.Append(stmt).AppendLine("}"); - sb.Append(stmt).AppendLine("else"); - sb.Append(stmt).AppendLine("{"); - AppendColumnKindParse(sb, stmt + " ", c, col, runtimeFqn, elementType, i, "__u" + i); - sb.Append(stmt).Append(" __v").Append(i).Append(" = __u").Append(i).AppendLine(";"); - sb.Append(stmt).AppendLine("}"); - } - - sb.Append(stmt).Append("model = new ").Append(model.FullyQualifiedName).AppendLine(); - sb.Append(stmt).AppendLine("{"); - for (int i = 0; i < columns.Length; i++) - { - sb.Append(stmt).Append(" ").Append(columns[i].Name).Append(" = __v").Append(i).AppendLine(","); - } - sb.Append(stmt).AppendLine("};"); - sb.Append(stmt).AppendLine("return true;"); + return "(byte)" + SymbolDisplay.FormatLiteral(value, true); } - // Emits the kind-specific parse statement for one column into a local named . Shared by the non-nullable case (localName = "__v{i}") and the nullable - // case (localName = "__u{i}", a temp later boxed into the "__v{i}" nullable local) — the - // converter field is always "__converter{i}" (keyed by column index, not the local's name). - private static void AppendColumnKindParse(StringBuilder sb, string stmt, ColumnInfo c, string col, string runtimeFqn, string elementType, int i, string localName) + private static string AlignmentLiteral(int value) { - string trimArg = TrimArgSuffix(elementType, c, i); - bool isDateTimeType = IsDateTimeType(c.TypeFqn); - if (c.Format is not null && isDateTimeType) - { - string exactHelper = c.TypeFqn.Contains("DateTimeOffset") ? "TryDateTimeOffsetExact" - : c.TypeFqn.Contains("DateTime") ? "TryDateTimeExact" - : c.TypeFqn.Contains("DateOnly") ? "TryDateOnlyExact" - : "TryTimeOnlyExact"; - - sb.Append(stmt).Append("if (!").Append(runtimeFqn).Append('.').Append(exactHelper).Append('(').Append(col) - .Append(", ").Append(SymbolDisplay.FormatLiteral(c.Format, true)).Append(", formatProvider, out var ").Append(localName).Append(trimArg).AppendLine(")) { model = default!; return false; }"); - return; - } - - switch (c.Kind) - { - case ColumnKind.String: - sb.Append(stmt).Append("string ").Append(localName) - .Append(" = ").Append(runtimeFqn).Append(".String(").Append(col).Append(", stringPool").Append(trimArg).AppendLine(");"); - break; - case ColumnKind.Double: - sb.Append(stmt).Append("if (!").Append(runtimeFqn).Append(".TryDouble(").Append(col) - .Append(", formatProvider, out double ").Append(localName).Append(trimArg).AppendLine(")) { model = default!; return false; }"); - break; - case ColumnKind.Float: - sb.Append(stmt).Append("if (!").Append(runtimeFqn).Append(".TryFloat(").Append(col) - .Append(", formatProvider, out float ").Append(localName).Append(trimArg).AppendLine(")) { model = default!; return false; }"); - break; - case ColumnKind.SpanParsable: - sb.Append(stmt).Append("if (!").Append(runtimeFqn).Append(".TryParse<").Append(c.TypeFqn).Append(">(").Append(col) - .Append(", formatProvider, out ").Append(c.TypeFqn).Append(' ').Append(localName).Append(trimArg).AppendLine(")) { model = default!; return false; }"); - break; - case ColumnKind.Converter: - sb.Append(stmt).Append("if (!").Append(runtimeFqn).Append(".TryConvert<").Append(c.TypeFqn).Append(", ").Append(c.ConverterFqn) - .Append(">(").Append(col).Append(", formatProvider, __converter").Append(i).Append(", out ").Append(c.TypeFqn) - .Append(' ').Append(localName).Append(trimArg).AppendLine(")) { model = default!; return false; }"); - break; - default: - // Should not happen, already filtered in Emit with a diagnostic. - break; - } + return value == 1 + ? "global::FixedWidthParser.Attributes.Alignment.Right" + : "global::FixedWidthParser.Attributes.Alignment.Left"; } - // Trailing trimChar argument for a runtime parse call: omitted when the column uses the - // default (space) trim, so unconfigured columns keep emitting the exact same call shape as - // before this option existed. Char columns pass the literal directly; byte columns reference - // the precomputed __trim{i} field (AppendUtf8TrimFields) since a non-ASCII char can't be a - // byte literal — the field does that conversion once, with a clear throw if it doesn't fit. - private static string TrimArgSuffix(string elementType, ColumnInfo c, int i) + private static string OverflowLiteral(int resolvedValue) { - if (c.TrimChar == ' ' && c.TrimMode == 0) + return resolvedValue switch { - return string.Empty; - } - string trimArg = elementType == "char" - ? SymbolDisplay.FormatLiteral(c.TrimChar, true) - : "__trim" + i; - return ", " + trimArg + ", " + TrimModeLiteral(c.TrimMode); + 1 => "global::FixedWidthParser.Attributes.OverflowBehavior.Truncate", + 2 => "global::FixedWidthParser.Attributes.OverflowBehavior.Throw", + _ => "global::FixedWidthParser.Attributes.OverflowBehavior.Default", + }; } private static string TrimModeLiteral(int value) @@ -866,13 +686,13 @@ private static string TrimModeLiteral(int value) }; } - private enum ColumnKind { String, Double, Float, SpanParsable, Converter, Unsupported } + private enum ColumnKind { String, Double, Float, Utf8Parsable, DecodedParsable, Exact, Converter, BadConverter, Unsupported } private enum WriteKind { String, Formattable, Converter, Unsupported } private readonly struct ColumnInfo( - string name, int start, int length, ColumnKind kind, string typeFqn, string? converterFqn, bool isNullable, bool isCharParsable, bool isUtf8Parsable, - WriteKind writeKind, bool isCharFormattable, int alignment, char padding, string? format, int overflow, char trimChar, int trimMode) : IEquatable + string name, int start, int length, ColumnKind kind, string typeFqn, string? converterFqn, bool isNullable, + WriteKind writeKind, int alignment, char padding, string? format, int overflow, char trimChar, int trimMode) : IEquatable { public readonly string Name = name; public readonly int Start = start; @@ -881,10 +701,7 @@ private readonly struct ColumnInfo( public readonly string TypeFqn = typeFqn; public readonly string? ConverterFqn = converterFqn; public readonly bool IsNullable = isNullable; - public readonly bool IsCharParsable = isCharParsable; - public readonly bool IsUtf8Parsable = isUtf8Parsable; public readonly WriteKind WKind = writeKind; - public readonly bool IsCharFormattable = isCharFormattable; public readonly int Alignment = alignment; public readonly char Padding = padding; public readonly string? Format = format; @@ -898,16 +715,13 @@ public bool Equals(ColumnInfo other) && Length == other.Length && Kind == other.Kind && IsNullable == other.IsNullable - && IsCharParsable == other.IsCharParsable && WKind == other.WKind - && IsCharFormattable == other.IsCharFormattable && Alignment == other.Alignment && Padding == other.Padding && Overflow == other.Overflow && TrimChar == other.TrimChar && TrimMode == other.TrimMode && string.Equals(Format, other.Format, StringComparison.Ordinal) - && IsUtf8Parsable == other.IsUtf8Parsable && string.Equals(Name, other.Name, StringComparison.Ordinal) && string.Equals(TypeFqn, other.TypeFqn, StringComparison.Ordinal) && string.Equals(ConverterFqn, other.ConverterFqn, StringComparison.Ordinal); @@ -932,10 +746,7 @@ public override int GetHashCode() hash = hash * 31 + StringComparer.Ordinal.GetHashCode(TypeFqn); hash = hash * 31 + (ConverterFqn is null ? 0 : StringComparer.Ordinal.GetHashCode(ConverterFqn)); hash = hash * 31 + (IsNullable ? 1 : 0); - hash = hash * 31 + (IsCharParsable ? 1 : 0); - hash = hash * 31 + (IsUtf8Parsable ? 1 : 0); hash = hash * 31 + (int)WKind; - hash = hash * 31 + (IsCharFormattable ? 1 : 0); hash = hash * 31 + Alignment; hash = hash * 31 + Padding; hash = hash * 31 + Overflow; @@ -948,8 +759,7 @@ public override int GetHashCode() } private sealed class ModelInfo(string? ns, string typeKeywords, string typeName, string fullyQualifiedName, - EquatableArray columns, bool isPartial, bool isNested, - bool implementsChar, bool implementsUtf8, Location location) : IEquatable + EquatableArray columns, bool isPartial, bool isNested, Location location) : IEquatable { public string? Namespace { get; } = ns; public string TypeKeywords { get; } = typeKeywords; @@ -958,20 +768,16 @@ private sealed class ModelInfo(string? ns, string typeKeywords, string typeName, public EquatableArray Columns { get; } = columns; public bool IsPartial { get; } = isPartial; public bool IsNested { get; } = isNested; - public bool ImplementsChar { get; } = implementsChar; - public bool ImplementsUtf8 { get; } = implementsUtf8; public Location Location { get; } = location; // Value equality (excluding Location, which doesn't affect the emitted source) so the - // incremental pipeline caches correctly: toggling a marker or a column type re-runs Emit. + // incremental pipeline caches correctly. [ExcludeFromCodeCoverage] public bool Equals(ModelInfo? other) { return other is not null && IsPartial == other.IsPartial && IsNested == other.IsNested - && ImplementsChar == other.ImplementsChar - && ImplementsUtf8 == other.ImplementsUtf8 && string.Equals(Namespace, other.Namespace, StringComparison.Ordinal) && string.Equals(TypeKeywords, other.TypeKeywords, StringComparison.Ordinal) && string.Equals(TypeName, other.TypeName, StringComparison.Ordinal) @@ -998,8 +804,6 @@ public override int GetHashCode() hash = hash * 31 + Columns.GetHashCode(); hash = hash * 31 + (IsPartial ? 1 : 0); hash = hash * 31 + (IsNested ? 1 : 0); - hash = hash * 31 + (ImplementsChar ? 1 : 0); - hash = hash * 31 + (ImplementsUtf8 ? 1 : 0); return hash; } } diff --git a/src/FixedWidthParser.Generator/FixedWidthParser.Generator.csproj b/src/FixedWidthParser.Generator/FixedWidthParser.Generator.csproj index 881ffc0..c90cb18 100644 --- a/src/FixedWidthParser.Generator/FixedWidthParser.Generator.csproj +++ b/src/FixedWidthParser.Generator/FixedWidthParser.Generator.csproj @@ -1,11 +1,13 @@ - netstandard2.1 + netstandard2.0 latest enable disable true + true + true true false diff --git a/src/FixedWidthParser/CompatibilitySuppressions.xml b/src/FixedWidthParser/CompatibilitySuppressions.xml new file mode 100644 index 0000000..9970ad0 --- /dev/null +++ b/src/FixedWidthParser/CompatibilitySuppressions.xml @@ -0,0 +1,505 @@ + + + + + CP0001 + T:FixedWidthParser.Attributes.ColumnLayoutValidator + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.FixedWidthUtf8 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Formatters.IColumnFormatter`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.IUtf8FixedWidthModel`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Parsers.Utf8FixedWidthParser`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Processors.ColumnParser`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Processors.ColumnParserRegistry + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Processors.ColumnValueParser`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Processors.Utf8ColumnParser`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Processors.Utf8ColumnParserRegistry + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Processors.Utf8ColumnValueParser`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.AsyncRecordEnumeratorCore`2 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.FixedWidthByteReader`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.FixedWidthRecordEnumerable`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.GeneratedFixedWidthAsyncRecordEnumerable`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.GeneratedFixedWidthRecordEnumerable`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.GeneratedUtf8FixedWidthAsyncRecordEnumerable`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.GeneratedUtf8FixedWidthPipeRecordEnumerable`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.GeneratedUtf8FixedWidthRecordEnumerable`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.GeneratedUtf8LineParser`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.ILineParser`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.IUtf8LineParser`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.ReflectionUtf8LineParser`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.Utf8AsyncRecordEnumeratorCore`2 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.Utf8FixedWidthAsyncRecordEnumerable`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.Utf8FixedWidthPipeRecordEnumerable`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.Utf8FixedWidthRecordEnumerable`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Readers.Utf8PipeRecordEnumeratorCore`2 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.RefAction`2 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.RefGetter`2 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Utf8FixedWidthRuntime + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.FixedWidth.Read``1(System.IO.Stream,System.Text.Encoding,System.Boolean,System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,System.Int32) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.FixedWidth.Read``1(System.IO.TextReader,System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,System.Int32) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.FixedWidth.ReadAsync``1(System.IO.Stream,System.Text.Encoding,System.Boolean,System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,System.Int32) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.FixedWidth.ReadAsync``1(System.IO.TextReader,System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,System.Int32) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.FixedWidth.ReadFile``1(System.String,System.Text.Encoding,System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,System.Int32) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.FixedWidth.ReadFileAsync``1(System.String,System.Text.Encoding,System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,System.Int32) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.FixedWidth.TryParse``1(System.ReadOnlySpan{System.Char},System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,``0@) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.FixedWidthRuntime.String(System.ReadOnlySpan{System.Char},CommunityToolkit.HighPerformance.Buffers.StringPool) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.FixedWidthRuntime.TryDouble(System.ReadOnlySpan{System.Char},System.IFormatProvider,System.Double@) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.FixedWidthRuntime.TryFloat(System.ReadOnlySpan{System.Char},System.IFormatProvider,System.Single@) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.FixedWidthRuntime.TryParse``1(System.ReadOnlySpan{System.Char},System.IFormatProvider,``0@) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Formatters.ColumnFormatOptions.WriteInto(System.ReadOnlySpan{System.Char},System.Span{System.Char},System.String) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.IFixedWidthModel`1.TryParse(System.ReadOnlySpan{System.Char},System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,`0@) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Parsers.FixedWidthParser`1.TryParse(System.ReadOnlySpan{System.Char},System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,`0@) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Readers.FixedWidthReader`1.#ctor(System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,System.Int32) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Readers.FixedWidthReader`1.Read(System.IO.Stream,System.Text.Encoding,System.Boolean) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Readers.FixedWidthReader`1.Read(System.IO.TextReader) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Readers.FixedWidthReader`1.ReadAsync(System.IO.Stream,System.Text.Encoding,System.Boolean) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Readers.FixedWidthReader`1.ReadAsync(System.IO.TextReader) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Readers.FixedWidthReader`1.ReadFile(System.String,System.Text.Encoding) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Readers.FixedWidthReader`1.ReadFileAsync(System.String,System.Text.Encoding) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Readers.GeneratedLineParser`1.TryParse(System.ReadOnlySpan{System.Char},System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,`0@) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Readers.ReflectionLineParser`1.#ctor(FixedWidthParser.Parsers.FixedWidthParser{`0}) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Readers.ReflectionLineParser`1.TryParse(System.ReadOnlySpan{System.Char},System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,`0@) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Writers.FixedWidthWriter`1.#ctor + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Writers.FixedWidthWriter`1.Write(System.IO.Stream,`0@,System.IFormatProvider) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Writers.FixedWidthWriter`1.Write(System.IO.StreamWriter,`0@,System.IFormatProvider) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Writers.FixedWidthWriter`1.WriteAsync(System.IO.Stream,`0,System.IFormatProvider,System.Threading.CancellationToken) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Writers.FixedWidthWriter`1.WriteAsync(System.IO.StreamWriter,`0,System.IFormatProvider,System.Threading.CancellationToken) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Writers.FixedWidthWriter`1.WriteMany(System.IO.Stream,System.Collections.Generic.IEnumerable{`0},System.IFormatProvider) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Writers.FixedWidthWriter`1.WriteMany(System.IO.Stream,System.ReadOnlySpan{`0},System.IFormatProvider) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Writers.FixedWidthWriter`1.WriteMany(System.IO.StreamWriter,System.Collections.Generic.IEnumerable{`0},System.IFormatProvider) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Writers.FixedWidthWriter`1.WriteMany(System.IO.StreamWriter,System.ReadOnlySpan{`0},System.IFormatProvider) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Writers.FixedWidthWriter`1.WriteManyAsync(System.IO.Stream,System.Collections.Generic.IEnumerable{`0},System.IFormatProvider,System.Threading.CancellationToken) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0002 + M:FixedWidthParser.Writers.FixedWidthWriter`1.WriteManyAsync(System.IO.StreamWriter,System.Collections.Generic.IEnumerable{`0},System.IFormatProvider,System.Threading.CancellationToken) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0006 + M:FixedWidthParser.IFixedWidthModel`1.TryFormat(`0@,System.Span{System.Byte},FixedWidthParser.FixedWidthContext@,System.Int32@) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0006 + M:FixedWidthParser.IFixedWidthModel`1.TryParse(System.ReadOnlySpan{System.Byte},FixedWidthParser.FixedWidthContext@,`0@) + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0008 + T:FixedWidthParser.Readers.GeneratedLineParser`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0008 + T:FixedWidthParser.Readers.ReflectionLineParser`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + PKV006 + net8.0 + + \ No newline at end of file diff --git a/src/FixedWidthParser/Delegates.cs b/src/FixedWidthParser/Delegates.cs index cc4786a..d63bce5 100644 --- a/src/FixedWidthParser/Delegates.cs +++ b/src/FixedWidthParser/Delegates.cs @@ -1,10 +1,5 @@ namespace FixedWidthParser { -#if NET9_0_OR_GREATER - public delegate void RefAction(ref TModel model, TProperty value) where TModel : allows ref struct; - public delegate TProperty RefGetter(in TModel model) where TModel : allows ref struct; -#else - public delegate void RefAction(ref TModel model, TProperty value); - public delegate TProperty RefGetter(in TModel model); -#endif -} \ No newline at end of file + internal delegate void RefAction(ref TModel model, TProperty value) where TModel : allows ref struct; + internal delegate TProperty RefGetter(in TModel model) where TModel : allows ref struct; +} diff --git a/src/FixedWidthParser/FixedWidth.cs b/src/FixedWidthParser/FixedWidth.cs index ace9596..337bddb 100644 --- a/src/FixedWidthParser/FixedWidth.cs +++ b/src/FixedWidthParser/FixedWidth.cs @@ -1,135 +1,71 @@ -using CommunityToolkit.HighPerformance.Buffers; +using System.IO.Pipelines; using FixedWidthParser.Readers; -using System.Text; namespace FixedWidthParser { /// - /// Entry points for the source-generated, reflection-free parse path. Dispatches to the model's - /// generated through the type parameter, so the call - /// is statically resolved (no reflection, no delegates, JIT-devirtualized). + /// Entry points for source-generated models. Dispatches to the model's generated + /// and + /// through the type parameter, so the call is statically resolved (no reflection, no delegates). + /// Column positions are bytes. /// public static class FixedWidth { - /// Parses a single fixed-width line into using its generated parser. - public static bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model) -#if NET9_0_OR_GREATER + /// Parses a single line, without its line break, into . + public static bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TModel model) where TModel : IFixedWidthModel, allows ref struct -#else - where TModel : IFixedWidthModel -#endif { - return TModel.TryParse(line, formatProvider, stringPool, out model); + return TModel.TryParse(line, in context, out model); } - /// Formats into using its generated writer. - public static bool TryFormat(in TModel model, Span destination, IFormatProvider? formatProvider, out int charsWritten) -#if NET9_0_OR_GREATER + /// Formats into , without a line break. + public static bool TryFormat(in TModel model, Span destination, in FixedWidthContext context, out int bytesWritten) where TModel : IFixedWidthModel, allows ref struct -#else - where TModel : IFixedWidthModel -#endif { - return TModel.TryFormat(in model, destination, formatProvider, out charsWritten); - } - - /// Reads source-generated models from an existing . - public static FixedWidthRecordEnumerable> Read( - TextReader reader, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) - where TModel : IFixedWidthModel - { - ArgumentNullException.ThrowIfNull(reader); - ValidateBufferSize(bufferSize); - return new FixedWidthRecordEnumerable>( - default, TextReaderSource.FromReader(reader), formatProvider, stringPool, bufferSize); + return TModel.TryFormat(in model, destination, in context, out bytesWritten); } - /// Reads source-generated models from a . - public static FixedWidthRecordEnumerable> Read( - Stream stream, - Encoding? encoding = null, - bool leaveOpen = false, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) + /// Reads from a stream, once. controls whether the stream is disposed when iteration ends. + public static FixedWidthRecordEnumerable> Read(Stream stream, bool leaveOpen = false, FixedWidthReaderOptions? options = null) where TModel : IFixedWidthModel { ArgumentNullException.ThrowIfNull(stream); - ValidateBufferSize(bufferSize); - var enc = encoding ?? Encoding.UTF8; - return new FixedWidthRecordEnumerable>( - default, TextReaderSource.FromStream(stream, enc, leaveOpen), formatProvider, stringPool, bufferSize); + return new(default, stream, ownsStream: !leaveOpen, FixedWidthReaderOptions.Validate(options)); } - /// Reads source-generated models from a file, reopening it for each enumeration. - public static FixedWidthRecordEnumerable> ReadFile( - string path, - Encoding? encoding = null, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) + /// Reads from a file, reopening it on each enumeration. + public static FixedWidthRecordEnumerable> ReadFile(string path, FixedWidthReaderOptions? options = null) where TModel : IFixedWidthModel { ArgumentException.ThrowIfNullOrEmpty(path); - ValidateBufferSize(bufferSize); - var enc = encoding ?? Encoding.UTF8; - return new FixedWidthRecordEnumerable>( - default, TextReaderSource.FromFile(path, enc, useAsync: false), formatProvider, stringPool, bufferSize); + return new(default, path, FixedWidthReaderOptions.Validate(options)); } - /// Reads source-generated models from an existing via await foreach. - public static FixedWidthAsyncRecordEnumerable> ReadAsync( - TextReader reader, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) + /// Reads from a stream via await foreach, once. controls whether the stream is disposed when iteration ends. + public static FixedWidthAsyncRecordEnumerable> ReadAsync(Stream stream, bool leaveOpen = false, FixedWidthReaderOptions? options = null) where TModel : IFixedWidthModel { - ArgumentNullException.ThrowIfNull(reader); - ValidateBufferSize(bufferSize); - return new FixedWidthAsyncRecordEnumerable>( - default, TextReaderSource.FromReader(reader), formatProvider, stringPool, bufferSize); + ArgumentNullException.ThrowIfNull(stream); + return new(default, stream, ownsStream: !leaveOpen, FixedWidthReaderOptions.Validate(options)); } - /// Reads source-generated models from a via await foreach. - public static FixedWidthAsyncRecordEnumerable> ReadAsync( - Stream stream, - Encoding? encoding = null, - bool leaveOpen = false, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) + /// + /// Reads from a via await foreach, letting the pipe own buffering. + /// controls whether the reader is completed when iteration ends. + /// + public static FixedWidthPipeRecordEnumerable> ReadAsync(PipeReader reader, bool leaveOpen = false, FixedWidthReaderOptions? options = null) where TModel : IFixedWidthModel { - ArgumentNullException.ThrowIfNull(stream); - ValidateBufferSize(bufferSize); - var enc = encoding ?? Encoding.UTF8; - return new FixedWidthAsyncRecordEnumerable>( - default, TextReaderSource.FromStream(stream, enc, leaveOpen), formatProvider, stringPool, bufferSize); + ArgumentNullException.ThrowIfNull(reader); + return new(default, reader, completeReader: !leaveOpen, FixedWidthReaderOptions.Validate(options)); } - /// Reads source-generated models from a file asynchronously, reopening it for each enumeration. - public static FixedWidthAsyncRecordEnumerable> ReadFileAsync( - string path, - Encoding? encoding = null, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) + /// Reads from a file via await foreach with asynchronous I/O, reopening it on each enumeration. + public static FixedWidthAsyncRecordEnumerable> ReadFileAsync(string path, FixedWidthReaderOptions? options = null) where TModel : IFixedWidthModel { ArgumentException.ThrowIfNullOrEmpty(path); - ValidateBufferSize(bufferSize); - var enc = encoding ?? Encoding.UTF8; - return new FixedWidthAsyncRecordEnumerable>( - default, TextReaderSource.FromFile(path, enc, useAsync: true), formatProvider, stringPool, bufferSize); - } - - private static void ValidateBufferSize(int bufferSize) - { - ArgumentOutOfRangeException.ThrowIfLessThan(bufferSize, 1); + return new(default, path, FixedWidthReaderOptions.Validate(options)); } } } diff --git a/src/FixedWidthParser/FixedWidthContext.cs b/src/FixedWidthParser/FixedWidthContext.cs new file mode 100644 index 0000000..6965103 --- /dev/null +++ b/src/FixedWidthParser/FixedWidthContext.cs @@ -0,0 +1,65 @@ +using System.Globalization; +using System.Text; +using CommunityToolkit.HighPerformance.Buffers; +using FixedWidthParser.Readers; + +namespace FixedWidthParser +{ + /// + /// Settings a model parser, model writer or converter needs for one line, handed to every + /// generated TryParse/TryFormat and every call. + /// + /// + /// Passed by reference. New settings arrive as new properties, so a converter + /// written against this type keeps compiling. means the invariant culture, + /// UTF-8 and no string pool; construct one with an object initializer to unit-test a converter. + /// + public readonly record struct FixedWidthContext + { + private readonly IFormatProvider? _formatProvider; + private readonly Encoding? _encoding; + + /// The format provider for numbers and dates. Defaults to . + public IFormatProvider FormatProvider + { + get => _formatProvider ?? CultureInfo.InvariantCulture; + init => _formatProvider = value; + } + + /// + /// The encoding of the text in the line, used to decode and encode string columns, dates parsed + /// with a format, and types that only parse from characters. Must be ASCII-compatible (UTF-8, + /// Latin-1, Windows-1252, …), since column positions are bytes. Defaults to UTF-8. + /// + public Encoding Encoding + { + get => _encoding ?? Encoding.UTF8; + init => _encoding = value; + } + + /// Interns string columns when set; otherwise each string column is a new string. + public StringPool? StringPool { get; init; } + + internal ParseFailure? Failure { get; init; } + + /// + /// Records that rejected the line, so a reader can name it in its + /// . Generated parsers call this before returning + /// ; outside a reader's diagnosis it does nothing. + /// + public void ReportFailedColumn(string columnName, int start, int length) + { + Failure?.SetColumn(columnName, start, length); + } + + /// + /// Records that the line is shorter than the the layout needs. + /// Generated parsers call this before returning ; outside a reader's + /// diagnosis it does nothing. + /// + public void ReportShortLine(int requiredLength) + { + Failure?.SetShortLine(requiredLength); + } + } +} diff --git a/src/FixedWidthParser/FixedWidthParseException.cs b/src/FixedWidthParser/FixedWidthParseException.cs new file mode 100644 index 0000000..7292c1e --- /dev/null +++ b/src/FixedWidthParser/FixedWidthParseException.cs @@ -0,0 +1,60 @@ +using System.Diagnostics.CodeAnalysis; + +namespace FixedWidthParser +{ + /// + /// Thrown by a reader when a line does not parse into the model, or handed to + /// . Derives from , so + /// existing catch (FormatException) blocks still catch it. + /// + public sealed class FixedWidthParseException : FormatException + { + /// Creates an exception with no message. + [ExcludeFromCodeCoverage] + public FixedWidthParseException() + { + } + + /// Creates an exception with the given message. + [ExcludeFromCodeCoverage] + public FixedWidthParseException(string message) + : base(message) + { + } + + /// Creates an exception with the given message and inner exception. + [ExcludeFromCodeCoverage] + public FixedWidthParseException(string message, Exception innerException) + : base(message, innerException) + { + } + + internal FixedWidthParseException(string message, long lineNumber, string? columnName, int columnStart, int columnLength, string? rawValue) + : base(message) + { + LineNumber = lineNumber; + ColumnName = columnName; + ColumnStart = columnStart; + ColumnLength = columnLength; + RawValue = rawValue; + } + + /// The 1-based number of the line that failed. Empty lines count. + public long LineNumber { get; } + + /// + /// The column that rejected the line, or when the line is shorter than the + /// layout or the model's parser did not report a column. + /// + public string? ColumnName { get; } + + /// The 0-based start of , or -1 when there is no column. + public int ColumnStart { get; } = -1; + + /// The width of , or 0 when there is no column. + public int ColumnLength { get; } + + /// The untrimmed text of , or when there is no column. + public string? RawValue { get; } + } +} diff --git a/src/FixedWidthParser/FixedWidthParser.csproj b/src/FixedWidthParser/FixedWidthParser.csproj index d63184e..8e436d4 100644 --- a/src/FixedWidthParser/FixedWidthParser.csproj +++ b/src/FixedWidthParser/FixedWidthParser.csproj @@ -1,7 +1,7 @@ - net8.0;net10.0 + net10.0 latest enable enable @@ -11,14 +11,29 @@ true - $(NoWarn);CS1591 + true + true + + $([System.String]::Copy('$(NoWarn)').Replace('RS0016', '').Replace('RS0037', ''));CS1591;S2743;CA1720;S3011 + + + + + true + 1.1.1 FixedWidthParser.NET - 1.0.0 + 2.0.0-dev Gabriel Matte FixedWidthParser High-performance, low-allocation parsing and writing of fixed-width (flat) files for .NET. Attribute-driven column mapping, sync/async batch reading (IEnumerable/IAsyncEnumerable), configurable formatting, culture awareness and ref struct support. @@ -49,7 +64,15 @@ - + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + /// Settings for reading lines: and the + /// read methods. Every property has a default, so + /// options and new FixedWidthReaderOptions() read the same way. + /// + public sealed class FixedWidthReaderOptions + { + internal static readonly FixedWidthReaderOptions Default = new(); + + /// The format provider for numbers and dates. means the invariant culture. + public IFormatProvider? FormatProvider { get; init; } + + /// + /// The encoding of the file. Must be ASCII-compatible, since column positions are bytes. + /// means UTF-8, and only then is a leading byte order mark skipped. + /// + public Encoding? Encoding { get; init; } + + /// Interns string columns when set; otherwise each string column is a new string. + public StringPool? StringPool { get; init; } + + /// The number of bytes read from the source at a time. Defaults to 65,536. + public int BufferSize { get; init; } = 65536; + + /// + /// The longest line accepted, in bytes, without its line break. A longer line throws + /// , which bounds the memory a source without line breaks can + /// make the reader buffer. Defaults to 1,048,576. + /// + public int MaxLineLength { get; init; } = 1024 * 1024; + + /// + /// Called when a line fails to parse. Return to skip the line and keep + /// reading, or to throw the exception. When , the + /// reader throws. + /// + public Func? OnParseError { get; init; } + + internal FixedWidthContext Context => new() { FormatProvider = FormatProvider!, Encoding = Encoding!, StringPool = StringPool }; + + internal bool SkipsByteOrderMark => Encoding is null || Encoding.CodePage == Encoding.UTF8.CodePage; + + internal static FixedWidthReaderOptions Validate(FixedWidthReaderOptions? options) + { + options ??= Default; + if (options.BufferSize < 1) + { + throw new ArgumentOutOfRangeException(nameof(options), options.BufferSize, "BufferSize must be at least 1."); + } + if (options.MaxLineLength < 1) + { + throw new ArgumentOutOfRangeException(nameof(options), options.MaxLineLength, "MaxLineLength must be at least 1."); + } + return options; + } + } +} diff --git a/src/FixedWidthParser/FixedWidthRuntime.cs b/src/FixedWidthParser/FixedWidthRuntime.cs index 7b4eee1..51d07cd 100644 --- a/src/FixedWidthParser/FixedWidthRuntime.cs +++ b/src/FixedWidthParser/FixedWidthRuntime.cs @@ -2,7 +2,8 @@ using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Runtime.CompilerServices; -using CommunityToolkit.HighPerformance.Buffers; +using System.Text; +using FixedWidthParser.Attributes; using FixedWidthParser.Formatters; using FixedWidthParser.Processors; @@ -10,100 +11,161 @@ namespace FixedWidthParser { /// /// Low-level helpers called by the source-generated - /// and implementations. They centralize the column - /// slicing, string pooling, numeric parsing/formatting so the generated code stays small and - /// matches the runtime parser/writer's semantics exactly. + /// and implementations, and by the runtime parser + /// and writer, so both paths share one set of semantics. /// Public because the generated code lives in the consumer assembly; not intended for direct use. /// public static class FixedWidthRuntime { + private const int StackBufferSize = 256; + [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ReadOnlySpan TrimColumn(ReadOnlySpan column, char trimChar, Attributes.TrimMode trimMode) + public static ReadOnlySpan TrimColumn(ReadOnlySpan column, byte trimChar, TrimMode trimMode) { return trimMode switch { - Attributes.TrimMode.Leading => column.TrimStart(trimChar), - Attributes.TrimMode.Both => column.Trim(trimChar), + TrimMode.Leading => column.TrimStart(trimChar), + TrimMode.Both => column.Trim(trimChar), _ => column.TrimEnd(trimChar), }; } - /// Materializes a string column: trims spaces and interns via the pool when supplied. + /// True when the column holds only and spaces, so a nullable column reads as . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsBlank(ReadOnlySpan column, byte trimChar) + { + return column.Trim(trimChar).Trim((byte)' ').IsEmpty; + } + + /// Decodes a string column with the context's encoding, interning it through the context's pool when there is one. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static string String(ReadOnlySpan column, StringPool? stringPool, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) + public static string String(ReadOnlySpan column, in FixedWidthContext context, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) { var trimmed = TrimColumn(column, trimChar, trimMode); - return stringPool is null ? trimmed.ToString() : stringPool.GetOrAdd(trimmed); + return context.StringPool is { } pool ? pool.GetOrAdd(trimmed, context.Encoding) : context.Encoding.GetString(trimmed); } - /// Parses a column, honoring the provider's decimal separator (see ). + /// Parses a column, honoring the provider's decimal separator. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryDouble(ReadOnlySpan column, IFormatProvider? formatProvider, out double value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) + public static bool TryDouble(ReadOnlySpan column, in FixedWidthContext context, out double value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) { - return CultureHelpers.TryParseDouble(column, formatProvider, out value, trimChar, trimMode); + return CultureHelpers.TryParseDouble(column, in context, out value, trimChar, trimMode); } - /// Parses a column, honoring the provider's decimal separator (see ). + /// Parses a column, honoring the provider's decimal separator. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryFloat(ReadOnlySpan column, IFormatProvider? formatProvider, out float value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) + public static bool TryFloat(ReadOnlySpan column, in FixedWidthContext context, out float value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) { - return CultureHelpers.TryParseFloat(column, formatProvider, out value, trimChar, trimMode); + return CultureHelpers.TryParseFloat(column, in context, out value, trimChar, trimMode); } - /// - /// Parses any column (int, decimal, DateTime, …), trimming - /// spaces. A means - /// (not the BCL's own default of - /// ) — a fixed-width file is a machine layout, and this - /// keeps every column type in a record agreeing on what "no provider" means, matching the - /// double/float columns (see ), which already treat null - /// as invariant. - /// + /// Parses an column straight from its bytes. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryParse(ReadOnlySpan column, IFormatProvider? formatProvider, [MaybeNullWhen(false)] out TValue value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) + public static bool TryParse(ReadOnlySpan column, in FixedWidthContext context, [MaybeNullWhen(false)] out TValue value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) + where TValue : IUtf8SpanParsable + { + return TValue.TryParse(TrimColumn(column, trimChar, trimMode), context.FormatProvider, out value); + } + + /// Parses an column that has no UTF-8 parser, decoding it with the context's encoding first. + [SkipLocalsInit] + public static bool TryParseDecoded(ReadOnlySpan column, in FixedWidthContext context, [MaybeNullWhen(false)] out TValue value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) where TValue : ISpanParsable { - return TValue.TryParse(TrimColumn(column, trimChar, trimMode), formatProvider ?? CultureInfo.InvariantCulture, out value); + Span stack = stackalloc char[StackBufferSize]; + var text = Decode(TrimColumn(column, trimChar, trimMode), context.Encoding, stack, out char[]? rented); + try + { + return TValue.TryParse(text, context.FormatProvider, out value); + } + finally + { + Return(rented); + } } - /// Parses a column via a FixedColumnAttribute.Converter instance, trimming spaces. + /// Parses a column via a FixedColumnAttribute.Converter instance. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool TryConvert( - ReadOnlySpan column, IFormatProvider? formatProvider, TConverter converter, [MaybeNullWhen(false)] out TValue value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) + ReadOnlySpan column, in FixedWidthContext context, TConverter converter, [MaybeNullWhen(false)] out TValue value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) where TConverter : IFixedWidthConverter { - return converter.TryParse(TrimColumn(column, trimChar, trimMode), formatProvider, out value); + return converter.TryParse(TrimColumn(column, trimChar, trimMode), in context, out value); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryDateTimeExact(ReadOnlySpan column, string format, IFormatProvider? formatProvider, out DateTime value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) + [SkipLocalsInit] + public static bool TryDateTimeExact(ReadOnlySpan column, string format, in FixedWidthContext context, out DateTime value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) { - return DateTime.TryParseExact(TrimColumn(column, trimChar, trimMode), format, formatProvider ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out value); + Span stack = stackalloc char[StackBufferSize]; + var text = Decode(TrimColumn(column, trimChar, trimMode), context.Encoding, stack, out char[]? rented); + try + { + return DateTime.TryParseExact(text, format, context.FormatProvider, DateTimeStyles.None, out value); + } + finally + { + Return(rented); + } } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryDateOnlyExact(ReadOnlySpan column, string format, IFormatProvider? formatProvider, out DateOnly value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) + [SkipLocalsInit] + public static bool TryDateOnlyExact(ReadOnlySpan column, string format, in FixedWidthContext context, out DateOnly value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) { - return DateOnly.TryParseExact(TrimColumn(column, trimChar, trimMode), format, formatProvider ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out value); + Span stack = stackalloc char[StackBufferSize]; + var text = Decode(TrimColumn(column, trimChar, trimMode), context.Encoding, stack, out char[]? rented); + try + { + return DateOnly.TryParseExact(text, format, context.FormatProvider, DateTimeStyles.None, out value); + } + finally + { + Return(rented); + } } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryTimeOnlyExact(ReadOnlySpan column, string format, IFormatProvider? formatProvider, out TimeOnly value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) + [SkipLocalsInit] + public static bool TryTimeOnlyExact(ReadOnlySpan column, string format, in FixedWidthContext context, out TimeOnly value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) { - return TimeOnly.TryParseExact(TrimColumn(column, trimChar, trimMode), format, formatProvider ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out value); + Span stack = stackalloc char[StackBufferSize]; + var text = Decode(TrimColumn(column, trimChar, trimMode), context.Encoding, stack, out char[]? rented); + try + { + return TimeOnly.TryParseExact(text, format, context.FormatProvider, DateTimeStyles.None, out value); + } + finally + { + Return(rented); + } } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryDateTimeOffsetExact(ReadOnlySpan column, string format, IFormatProvider? formatProvider, out DateTimeOffset value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) + [SkipLocalsInit] + public static bool TryDateTimeOffsetExact(ReadOnlySpan column, string format, in FixedWidthContext context, out DateTimeOffset value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) { - return DateTimeOffset.TryParseExact(TrimColumn(column, trimChar, trimMode), format, formatProvider ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out value); + Span stack = stackalloc char[StackBufferSize]; + var text = Decode(TrimColumn(column, trimChar, trimMode), context.Encoding, stack, out char[]? rented); + try + { + return DateTimeOffset.TryParseExact(text, format, context.FormatProvider, DateTimeStyles.None, out value); + } + finally + { + Return(rented); + } } - /// Formats a string column: fills per (alignment/padding/overflow). - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void FormatString(string value, Span slice, ColumnFormatOptions options, string columnName) + /// + /// Encodes a string column into per . A string + /// truncated to fit keeps whole characters, so a multi-byte character is never split. + /// + public static void FormatString(string value, Span slice, in FixedWidthContext context, ColumnFormatOptions options, string columnName) { - options.WriteInto(value.AsSpan(), slice, columnName); + ReadOnlySpan text = value; + var encoding = context.Encoding; + if (options.Overflow == OverflowBehavior.Truncate && encoding.GetByteCount(text) > slice.Length) + { + text = FitText(text, slice.Length, encoding, options.Alignment); + } + WriteText(text, slice, encoding, options, columnName); } /// @@ -112,17 +174,14 @@ public static void FormatString(string value, Span slice, ColumnFormatOpti /// loop for the rare value that needs more room. /// [SkipLocalsInit] - public static void FormatValue(TValue value, Span slice, IFormatProvider? formatProvider, ColumnFormatOptions options, string columnName) + public static void FormatValue(TValue value, Span slice, in FixedWidthContext context, ColumnFormatOptions options, string columnName) where TValue : ISpanFormattable { - // Null means invariant here too (see TryParse above) — otherwise a null-provider - // write would use CurrentCulture while a null-provider parse of that same value uses - // invariant, breaking the default round-trip on any non-invariant machine. - formatProvider ??= CultureInfo.InvariantCulture; + var formatProvider = context.FormatProvider; Span stack = stackalloc char[64]; if (value.TryFormat(stack, out int written, options.Format, formatProvider)) { - options.WriteInto(stack[..written], slice, columnName); + WriteText(stack[..written], slice, context.Encoding, options, columnName); return; } @@ -134,7 +193,7 @@ public static void FormatValue(TValue value, Span slice, IFormatPr { if (value.TryFormat(rented, out int w, options.Format, formatProvider)) { - options.WriteInto(rented.AsSpan(0, w), slice, columnName); + WriteText(rented.AsSpan(0, w), slice, context.Encoding, options, columnName); return; } } @@ -155,23 +214,23 @@ public static void FormatValue(TValue value, Span slice, IFormatPr /// [SkipLocalsInit] public static void FormatConvert( - TValue value, Span slice, IFormatProvider? formatProvider, TConverter converter, ColumnFormatOptions options, string columnName) + TValue value, Span slice, in FixedWidthContext context, TConverter converter, ColumnFormatOptions options, string columnName) where TConverter : IFixedWidthConverter { - Span stack = stackalloc char[64]; - if (converter.TryFormat(value, stack, formatProvider, out int written)) + Span stack = stackalloc byte[StackBufferSize]; + if (converter.TryFormat(value, stack, in context, out int written)) { options.WriteInto(stack[..written], slice, columnName); return; } - int maxSize = Math.Max(1024, slice.Length * 16); - for (int size = 512; size <= maxSize; size *= 2) + int maxSize = Math.Max(4096, slice.Length * 16); + for (int size = 1024; size <= maxSize; size *= 2) { - char[] rented = ArrayPool.Shared.Rent(size); + byte[] rented = ArrayPool.Shared.Rent(size); try { - if (converter.TryFormat(value, rented, formatProvider, out int w)) + if (converter.TryFormat(value, rented, in context, out int w)) { options.WriteInto(rented.AsSpan(0, w), slice, columnName); return; @@ -179,13 +238,112 @@ public static void FormatConvert( } finally { - ArrayPool.Shared.Return(rented); + ArrayPool.Shared.Return(rented); } } throw new InvalidOperationException( $"Value of type \"{typeof(TValue)}\" for column \"{columnName}\" could not be formatted by " + - $"converter \"{typeof(TConverter)}\" within {maxSize} characters."); + $"converter \"{typeof(TConverter)}\" within {maxSize} bytes."); + } + + internal static byte ToAsciiByte(char value, string columnName, string setting) + { + if (value > '\x7F') + { + throw new InvalidOperationException( + $"The {setting} '{value}' (U+{(int)value:X4}) of column \"{columnName}\" is not an ASCII character; column positions are bytes, so it must be one byte in every supported encoding."); + } + return (byte)value; + } + + // Numbers and dates are ASCII in almost every culture, so they narrow straight into bytes; + // anything else goes through the encoding. + [SkipLocalsInit] + private static void WriteText(ReadOnlySpan text, Span slice, Encoding encoding, ColumnFormatOptions options, string columnName) + { + bool ascii = System.Text.Ascii.IsValid(text); + int byteCount = ascii ? text.Length : encoding.GetByteCount(text); + byte[]? rented = byteCount <= StackBufferSize ? null : ArrayPool.Shared.Rent(byteCount); + Span bytes = rented is null ? stackalloc byte[StackBufferSize] : rented; + try + { + int written = ascii + ? NarrowAscii(text, bytes) + : encoding.GetBytes(text, bytes); + options.WriteInto(bytes[..written], slice, columnName); + } + finally + { + if (rented is not null) + { + ArrayPool.Shared.Return(rented); + } + } + } + + private static int NarrowAscii(ReadOnlySpan text, Span bytes) + { + System.Text.Ascii.FromUtf16(text, bytes, out int written); + return written; + } + + // The longest run of whole characters, from the alignment side, whose encoding fits in width bytes. + private static ReadOnlySpan FitText(ReadOnlySpan text, int width, Encoding encoding, Alignment alignment) + { + int bytes = 0; + if (alignment == Alignment.Right) + { + int start = text.Length; + while (start > 0) + { + Rune.DecodeLastFromUtf16(text[..start], out _, out int consumed); + int size = encoding.GetByteCount(text.Slice(start - consumed, consumed)); + if (bytes + size > width) + { + break; + } + bytes += size; + start -= consumed; + } + return text[start..]; + } + + int end = 0; + while (end < text.Length) + { + Rune.DecodeFromUtf16(text[end..], out _, out int consumed); + int size = encoding.GetByteCount(text.Slice(end, consumed)); + if (bytes + size > width) + { + break; + } + bytes += size; + end += consumed; + } + return text[..end]; + } + + private static ReadOnlySpan Decode(ReadOnlySpan bytes, Encoding encoding, Span stack, out char[]? rented) + { + rented = null; + Span chars = stack; + int max = encoding.GetMaxCharCount(bytes.Length); + if (max > stack.Length) + { + rented = ArrayPool.Shared.Rent(max); + chars = rented; + } + int written = encoding.GetChars(bytes, chars); + return chars[..written]; + } + + private static void Return(char[]? rented) + { + if (rented is not null) + { + ArrayPool.Shared.Return(rented); + } } } } diff --git a/src/FixedWidthParser/FixedWidthUtf8.cs b/src/FixedWidthParser/FixedWidthUtf8.cs deleted file mode 100644 index 35ba694..0000000 --- a/src/FixedWidthParser/FixedWidthUtf8.cs +++ /dev/null @@ -1,109 +0,0 @@ -using System.IO.Pipelines; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Readers; - -namespace FixedWidthParser -{ - /// - /// UTF-8 / byte counterpart of : entry points for the source-generated, - /// reflection-free byte parse path. Dispatches to the model's generated - /// through the type parameter, so the call is - /// statically resolved (no reflection, no delegates, JIT-devirtualized). Reads bytes straight from a - /// — no , no transcode, no string per line. - /// Column offsets are measured in bytes (see ). - /// - public static class FixedWidthUtf8 - { - /// Parses a single UTF-8 fixed-width line into using its generated parser. - public static bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model) -#if NET9_0_OR_GREATER - where TModel : IUtf8FixedWidthModel, allows ref struct -#else - where TModel : IUtf8FixedWidthModel -#endif - { - return TModel.TryParse(line, formatProvider, stringPool, out model); - } - - - /// Reads source-generated models from a as raw bytes (single pass). - public static Utf8FixedWidthRecordEnumerable> Read( - Stream stream, - bool leaveOpen = false, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) - where TModel : IUtf8FixedWidthModel - { - ArgumentNullException.ThrowIfNull(stream); - ValidateBufferSize(bufferSize); - return new Utf8FixedWidthRecordEnumerable>( - default, stream, ownsStream: !leaveOpen, formatProvider, stringPool, bufferSize); - } - - /// Reads source-generated models from a file as raw bytes, reopening it for each enumeration. - public static Utf8FixedWidthRecordEnumerable> ReadFile( - string path, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) - where TModel : IUtf8FixedWidthModel - { - ArgumentException.ThrowIfNullOrEmpty(path); - ValidateBufferSize(bufferSize); - return new Utf8FixedWidthRecordEnumerable>( - default, path, formatProvider, stringPool, bufferSize); - } - - /// Reads source-generated models from a via await foreach (single pass). - public static Utf8FixedWidthAsyncRecordEnumerable> ReadAsync( - Stream stream, - bool leaveOpen = false, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) - where TModel : IUtf8FixedWidthModel - { - ArgumentNullException.ThrowIfNull(stream); - ValidateBufferSize(bufferSize); - return new Utf8FixedWidthAsyncRecordEnumerable>( - default, stream, ownsStream: !leaveOpen, formatProvider, stringPool, bufferSize); - } - - /// Reads source-generated models from a file via await foreach (true async I/O), reopening it for each enumeration. - public static Utf8FixedWidthAsyncRecordEnumerable> ReadFileAsync( - string path, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) - where TModel : IUtf8FixedWidthModel - { - ArgumentException.ThrowIfNullOrEmpty(path); - ValidateBufferSize(bufferSize); - return new Utf8FixedWidthAsyncRecordEnumerable>(default, path, formatProvider, stringPool, bufferSize); - } - - /// - /// Reads source-generated models from a via await foreach (single - /// pass), letting the pipe own buffering and read-ahead. Use this when the source is already a pipe - /// (e.g. a Kestrel request body, a socket, or an upstream System.IO.Pipelines stage). - /// controls whether the reader is completed when iteration ends. - /// - public static GeneratedUtf8FixedWidthPipeRecordEnumerable ReadAsync( - PipeReader reader, - bool leaveOpen = false, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null) - where TModel : IUtf8FixedWidthModel - { - ArgumentNullException.ThrowIfNull(reader); - return new GeneratedUtf8FixedWidthPipeRecordEnumerable( - reader, completeReader: !leaveOpen, formatProvider, stringPool); - } - - private static void ValidateBufferSize(int bufferSize) - { - ArgumentOutOfRangeException.ThrowIfLessThan(bufferSize, 1); - } - } -} diff --git a/src/FixedWidthParser/FixedWidthWriterOptions.cs b/src/FixedWidthParser/FixedWidthWriterOptions.cs new file mode 100644 index 0000000..624ceaa --- /dev/null +++ b/src/FixedWidthParser/FixedWidthWriterOptions.cs @@ -0,0 +1,21 @@ +using System.Text; + +namespace FixedWidthParser +{ + /// Settings for . Every property has a default. + public sealed class FixedWidthWriterOptions + { + internal static readonly FixedWidthWriterOptions Default = new(); + + /// The format provider for numbers and dates. means the invariant culture. + public IFormatProvider? FormatProvider { get; init; } + + /// The encoding of the file. Must be ASCII-compatible, since column positions are bytes. means UTF-8, written without a byte order mark. + public Encoding? Encoding { get; init; } + + /// The line break written after each record. Defaults to . + public string NewLine { get; init; } = Environment.NewLine; + + internal FixedWidthContext Context => new() { FormatProvider = FormatProvider!, Encoding = Encoding! }; + } +} diff --git a/src/FixedWidthParser/Formatters/ColumnFormatOptions.cs b/src/FixedWidthParser/Formatters/ColumnFormatOptions.cs index c715d5a..7e0c36b 100644 --- a/src/FixedWidthParser/Formatters/ColumnFormatOptions.cs +++ b/src/FixedWidthParser/Formatters/ColumnFormatOptions.cs @@ -1,3 +1,4 @@ +using System.Text; using FixedWidthParser.Attributes; namespace FixedWidthParser.Formatters @@ -6,40 +7,41 @@ namespace FixedWidthParser.Formatters /// Resolved formatting options for a column (alignment, padding, format string and an overflow /// policy already resolved to or /// ). Centralizes placing the content into the line slice, - /// shared by all formatters. + /// shared by the generated and runtime writers. must be ASCII. /// public readonly record struct ColumnFormatOptions(Alignment Alignment, char Padding, string? Format, OverflowBehavior Overflow) { /// - /// Writes into the column slice , + /// Writes the encoded into the column slice , /// applying alignment, padding and overflow. /// - public void WriteInto(ReadOnlySpan content, Span slice, string columnName) + public void WriteInto(ReadOnlySpan content, Span slice, string columnName) { int width = slice.Length; int length = content.Length; + byte padding = (byte)Padding; if (length <= width) { if (Alignment == Alignment.Right) { int pad = width - length; - if (Padding != ' ' && content.Length > 0 && (content[0] == '-' || content[0] == '+')) + if (Padding != ' ' && content.Length > 0 && (content[0] == (byte)'-' || content[0] == (byte)'+')) { slice[0] = content[0]; - slice[1..(pad + 1)].Fill(Padding); + slice[1..(pad + 1)].Fill(padding); content[1..].CopyTo(slice[(pad + 1)..]); } else { - slice[..pad].Fill(Padding); + slice[..pad].Fill(padding); content.CopyTo(slice[pad..]); } } else { content.CopyTo(slice); - slice[length..].Fill(Padding); + slice[length..].Fill(padding); } return; } @@ -47,10 +49,10 @@ public void WriteInto(ReadOnlySpan content, Span slice, string colum if (Overflow == OverflowBehavior.Throw) { throw new InvalidOperationException( - $"Value \"{content}\" ({length} chars) exceeds the width {width} of column \"{columnName}\"."); + $"Value \"{Encoding.UTF8.GetString(content)}\" ({length} bytes) exceeds the width {width} of column \"{columnName}\"."); } - // Truncate: keep the characters on the alignment side. + // Truncate: keep the bytes on the alignment side. if (Alignment == Alignment.Right) { content[(length - width)..].CopyTo(slice); diff --git a/src/FixedWidthParser/Formatters/ColumnFormatters.cs b/src/FixedWidthParser/Formatters/ColumnFormatters.cs new file mode 100644 index 0000000..2d4b37d --- /dev/null +++ b/src/FixedWidthParser/Formatters/ColumnFormatters.cs @@ -0,0 +1,59 @@ +using FixedWidthParser.Processors; + +namespace FixedWidthParser.Formatters +{ + /// Formats one column of a model into its slice of the line, for the runtime writer. + internal interface IColumnFormatter + { + void Format(in TModel model, Span line, in FixedWidthContext context); + } + + // The formatters are instantiated via reflection (Activator.CreateInstance) in FixedWidthWriter. +#pragma warning disable CA1812 // Avoid uninstantiated internal classes + internal sealed class StringColumnFormatter( + int start, int length, ColumnFormatOptions options, string columnName, RefGetter getter) + : IColumnFormatter + { + public void Format(in TModel model, Span line, in FixedWidthContext context) + { + FixedWidthRuntime.FormatString(getter(in model), line.Slice(start, length), in context, options, columnName); + } + } + + internal sealed class SpanFormattableColumnFormatter( + int start, int length, ColumnFormatOptions options, string columnName, RefGetter getter) + : IColumnFormatter where TProperty : ISpanFormattable + { + public void Format(in TModel model, Span line, in FixedWidthContext context) + { + FixedWidthRuntime.FormatValue(getter(in model), line.Slice(start, length), in context, options, columnName); + } + } + + internal sealed class ConverterColumnFormatter( + int start, int length, ColumnFormatOptions options, string columnName, RefGetter getter, TConverter converter) + : IColumnFormatter where TConverter : IFixedWidthConverter + { + public void Format(in TModel model, Span line, in FixedWidthContext context) + { + FixedWidthRuntime.FormatConvert(getter(in model), line.Slice(start, length), in context, converter, options, columnName); + } + } + + /// A T? column: fills the column with its trim character; a value formats through the T formatter. + internal sealed class NullableColumnFormatter( + int start, int length, byte fill, RefGetter getter, IColumnFormatter inner) + : IColumnFormatter where TUnderlying : struct + { + public void Format(in TModel model, Span line, in FixedWidthContext context) + { + if (!getter(in model).HasValue) + { + line.Slice(start, length).Fill(fill); + return; + } + inner.Format(in model, line, in context); + } + } +#pragma warning restore CA1812 +} diff --git a/src/FixedWidthParser/Formatters/ConverterColumnFormatter.cs b/src/FixedWidthParser/Formatters/ConverterColumnFormatter.cs deleted file mode 100644 index 69899df..0000000 --- a/src/FixedWidthParser/Formatters/ConverterColumnFormatter.cs +++ /dev/null @@ -1,16 +0,0 @@ -using FixedWidthParser.Processors; - -namespace FixedWidthParser.Formatters -{ - [System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", - Justification = "Instantiated via reflection (Activator.CreateInstance) in FixedWidthWriter.CreateFormatter.")] - internal sealed class ConverterColumnFormatter( - int start, int length, ColumnFormatOptions options, string columnName, RefGetter getter, TConverter converter) - : IColumnFormatter where TConverter : IFixedWidthConverter - { - public void Format(in TModel model, Span buffer, IFormatProvider? formatProvider) - { - FixedWidthRuntime.FormatConvert(getter(in model), buffer.Slice(start, length), formatProvider, converter, options, columnName); - } - } -} diff --git a/src/FixedWidthParser/Formatters/IColumnFormatter.cs b/src/FixedWidthParser/Formatters/IColumnFormatter.cs deleted file mode 100644 index 6885cd9..0000000 --- a/src/FixedWidthParser/Formatters/IColumnFormatter.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace FixedWidthParser.Formatters -{ - public interface IColumnFormatter - { - void Format(in TModel model, Span buffer, IFormatProvider? formatProvider); - } -} \ No newline at end of file diff --git a/src/FixedWidthParser/Formatters/NullableColumnFormatter.cs b/src/FixedWidthParser/Formatters/NullableColumnFormatter.cs deleted file mode 100644 index 347310b..0000000 --- a/src/FixedWidthParser/Formatters/NullableColumnFormatter.cs +++ /dev/null @@ -1,26 +0,0 @@ -namespace FixedWidthParser.Formatters -{ - /// - /// Wraps another (built for the underlying T of a - /// T? column) so a value writes as a blank (padding-filled) column - /// instead of delegating to the inner formatter — no duplicated formatting logic per type. - /// - [System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", - Justification = "Instantiated via reflection (Activator.CreateInstance) in FixedWidthWriter.CreateFormatter.")] - internal sealed class NullableColumnFormatter( - int start, int length, char trimChar, RefGetter getter, IColumnFormatter inner) - : IColumnFormatter where TUnderlying : struct - { - private readonly char _fillChar = trimChar; - - public void Format(in TModel model, Span buffer, IFormatProvider? formatProvider) - { - if (!getter(in model).HasValue) - { - buffer.Slice(start, length).Fill(_fillChar); - return; - } - inner.Format(in model, buffer, formatProvider); - } - } -} diff --git a/src/FixedWidthParser/Formatters/SpanFormattableColumnFormatter.cs b/src/FixedWidthParser/Formatters/SpanFormattableColumnFormatter.cs deleted file mode 100644 index 5988c6c..0000000 --- a/src/FixedWidthParser/Formatters/SpanFormattableColumnFormatter.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace FixedWidthParser.Formatters -{ - [System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", - Justification = "Instantiated via reflection (Activator.CreateInstance) in FixedWidthWriter.CreateFormatter.")] - internal sealed class SpanFormattableColumnFormatter( - int start, int length, ColumnFormatOptions options, string columnName, RefGetter getter) - : IColumnFormatter where TProperty : ISpanFormattable - { - public void Format(in TModel model, Span buffer, IFormatProvider? formatProvider) - { - FixedWidthRuntime.FormatValue(getter(in model), buffer.Slice(start, length), formatProvider, options, columnName); - } - } -} diff --git a/src/FixedWidthParser/Formatters/StringColumnFormatter.cs b/src/FixedWidthParser/Formatters/StringColumnFormatter.cs deleted file mode 100644 index 6c4c077..0000000 --- a/src/FixedWidthParser/Formatters/StringColumnFormatter.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace FixedWidthParser.Formatters -{ - [System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", - Justification = "Instantiated via reflection (Activator.CreateInstance) in FixedWidthWriter.CreateFormatter.")] - internal sealed class StringColumnFormatter( - int start, int length, ColumnFormatOptions options, string columnName, RefGetter getter) - : IColumnFormatter - { - public void Format(in TModel model, Span buffer, IFormatProvider? formatProvider) - { - FixedWidthRuntime.FormatString(getter(in model), buffer.Slice(start, length), options, columnName); - } - } -} diff --git a/src/FixedWidthParser/IFixedWidthModel.cs b/src/FixedWidthParser/IFixedWidthModel.cs index ffed152..6114806 100644 --- a/src/FixedWidthParser/IFixedWidthModel.cs +++ b/src/FixedWidthParser/IFixedWidthModel.cs @@ -1,34 +1,28 @@ -using CommunityToolkit.HighPerformance.Buffers; - namespace FixedWidthParser { /// - /// Implemented by models that opt into the compile-time generated parser. Declaring this - /// interface on a model is the trigger for the source generator, which - /// fills in with fully-inlined, reflection-free parsing (mirroring the - /// runtime semantics). Mirrors the shape of - /// so a generic, devirtualized read path is possible. + /// Implemented by models that opt into the compile-time generated parser and writer. Declaring + /// this interface on a model is the trigger for the source generator, + /// which fills in and with reflection-free code + /// mirroring and . + /// Column positions are bytes. /// -#if NET9_0_OR_GREATER public interface IFixedWidthModel where TSelf : IFixedWidthModel, allows ref struct -#else - public interface IFixedWidthModel where TSelf : IFixedWidthModel -#endif { /// - /// Parses a single fixed-width line into . Returns - /// (rejecting the line) when a non-string column fails to parse. + /// Parses a single fixed-width line, without its line break, into . + /// Returns (rejecting the line) when the line is shorter than the layout + /// or a non-string column fails to parse, after reporting why through + /// or . /// - static abstract bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TSelf model); + static abstract bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TSelf model); /// - /// Formats into , mirroring - /// : fills unused portions of each column with its - /// configured padding character, honors per-column alignment/format/overflow, and returns - /// only when is shorter than the model's - /// fixed-width line length (a column value that doesn't fit throws or truncates per its - /// FixedColumnAttribute.Overflow, it does not make this return ). + /// Formats into , without a line break, + /// mirroring . Returns only + /// when is shorter than the line; a value that does not fit its + /// column throws or truncates per its FixedColumnAttribute.Overflow. /// - static abstract bool TryFormat(in TSelf model, Span destination, IFormatProvider? formatProvider, out int charsWritten); + static abstract bool TryFormat(in TSelf model, Span destination, in FixedWidthContext context, out int bytesWritten); } } diff --git a/src/FixedWidthParser/IUtf8FixedWidthModel.cs b/src/FixedWidthParser/IUtf8FixedWidthModel.cs deleted file mode 100644 index ed7af64..0000000 --- a/src/FixedWidthParser/IUtf8FixedWidthModel.cs +++ /dev/null @@ -1,30 +0,0 @@ -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser -{ - /// - /// UTF-8 / byte counterpart of : implemented by models that - /// opt into the compile-time generated byte parser. Declaring this interface on a - /// model is the trigger for the source generator, which fills in - /// with fully-inlined, reflection-free parsing straight from raw UTF-8 bytes - /// (mirroring the runtime semantics). A model may - /// implement this, , or both — the two generated - /// TryParse methods are distinct overloads (one over , one over - /// ) and coexist in the same partial type. - /// Column offsets are measured in bytes, exact for the single-byte/ASCII payloads - /// typical of flat files (see ). - /// -#if NET9_0_OR_GREATER - public interface IUtf8FixedWidthModel where TSelf : IUtf8FixedWidthModel, allows ref struct -#else - public interface IUtf8FixedWidthModel where TSelf : IUtf8FixedWidthModel -#endif - { - /// - /// Parses a single UTF-8 fixed-width line into . Returns - /// (rejecting the line) when the line is shorter (in bytes) than the - /// layout or a non-string column fails to parse. - /// - static abstract bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TSelf model); - } -} diff --git a/src/FixedWidthParser/Parsers/FixedWidthParser.cs b/src/FixedWidthParser/Parsers/FixedWidthParser.cs index f7d31f8..95594aa 100644 --- a/src/FixedWidthParser/Parsers/FixedWidthParser.cs +++ b/src/FixedWidthParser/Parsers/FixedWidthParser.cs @@ -1,33 +1,35 @@ using System.Runtime.ExceptionServices; -using FixedWidthParser.Processors; -using CommunityToolkit.HighPerformance.Buffers; namespace FixedWidthParser.Parsers { -#if NET9_0_OR_GREATER + /// + /// Runtime (reflection-built) parser for one fixed-width line of bytes. Accepts ref struct + /// models; for reading many lines use . + /// Column positions are bytes. + /// public sealed class FixedWidthParser where TModel : new(), allows ref struct -#else - public sealed class FixedWidthParser where TModel : new() -#endif { private static readonly Func _modelFactory; - private static readonly ColumnParserInfo>[] _processors; + private static readonly ColumnParserInfo[] _processors; private static readonly int _requiredLineLength; private static readonly ExceptionDispatchInfo? _buildError; // The build is static (once per type). Any layout/configuration error is captured and // rethrown from the instance constructor, so the caller gets a clean exception instead of // a TypeInitializationException on first use. +#pragma warning disable CA1810, S3963 // The try/catch needs a static constructor. static FixedWidthParser() { try { _modelFactory = ParserBuilder.BuildModelFactory(); - _processors = ParserBuilder.BuildProcessors>(ColumnParserFactory.Create); - _requiredLineLength = ParserBuilder.ComputeRequiredLineLength(_processors); + _processors = ParserBuilder.BuildProcessors(); + _requiredLineLength = ParserBuilder.ComputeRequiredLineLength(_processors); _buildError = null; } +#pragma warning disable CA1031 // Rethrown from the instance constructor. catch (Exception ex) +#pragma warning restore CA1031 { _modelFactory = null!; _processors = []; @@ -35,14 +37,24 @@ static FixedWidthParser() _buildError = ExceptionDispatchInfo.Capture(ex); } } +#pragma warning restore CA1810, S3963 + + /// Builds the parser, throwing when the model's layout is invalid. public FixedWidthParser() { _buildError?.Throw(); } - public bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model) + + /// + /// Parses a single line, without its line break, into . Returns + /// when the line is shorter than the layout or a non-string column fails + /// to parse, after reporting why through . + /// + public bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TModel model) { if (line.Length < _requiredLineLength) { + context.ReportShortLine(_requiredLineLength); model = default!; return false; } @@ -50,9 +62,9 @@ public bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, S model = _modelFactory(); foreach (ref readonly var processor in _processors.AsSpan()) { - var column = line.Slice(processor.Start, processor.Length); - if (!processor.Parse(column, formatProvider, stringPool, ref model)) + if (!processor.Parse(line.Slice(processor.Start, processor.Length), in context, ref model)) { + context.ReportFailedColumn(processor.Name, processor.Start, processor.Length); return false; } } diff --git a/src/FixedWidthParser/Parsers/ParserBuilder.cs b/src/FixedWidthParser/Parsers/ParserBuilder.cs index 6ac3967..79ed001 100644 --- a/src/FixedWidthParser/Parsers/ParserBuilder.cs +++ b/src/FixedWidthParser/Parsers/ParserBuilder.cs @@ -1,63 +1,45 @@ using System.Linq.Expressions; using System.Runtime.InteropServices; using FixedWidthParser.Attributes; +using FixedWidthParser.Processors; namespace FixedWidthParser.Parsers { - /// - /// One column of a built parser layout: the column's range plus the compiled parse-and-assign - /// delegate ( is for - /// the char parser or for the byte parser). - /// + /// One column of a built parser layout: the column's range and name plus its compiled parse-and-assign delegate. [StructLayout(LayoutKind.Auto)] - internal readonly record struct ColumnParserInfo(int Start, int Length, TParser Parse); + internal readonly record struct ColumnParserInfo(int Start, int Length, string Name, ColumnParser Parse) + where TModel : allows ref struct; - /// - /// Shared static-build logic for and - /// : the compiled model factory, the per-column setter - /// compilation, and the required line length. The two parsers differ only in the factory that - /// turns a (member type, compiled setter) pair into a column parser delegate, so that factory is - /// passed into as a delegate. - /// + /// Static-build logic for : the compiled model factory, the per-column setters and the required line length. internal static class ParserBuilder { -#if NET9_0_OR_GREATER public static Func BuildModelFactory() where TModel : new(), allows ref struct -#else - public static Func BuildModelFactory() where TModel : new() -#endif { var lambda = Expression.Lambda>(Expression.New(typeof(TModel))); return lambda.Compile(); } - public static ColumnParserInfo[] BuildProcessors(Func createParser) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif + public static ColumnParserInfo[] BuildProcessors() where TModel : allows ref struct { - List> processors = []; + List> processors = []; ModelColumns.ForEachColumn(typeof(TModel), (member, attribute) => { var (memberType, model, access) = ModelColumns.MemberAccess(typeof(TModel), member); var value = Expression.Parameter(memberType, "value"); var actionType = typeof(RefAction<,>).MakeGenericType(typeof(TModel), memberType); var setter = Expression.Lambda(actionType, Expression.Assign(access, value), model, value).Compile(); - processors.Add(new(attribute.Start, attribute.Length, createParser(memberType, setter, attribute.Converter, member.Name, attribute.TrimChar, attribute.TrimMode, attribute.Format))); + var parse = ColumnParserFactory.Create(memberType, setter, attribute.Converter, member.Name, attribute.TrimChar, attribute.TrimMode, attribute.Format); + processors.Add(new(attribute.Start, attribute.Length, member.Name, parse)); }); return [.. processors]; } - public static int ComputeRequiredLineLength(ReadOnlySpan> processors) + public static int ComputeRequiredLineLength(ReadOnlySpan> processors) where TModel : allows ref struct { int required = 0; foreach (ref readonly var processor in processors) { - int end = processor.Start + processor.Length; - if (end > required) - { - required = end; - } + required = Math.Max(required, processor.Start + processor.Length); } return required; } diff --git a/src/FixedWidthParser/Parsers/Utf8FixedWidthParser.cs b/src/FixedWidthParser/Parsers/Utf8FixedWidthParser.cs deleted file mode 100644 index b3c31b7..0000000 --- a/src/FixedWidthParser/Parsers/Utf8FixedWidthParser.cs +++ /dev/null @@ -1,81 +0,0 @@ -using System.Runtime.ExceptionServices; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Processors; - -namespace FixedWidthParser.Parsers -{ - /// - /// UTF-8 counterpart of : parses a fixed-width line directly - /// from raw of (UTF-8), skipping the - /// UTF-8 → UTF-16 transcode that the char parser pays via a . - /// - /// Column offsets are measured in bytes. This is exact for the single-byte (ASCII) data - /// that fixed-width/flat files almost always use; for content with multi-byte UTF-8 (e.g. accented - /// characters) a byte offset is not the same as a character offset, so this parser is intended for - /// single-byte payloads. Non-string columns parse via a FixedColumnAttribute.Converter, csFastFloat - /// (double/float) or ; string columns are UTF-8 decoded and, when a - /// is supplied, interned through it (otherwise a fresh string per value). - /// - /// -#if NET9_0_OR_GREATER - public sealed class Utf8FixedWidthParser where TModel : new(), allows ref struct -#else - public sealed class Utf8FixedWidthParser where TModel : new() -#endif - { - private static readonly Func _modelFactory; - private static readonly ColumnParserInfo>[] _processors; - private static readonly int _requiredLineLength; - private static readonly ExceptionDispatchInfo? _buildError; - - // The build is static (once per type). Any layout/configuration error is captured and - // rethrown from the instance constructor, so the caller gets a clean exception instead of - // a TypeInitializationException on first use. Mirrors FixedWidthParser. - static Utf8FixedWidthParser() - { - try - { - _modelFactory = ParserBuilder.BuildModelFactory(); - _processors = ParserBuilder.BuildProcessors>(Utf8ColumnParserFactory.Create); - _requiredLineLength = ParserBuilder.ComputeRequiredLineLength(_processors); - _buildError = null; - } - catch (Exception ex) - { - _modelFactory = null!; - _processors = []; - _requiredLineLength = 0; - _buildError = ExceptionDispatchInfo.Capture(ex); - } - } - public Utf8FixedWidthParser() - { - _buildError?.Throw(); - } - /// - /// Parses a single UTF-8 fixed-width line into . Returns - /// when the line is shorter (in bytes) than the configured layout or - /// when a non-string column fails to parse. When is supplied, - /// string columns are interned through it (decoded as UTF-8). - /// - public bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model) - { - if (line.Length < _requiredLineLength) - { - model = default!; - return false; - } - - model = _modelFactory(); - foreach (ref readonly var processor in _processors.AsSpan()) - { - var column = line.Slice(processor.Start, processor.Length); - if (!processor.Parse(column, formatProvider, stringPool, ref model)) - { - return false; - } - } - return true; - } - } -} diff --git a/src/FixedWidthParser/Processors/ColumnParser.cs b/src/FixedWidthParser/Processors/ColumnParser.cs index 7a77146..23f1219 100644 --- a/src/FixedWidthParser/Processors/ColumnParser.cs +++ b/src/FixedWidthParser/Processors/ColumnParser.cs @@ -1,20 +1,9 @@ -using CommunityToolkit.HighPerformance.Buffers; - namespace FixedWidthParser.Processors { /// - /// Parses the already-sliced text of a single column and assigns it to the model. The parser - /// slices each column out of the line (honoring start/length) and hands the resulting span here, - /// so this delegate never deals with offsets. Replaces the old IColumnProcessor - /// abstraction: one delegate per column. Returns to reject the line. + /// Parses the already-sliced bytes of a single column and assigns the value to the model. Returns + /// to reject the line. /// - public delegate bool ColumnParser( - ReadOnlySpan column, - IFormatProvider? formatProvider, - StringPool? stringPool, -#if NET9_0_OR_GREATER - ref TModel model) where TModel : allows ref struct; -#else - ref TModel model); -#endif + internal delegate bool ColumnParser(ReadOnlySpan column, in FixedWidthContext context, ref TModel model) + where TModel : allows ref struct; } diff --git a/src/FixedWidthParser/Processors/ColumnParserFactory.cs b/src/FixedWidthParser/Processors/ColumnParserFactory.cs index 4713474..413d6c7 100644 --- a/src/FixedWidthParser/Processors/ColumnParserFactory.cs +++ b/src/FixedWidthParser/Processors/ColumnParserFactory.cs @@ -1,97 +1,133 @@ -using System.Globalization; using System.Reflection; -using CommunityToolkit.HighPerformance.Buffers; +using FixedWidthParser.Attributes; namespace FixedWidthParser.Processors { /// /// Combines a column's compiled setter and a value parser into a single - /// closure. The column span is sliced by the parser and passed - /// in, so these closures only parse and assign. Resolution order: a FixedColumnAttribute.Converter - /// when set, then string (special-cased: never fails, uses the ), then - /// double/float (csFastFloat), then the fallback. + /// closure that calls the same + /// helpers as generated code. Resolution order: a FixedColumnAttribute.Converter, a date or + /// time with a Format, string, double/float, then + /// , then over the decoded text. /// internal static class ColumnParserFactory { - private static readonly MethodInfo BuildParsableMethod = - typeof(ColumnParserFactory).GetMethod(nameof(BuildParsable), BindingFlags.NonPublic | BindingFlags.Static)!; - private static readonly MethodInfo BuildConverterMethod = - typeof(ColumnParserFactory).GetMethod(nameof(BuildConverter), BindingFlags.NonPublic | BindingFlags.Static)!; - private static readonly MethodInfo BuildDoubleMethod = - typeof(ColumnParserFactory).GetMethod(nameof(BuildDouble), BindingFlags.NonPublic | BindingFlags.Static)!; - private static readonly MethodInfo BuildFloatMethod = - typeof(ColumnParserFactory).GetMethod(nameof(BuildFloat), BindingFlags.NonPublic | BindingFlags.Static)!; - private static readonly MethodInfo BuildNullableMethod = - typeof(ColumnParserFactory).GetMethod(nameof(BuildNullable), BindingFlags.NonPublic | BindingFlags.Static)!; - - /// - /// Creates a for the given member type and setter. The setter is a compiled expression assigning the parsed value to the member. - /// When is a , a blank (trimmed-empty) column assigns - /// without invoking the underlying parser; otherwise resolution for the underlying T - /// proceeds exactly as for a non-nullable column. When is set (from - /// FixedColumnAttribute.Converter), it takes priority over the fallback. - /// - /// must be a RefAction<TModel, valueType> assigning the parsed value to the member. - public static ColumnParser Create(Type valueType, Delegate setter, Type? converterType, string memberName, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing, string? format = null) -#if NET9_0_OR_GREATER + private const BindingFlags Private = BindingFlags.NonPublic | BindingFlags.Static; + + public static ColumnParser Create(Type valueType, Delegate setter, Type? converterType, string memberName, char trimChar, TrimMode trimMode, string? format) where TModel : allows ref struct -#endif { + byte trim = FixedWidthRuntime.ToAsciiByte(trimChar, memberName, nameof(FixedColumnAttribute.TrimChar)); var underlyingType = Nullable.GetUnderlyingType(valueType); if (underlyingType is not null) { - return (ColumnParser)BuildNullableMethod.MakeGenericMethod(typeof(TModel), underlyingType) - .Invoke(null, [setter, converterType, memberName, trimChar, trimMode, format])!; + return (ColumnParser)Method(nameof(BuildNullable), typeof(TModel), underlyingType) + .Invoke(null, [setter, converterType, memberName, trim, trimMode, format])!; + } + return CreateValue(valueType, setter, converterType, memberName, trim, trimMode, format); + } + + private static ColumnParser CreateValue(Type valueType, Delegate setter, Type? converterType, string memberName, byte trim, TrimMode trimMode, string? format) + where TModel : allows ref struct + { + if (converterType is not null) + { + var requiredInterface = typeof(IFixedWidthConverter<>).MakeGenericType(valueType); + if (!requiredInterface.IsAssignableFrom(converterType)) + { + throw new InvalidOperationException( + $"Converter '{converterType}' for column '{memberName}' must implement '{requiredInterface}'."); + } + var converter = Activator.CreateInstance(converterType) + ?? throw new InvalidOperationException($"Could not instantiate converter '{converterType}' for column '{memberName}'."); + return (ColumnParser)Method(nameof(BuildConverter), typeof(TModel), valueType, converterType) + .Invoke(null, [setter, converter, trim, trimMode])!; + } + + if (format is not null) + { + if (valueType == typeof(DateTime)) + { + return BuildDateTime((RefAction)setter, format, trim, trimMode); + } + if (valueType == typeof(DateOnly)) + { + return BuildDateOnly((RefAction)setter, format, trim, trimMode); + } + if (valueType == typeof(TimeOnly)) + { + return BuildTimeOnly((RefAction)setter, format, trim, trimMode); + } + if (valueType == typeof(DateTimeOffset)) + { + return BuildDateTimeOffset((RefAction)setter, format, trim, trimMode); + } } - return ColumnParserFactoryShared.Create>( - valueType, setter, converterType, memberName, trimChar, trimMode, format, typeof(IFixedWidthConverter<>), - BuildParsableMethod, BuildConverterMethod, BuildDoubleMethod, BuildFloatMethod, BuildString, BuildExact); + if (valueType == typeof(string)) + { + return BuildString((RefAction)setter, trim, trimMode); + } + if (valueType == typeof(double)) + { + return BuildDouble((RefAction)setter, trim, trimMode); + } + if (valueType == typeof(float)) + { + return BuildFloat((RefAction)setter, trim, trimMode); + } + if (Implements(valueType, typeof(IUtf8SpanParsable<>))) + { + return (ColumnParser)Method(nameof(BuildUtf8Parsable), typeof(TModel), valueType).Invoke(null, [setter, trim, trimMode])!; + } + if (Implements(valueType, typeof(ISpanParsable<>))) + { + return (ColumnParser)Method(nameof(BuildDecodedParsable), typeof(TModel), valueType).Invoke(null, [setter, trim, trimMode])!; + } + + throw new InvalidOperationException( + $"Column '{memberName}' has type '{valueType}', which is not string, double, float, IUtf8SpanParsable or ISpanParsable; set a Converter for it."); } - // A nullable column (T?): a blank (trimmed-empty) column assigns null without invoking the - // underlying parser. Otherwise, the underlying T resolves exactly as a non-nullable column - // would (converter → double/float → ISpanParsable fallback), via an adapter setter that boxes - // the result into the T? setter — no duplicated parse logic per type. - private static ColumnParser BuildNullable(Delegate setter, Type? converterType, string memberName, char trimChar, Attributes.TrimMode trimMode, string? format) -#if NET9_0_OR_GREATER + private static bool Implements(Type valueType, Type openInterface) + { + // Not MakeGenericType: it throws for a type that breaks the interface's self-constraint. + return Array.Exists(valueType.GetInterfaces(), i => + i.IsGenericType && i.GetGenericTypeDefinition() == openInterface && i.GenericTypeArguments[0] == valueType); + } + + private static MethodInfo Method(string name, params Type[] typeArguments) + { + return typeof(ColumnParserFactory).GetMethod(name, Private)!.MakeGenericMethod(typeArguments); + } + + // A nullable column (T?): a blank column assigns null without invoking the underlying parser. + private static ColumnParser BuildNullable(Delegate setter, Type? converterType, string memberName, byte trim, TrimMode trimMode, string? format) where TModel : allows ref struct -#endif where TUnderlying : struct { var nullableSetter = (RefAction)setter; RefAction adapted = (ref TModel model, TUnderlying value) => nullableSetter(ref model, value); + var inner = CreateValue(typeof(TUnderlying), adapted, converterType, memberName, trim, trimMode, format); - var inner = ColumnParserFactoryShared.Create>( - typeof(TUnderlying), adapted, converterType, memberName, trimChar, trimMode, format, typeof(IFixedWidthConverter<>), - BuildParsableMethod, BuildConverterMethod, BuildDoubleMethod, BuildFloatMethod, BuildString, BuildExact); - - return (column, formatProvider, stringPool, ref model) => + return (column, in context, ref model) => { - if (column.Trim(trimChar).Trim(' ').IsEmpty) + if (FixedWidthRuntime.IsBlank(column, trim)) { nullableSetter(ref model, null); return true; } - return inner(column, formatProvider, stringPool, ref model); + return inner(column, in context, ref model); }; } - // Fallback for types with no dedicated builder: call the static-abstract ISpanParsable.TryParse - // directly inside the column closure. This is a constrained call the JIT devirtualizes to the - // concrete TValue.TryParse (and can inline), saving one indirect call per column and no - // value-parser delegate allocated. - private static ColumnParser BuildParsable(RefAction setter, char trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER + private static ColumnParser BuildUtf8Parsable(RefAction setter, byte trim, TrimMode trimMode) where TModel : allows ref struct -#endif - where TValue : ISpanParsable + where TValue : IUtf8SpanParsable { - return (column, formatProvider, _, ref model) => + return (column, in context, ref model) => { - // Null means invariant (matches FixedWidthRuntime.TryParse and the double/float - // columns) rather than the BCL's own CurrentCulture default. - if (!TValue.TryParse(FixedWidthRuntime.TrimColumn(column, trimChar, trimMode), formatProvider ?? CultureInfo.InvariantCulture, out var value)) + if (!FixedWidthRuntime.TryParse(column, in context, out var value, trim, trimMode)) { return false; } @@ -100,14 +136,13 @@ private static ColumnParser BuildParsable(RefAction BuildDouble(RefAction setter, char trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER + private static ColumnParser BuildDecodedParsable(RefAction setter, byte trim, TrimMode trimMode) where TModel : allows ref struct -#endif + where TValue : ISpanParsable { - return (column, formatProvider, _, ref model) => + return (column, in context, ref model) => { - if (!CultureHelpers.TryParseDouble(column, formatProvider, out double value, trimChar, trimMode)) + if (!FixedWidthRuntime.TryParseDecoded(column, in context, out var value, trim, trimMode)) { return false; } @@ -116,14 +151,13 @@ private static ColumnParser BuildDouble(RefAction BuildFloat(RefAction setter, char trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER + private static ColumnParser BuildConverter(RefAction setter, TConverter converter, byte trim, TrimMode trimMode) where TModel : allows ref struct -#endif + where TConverter : IFixedWidthConverter { - return (column, formatProvider, _, ref model) => + return (column, in context, ref model) => { - if (!CultureHelpers.TryParseFloat(column, formatProvider, out float value, trimChar, trimMode)) + if (!FixedWidthRuntime.TryConvert(column, in context, converter, out var value, trim, trimMode)) { return false; } @@ -132,18 +166,22 @@ private static ColumnParser BuildFloat(RefAction }; } - // Attribute-driven custom converter (FixedColumnAttribute.Converter): the instance is created - // once (by the caller) and reused for every row, so it must be stateless. - private static ColumnParser BuildConverter( - RefAction setter, TConverter converter, char trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER + private static ColumnParser BuildString(RefAction setter, byte trim, TrimMode trimMode) where TModel : allows ref struct -#endif - where TConverter : IFixedWidthConverter { - return (column, formatProvider, _, ref model) => + return (column, in context, ref model) => { - if (!converter.TryParse(FixedWidthRuntime.TrimColumn(column, trimChar, trimMode), formatProvider, out var value)) + setter(ref model, FixedWidthRuntime.String(column, in context, trim, trimMode)); + return true; + }; + } + + private static ColumnParser BuildDouble(RefAction setter, byte trim, TrimMode trimMode) + where TModel : allows ref struct + { + return (column, in context, ref model) => + { + if (!FixedWidthRuntime.TryDouble(column, in context, out double value, trim, trimMode)) { return false; } @@ -152,56 +190,68 @@ private static ColumnParser BuildConverter( }; } - private static ColumnParser BuildString(RefAction setter, object trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER + private static ColumnParser BuildFloat(RefAction setter, byte trim, TrimMode trimMode) where TModel : allows ref struct -#endif { - char trim = (char)trimChar; - return (column, _, stringPool, ref model) => + return (column, in context, ref model) => { - var slice = FixedWidthRuntime.TrimColumn(column, trim, trimMode); - setter(ref model, stringPool is null ? slice.ToString() : stringPool.GetOrAdd(slice)); + if (!FixedWidthRuntime.TryFloat(column, in context, out float value, trim, trimMode)) + { + return false; + } + setter(ref model, value); return true; }; } - private static ColumnParser BuildExact(Type valueType, Delegate setter, string format, object trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER + private static ColumnParser BuildDateTime(RefAction setter, string format, byte trim, TrimMode trimMode) where TModel : allows ref struct -#endif { - char trim = (char)trimChar; - if (valueType == typeof(DateTime)) + return (column, in context, ref model) => { - return BuildExact((RefAction)setter, format, trim, trimMode, DateTime.TryParseExact); - } - if (valueType == typeof(DateOnly)) - { - return BuildExact((RefAction)setter, format, trim, trimMode, DateOnly.TryParseExact); - } - if (valueType == typeof(TimeOnly)) - { - return BuildExact((RefAction)setter, format, trim, trimMode, TimeOnly.TryParseExact); - } - if (valueType == typeof(DateTimeOffset)) + if (!FixedWidthRuntime.TryDateTimeExact(column, format, in context, out var value, trim, trimMode)) + { + return false; + } + setter(ref model, value); + return true; + }; + } + + private static ColumnParser BuildDateOnly(RefAction setter, string format, byte trim, TrimMode trimMode) + where TModel : allows ref struct + { + return (column, in context, ref model) => { - return BuildExact((RefAction)setter, format, trim, trimMode, DateTimeOffset.TryParseExact); - } - throw new InvalidOperationException($"Unsupported TryParseExact type {valueType}"); + if (!FixedWidthRuntime.TryDateOnlyExact(column, format, in context, out var value, trim, trimMode)) + { + return false; + } + setter(ref model, value); + return true; + }; } - private delegate bool ExactParser(ReadOnlySpan value, ReadOnlySpan format, IFormatProvider? formatProvider, DateTimeStyles styles, out TValue result); + private static ColumnParser BuildTimeOnly(RefAction setter, string format, byte trim, TrimMode trimMode) + where TModel : allows ref struct + { + return (column, in context, ref model) => + { + if (!FixedWidthRuntime.TryTimeOnlyExact(column, format, in context, out var value, trim, trimMode)) + { + return false; + } + setter(ref model, value); + return true; + }; + } - private static ColumnParser BuildExact(RefAction setter, string format, char trim, Attributes.TrimMode trimMode, ExactParser parser) -#if NET9_0_OR_GREATER + private static ColumnParser BuildDateTimeOffset(RefAction setter, string format, byte trim, TrimMode trimMode) where TModel : allows ref struct -#endif { - return (column, fp, _, ref model) => + return (column, in context, ref model) => { - var trimmed = FixedWidthRuntime.TrimColumn(column, trim, trimMode); - if (!parser(trimmed, format, fp ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out var value)) + if (!FixedWidthRuntime.TryDateTimeOffsetExact(column, format, in context, out var value, trim, trimMode)) { return false; } diff --git a/src/FixedWidthParser/Processors/ColumnParserFactoryShared.cs b/src/FixedWidthParser/Processors/ColumnParserFactoryShared.cs deleted file mode 100644 index 49c30ef..0000000 --- a/src/FixedWidthParser/Processors/ColumnParserFactoryShared.cs +++ /dev/null @@ -1,74 +0,0 @@ -using System.Reflection; - -namespace FixedWidthParser.Processors -{ - /// - /// Shared build-time orchestration for (char) and - /// (byte): the identical "converter → string special-case → - /// double/float (csFastFloat) special-case → / - /// fallback → wrap setter+parser" flow. The element-specific pieces (string/double/float/converter - /// builders and the reflected build-parsable method) are passed in, so the spans-of-different-element-type - /// closures stay in each factory while the reflection plumbing lives in one place. - /// - internal static class ColumnParserFactoryShared - { - public static TColumnParser Create( - Type valueType, - Delegate setter, - Type? converterType, - string memberName, - object trimChar, - Attributes.TrimMode trimMode, - string? format, - Type converterInterfaceDefinition, - MethodInfo buildParsableMethod, - MethodInfo buildConverterMethod, - MethodInfo buildDoubleMethod, - MethodInfo buildFloatMethod, - Func, object, Attributes.TrimMode, TColumnParser> buildString, - Func buildExact) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - where TColumnParser : Delegate - { - if (converterType is not null) - { - var requiredInterface = converterInterfaceDefinition.MakeGenericType(valueType); - if (!requiredInterface.IsAssignableFrom(converterType)) - { - throw new InvalidOperationException( - $"Converter '{converterType}' for column '{memberName}' must implement '{requiredInterface}'."); - } - var converterInstance = Activator.CreateInstance(converterType) - ?? throw new InvalidOperationException($"Could not instantiate converter '{converterType}' for column '{memberName}'."); - return (TColumnParser)buildConverterMethod.MakeGenericMethod(typeof(TModel), valueType, converterType) - .Invoke(null, [setter, converterInstance, trimChar, trimMode])!; - } - - if (format is not null && (valueType == typeof(DateTime) || valueType == typeof(DateOnly) || valueType == typeof(TimeOnly) || valueType == typeof(DateTimeOffset))) - { - return buildExact(valueType, setter, format, trimChar, trimMode); - } - - if (valueType == typeof(string)) - { - return buildString((RefAction)setter, trimChar, trimMode); - } - - // double/float take the csFastFloat fast path ahead of the ISpanParsable fallback (both - // implement ISpanParsable/IUtf8SpanParsable too, but csFastFloat is faster). - if (valueType == typeof(double)) - { - return (TColumnParser)buildDoubleMethod.MakeGenericMethod(typeof(TModel)).Invoke(null, [setter, trimChar, trimMode])!; - } - if (valueType == typeof(float)) - { - return (TColumnParser)buildFloatMethod.MakeGenericMethod(typeof(TModel)).Invoke(null, [setter, trimChar, trimMode])!; - } - - return (TColumnParser)buildParsableMethod.MakeGenericMethod(typeof(TModel), valueType) - .Invoke(null, [setter, trimChar, trimMode])!; - } - } -} diff --git a/src/FixedWidthParser/Processors/CultureHelpers.cs b/src/FixedWidthParser/Processors/CultureHelpers.cs index 9d3e65e..bed26ac 100644 --- a/src/FixedWidthParser/Processors/CultureHelpers.cs +++ b/src/FixedWidthParser/Processors/CultureHelpers.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Diagnostics.CodeAnalysis; using System.Globalization; +using System.Numerics; using System.Runtime.CompilerServices; using System.Text; using csFastFloat; @@ -8,15 +9,11 @@ namespace FixedWidthParser.Processors { /// - /// Culture-aware double/float parsing for the double/float column builders (char and - /// UTF-8). csFastFloat is only trusted for the invariant '.' decimal separator, and even then only - /// after checking every character of the (trimmed) field was consumed — csFastFloat does not fail - /// on trailing content it doesn't recognize, it silently stops and returns the leading prefix it - /// did parse (so e.g. a thousands separator it wasn't told about would otherwise truncate "1.234,56" - /// down to "1" and still report success). Any other separator falls back to the real, - /// -aware , - /// which csFastFloat's single decimal_separator override cannot express correctly (thousands - /// separators, sign placement, …) and which validates the whole input natively. + /// Culture-aware double/float parsing. csFastFloat is only trusted for the '.' decimal + /// separator, and even then only after checking every byte of the (trimmed) field was consumed — + /// csFastFloat does not fail on trailing content it doesn't recognize, it silently stops and returns + /// the leading prefix it did parse. Any other separator, and a dot-culture value using thousands + /// separators, falls back to the -aware BCL parser over the decoded text. /// internal static class CultureHelpers { @@ -26,28 +23,16 @@ internal static class CultureHelpers // no thrash when two providers are used alternately in the same process. private static readonly ConditionalWeakTable> _separatorCache = new(); - private sealed class CacheEntry + private sealed class CacheEntry(IFormatProvider provider, char separator) { - public readonly IFormatProvider Provider; - public readonly char Separator; - public CacheEntry(IFormatProvider provider, char separator) - { - Provider = provider; - Separator = separator; - } + public readonly IFormatProvider Provider = provider; + public readonly char Separator = separator; } private static CacheEntry? _cache; - /// - /// Decimal separator derived from the IFormatProvider (dot when null), memoized per provider. - /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static char GetDecimalSeparator(IFormatProvider? formatProvider) + public static char GetDecimalSeparator(IFormatProvider formatProvider) { - if (formatProvider is null) - { - return '.'; - } var cache = _cache; if (cache is not null && ReferenceEquals(formatProvider, cache.Provider)) { @@ -58,162 +43,60 @@ public static char GetDecimalSeparator(IFormatProvider? formatProvider) return separator; } - /// - /// Decimal separator as a single UTF-8 byte, for the byte-based numeric parsers. Only an ASCII - /// separator (<= 0x7F) can be represented as one byte; a non-ASCII separator would be a - /// multi-byte sequence in the UTF-8 data and is rejected with a clear error rather than silently - /// truncated to the wrong byte. - /// - /// The culture's decimal separator is not a single ASCII character. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static byte GetDecimalSeparatorByte(IFormatProvider? formatProvider) - { - char separator = GetDecimalSeparator(formatProvider); - if (separator > '\x7F') - { - ThrowNonAsciiSeparator(separator); - } - return (byte)separator; - } - - [MethodImpl(MethodImplOptions.NoInlining)] - [DoesNotReturn] - private static void ThrowNonAsciiSeparator(char separator) - { - throw new NotSupportedException( - $"The decimal separator '{separator}' (U+{(int)separator:X4}) is not a single ASCII byte. " + - "The UTF-8 byte parser only supports ASCII decimal separators; use the char-based parser for this culture."); - } - [MethodImpl(MethodImplOptions.NoInlining)] private static char Resolve(IFormatProvider formatProvider) { return NumberFormatInfo.GetInstance(formatProvider).NumberDecimalSeparator[0]; } - /// - /// Parses a (not yet trimmed) column as , honoring . - /// A provider means — a fixed-width - /// file is a machine layout, not UI text, so a caller that doesn't pass a provider should not be at - /// the mercy of the running thread's (which every other - /// column type, and this method's own BCL fallback below, would otherwise resolve to). - /// - public static bool TryParseDouble(ReadOnlySpan column, IFormatProvider? formatProvider, out double value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - // Also trim leading whitespace: csFastFloat's characters_consumed does not reliably include - // skipped leading whitespace across its double/float overloads (verified empirically — - // FastDoubleParser counts it, FastFloatParser doesn't), so a right-aligned (space-padded) - // column would otherwise fail the full-consumption check below even though the BCL fallback - // (which does allow leading whitespace) would have accepted it. - var trimmed = FixedWidthRuntime.TrimColumn(column, trimChar, trimMode).TrimStart(); - if (GetDecimalSeparator(formatProvider) == '.') - { - if (FastDoubleParser.TryParseDouble(trimmed, out int consumed, out value, decimal_separator: '.') - && consumed == trimmed.Length) - { - return true; - } - // csFastFloat's single decimal_separator override can't express thousands separators or - // sign placement; a dot-culture value using them (e.g. "1,234.56") fails the fast path - // above even though it's valid, so fall back to the real NumberFormatInfo-aware parser, - // which validates the whole input natively (still rejects genuine garbage like "12x"). - return double.TryParse(trimmed, NumberStyles.Float | NumberStyles.AllowThousands, formatProvider ?? CultureInfo.InvariantCulture, out value); - } - return double.TryParse(trimmed, NumberStyles.Float | NumberStyles.AllowThousands, formatProvider, out value); - } - - /// Parses a (not yet trimmed) column as , honoring (see for the null-provider contract). - public static bool TryParseFloat(ReadOnlySpan column, IFormatProvider? formatProvider, out float value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) + /// Parses a (not yet trimmed) column as , honoring the context's provider. + public static bool TryParseDouble(ReadOnlySpan column, in FixedWidthContext context, out double value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) { - var trimmed = FixedWidthRuntime.TrimColumn(column, trimChar, trimMode).TrimStart(); - if (GetDecimalSeparator(formatProvider) == '.') - { - if (FastFloatParser.TryParseFloat(trimmed, out int consumed, out value, decimal_separator: '.') - && consumed == trimmed.Length) - { - return true; - } - return float.TryParse(trimmed, NumberStyles.Float | NumberStyles.AllowThousands, formatProvider ?? CultureInfo.InvariantCulture, out value); - } - return float.TryParse(trimmed, NumberStyles.Float | NumberStyles.AllowThousands, formatProvider, out value); - } - - /// - /// UTF-8 counterpart of . - /// A non-ASCII separator still throws (see ) — the byte path - /// only supports separators representable as a single UTF-8 byte. - /// - public static bool TryParseDouble(ReadOnlySpan column, IFormatProvider? formatProvider, out double value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - byte separator = GetDecimalSeparatorByte(formatProvider); - var trimmed = Utf8FixedWidthRuntime.TrimColumn(column, trimChar, trimMode).TrimStart((byte)' '); - if (separator == (byte)'.' - && FastDoubleParser.TryParseDouble(trimmed, out int consumed, out value, decimal_separator: separator) + // Also trim leading spaces: csFastFloat's characters_consumed does not reliably include + // skipped leading whitespace, so a right-aligned column would otherwise fail the + // full-consumption check below. + var trimmed = FixedWidthRuntime.TrimColumn(column, trimChar, trimMode).TrimStart((byte)' '); + var formatProvider = context.FormatProvider; + if (GetDecimalSeparator(formatProvider) == '.' + && FastDoubleParser.TryParseDouble(trimmed, out int consumed, out value, decimal_separator: (byte)'.') && consumed == trimmed.Length) { return true; } - return TryParseDoubleViaTranscode(trimmed, formatProvider, out value); + return TryParseDecoded(trimmed, formatProvider, context.Encoding, out value); } - /// UTF-8 counterpart of . - public static bool TryParseFloat(ReadOnlySpan column, IFormatProvider? formatProvider, out float value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) + /// Parses a (not yet trimmed) column as , honoring the context's provider. + public static bool TryParseFloat(ReadOnlySpan column, in FixedWidthContext context, out float value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) { - byte separator = GetDecimalSeparatorByte(formatProvider); - var trimmed = Utf8FixedWidthRuntime.TrimColumn(column, trimChar, trimMode).TrimStart((byte)' '); - if (separator == (byte)'.' - && FastFloatParser.TryParseFloat(trimmed, out int consumed, out value, decimal_separator: separator) + var trimmed = FixedWidthRuntime.TrimColumn(column, trimChar, trimMode).TrimStart((byte)' '); + var formatProvider = context.FormatProvider; + if (GetDecimalSeparator(formatProvider) == '.' + && FastFloatParser.TryParseFloat(trimmed, out int consumed, out value, decimal_separator: (byte)'.') && consumed == trimmed.Length) { return true; } - return TryParseFloatViaTranscode(trimmed, formatProvider, out value); - } - - // Non-dot ASCII separators (e.g. ',') still need NumberFormatInfo-aware parsing (thousands - // separators, sign placement); there is no BCL span-based double.TryParse that both takes an - // IFormatProvider and operates on UTF-8 bytes, so the (already-trimmed, always-short) numeric - // field is transcoded to a small char buffer first. Also used as the dot-path's thousands-separator - // fallback above, so a null provider is normalized to invariant here too (see TryParseDouble's doc). - [SkipLocalsInit] - private static bool TryParseDoubleViaTranscode(ReadOnlySpan field, IFormatProvider? formatProvider, out double value) - { - formatProvider ??= CultureInfo.InvariantCulture; - const int stackThreshold = 128; - if (field.Length <= stackThreshold) - { - Span chars = stackalloc char[field.Length]; - int written = Encoding.UTF8.GetChars(field, chars); - return double.TryParse(chars[..written], NumberStyles.Float | NumberStyles.AllowThousands, formatProvider, out value); - } - char[] rented = ArrayPool.Shared.Rent(field.Length); - try - { - int written = Encoding.UTF8.GetChars(field, rented); - return double.TryParse(rented.AsSpan(0, written), NumberStyles.Float | NumberStyles.AllowThousands, formatProvider, out value); - } - finally - { - ArrayPool.Shared.Return(rented); - } + return TryParseDecoded(trimmed, formatProvider, context.Encoding, out value); } [SkipLocalsInit] - private static bool TryParseFloatViaTranscode(ReadOnlySpan field, IFormatProvider? formatProvider, out float value) + private static bool TryParseDecoded(ReadOnlySpan field, IFormatProvider formatProvider, Encoding encoding, [MaybeNullWhen(false)] out TValue value) + where TValue : INumberBase { - formatProvider ??= CultureInfo.InvariantCulture; - const int stackThreshold = 128; - if (field.Length <= stackThreshold) + const NumberStyles styles = NumberStyles.Float | NumberStyles.AllowThousands; + int max = encoding.GetMaxCharCount(field.Length); + if (max <= 128) { - Span chars = stackalloc char[field.Length]; - int written = Encoding.UTF8.GetChars(field, chars); - return float.TryParse(chars[..written], NumberStyles.Float | NumberStyles.AllowThousands, formatProvider, out value); + Span chars = stackalloc char[128]; + int written = encoding.GetChars(field, chars); + return TValue.TryParse(chars[..written], styles, formatProvider, out value); } - char[] rented = ArrayPool.Shared.Rent(field.Length); + char[] rented = ArrayPool.Shared.Rent(max); try { - int written = Encoding.UTF8.GetChars(field, rented); - return float.TryParse(rented.AsSpan(0, written), NumberStyles.Float | NumberStyles.AllowThousands, formatProvider, out value); + int written = encoding.GetChars(field, rented); + return TValue.TryParse(rented.AsSpan(0, written), styles, formatProvider, out value); } finally { diff --git a/src/FixedWidthParser/Processors/IFixedWidthConverter.cs b/src/FixedWidthParser/Processors/IFixedWidthConverter.cs index eddea80..6816795 100644 --- a/src/FixedWidthParser/Processors/IFixedWidthConverter.cs +++ b/src/FixedWidthParser/Processors/IFixedWidthConverter.cs @@ -6,22 +6,21 @@ namespace FixedWidthParser.Processors /// model's parser/writer is built) and reused for every row, so implementations must be stateless. /// One interface covers both directions: a fixed-width column always round-trips. /// + /// A later version never adds a member to this interface; new settings arrive on . public interface IFixedWidthConverter { - /// Parses the already-sliced, trimmed column text. Returns to reject the line. - bool TryParse(ReadOnlySpan field, IFormatProvider? formatProvider, out T value); + /// + /// Parses the already-sliced, trimmed column bytes. Decode text with + /// . Returns to reject the line. + /// + bool TryParse(ReadOnlySpan field, in FixedWidthContext context, out T value); - /// Formats into , returning the number of characters written. - bool TryFormat(T value, Span destination, IFormatProvider? formatProvider, out int written); - } - - /// UTF-8 counterpart of , used by the byte reader/writer. - public interface IUtf8FixedWidthConverter - { - /// Parses the already-sliced, trimmed column bytes. Returns to reject the line. - bool TryParse(ReadOnlySpan field, IFormatProvider? formatProvider, out T value); - - /// Formats into , returning the number of bytes written. - bool TryFormat(T value, Span destination, IFormatProvider? formatProvider, out int written); + /// + /// Formats into , encoding text with + /// , and returns the number of bytes written. Returns + /// when is too small; the writer retries + /// with a larger buffer. + /// + bool TryFormat(T value, Span destination, in FixedWidthContext context, out int written); } } diff --git a/src/FixedWidthParser/Processors/Utf8ColumnParser.cs b/src/FixedWidthParser/Processors/Utf8ColumnParser.cs deleted file mode 100644 index 4775cb9..0000000 --- a/src/FixedWidthParser/Processors/Utf8ColumnParser.cs +++ /dev/null @@ -1,26 +0,0 @@ -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser.Processors -{ - /// - /// UTF-8 counterpart of : parses the already-sliced - /// bytes of a single column and assigns it to the model. The byte parser slices each - /// column out of the raw UTF-8 line (honoring start/length, measured in bytes) and hands - /// the resulting span here, so this delegate never deals with offsets. Returns - /// to reject the line. - /// - /// A may be supplied to intern string columns: they are decoded via - /// (UTF-8), so a - /// repeated value is decoded once and the same instance is reused. - /// - /// - public delegate bool Utf8ColumnParser( - ReadOnlySpan column, - IFormatProvider? formatProvider, - StringPool? stringPool, -#if NET9_0_OR_GREATER - ref TModel model) where TModel : allows ref struct; -#else - ref TModel model); -#endif -} diff --git a/src/FixedWidthParser/Processors/Utf8ColumnParserFactory.cs b/src/FixedWidthParser/Processors/Utf8ColumnParserFactory.cs deleted file mode 100644 index 0f0c07f..0000000 --- a/src/FixedWidthParser/Processors/Utf8ColumnParserFactory.cs +++ /dev/null @@ -1,219 +0,0 @@ -using System.Globalization; -using System.Reflection; -using System.Text; - -namespace FixedWidthParser.Processors -{ - /// - /// UTF-8 counterpart of : combines a column's compiled setter and - /// a UTF-8 value parser into a single closure. The column - /// span is sliced (in bytes) by the parser and passed in, so these closures only parse and assign. - /// Resolution order: a FixedColumnAttribute.Converter when set, then string (special-cased: - /// never fails, UTF-8 decoded, interned through the supplied - /// when one is provided), then double/float (csFastFloat), then the - /// fallback. - /// - internal static class Utf8ColumnParserFactory - { - private static readonly MethodInfo BuildParsableMethod = - typeof(Utf8ColumnParserFactory).GetMethod(nameof(BuildParsable), BindingFlags.NonPublic | BindingFlags.Static)!; - private static readonly MethodInfo BuildConverterMethod = - typeof(Utf8ColumnParserFactory).GetMethod(nameof(BuildConverter), BindingFlags.NonPublic | BindingFlags.Static)!; - private static readonly MethodInfo BuildDoubleMethod = - typeof(Utf8ColumnParserFactory).GetMethod(nameof(BuildDouble), BindingFlags.NonPublic | BindingFlags.Static)!; - private static readonly MethodInfo BuildFloatMethod = - typeof(Utf8ColumnParserFactory).GetMethod(nameof(BuildFloat), BindingFlags.NonPublic | BindingFlags.Static)!; - private static readonly MethodInfo BuildNullableMethod = - typeof(Utf8ColumnParserFactory).GetMethod(nameof(BuildNullable), BindingFlags.NonPublic | BindingFlags.Static)!; - - /// - /// Creates a for the given member type and setter. The - /// setter is a compiled expression assigning the parsed value to the member. - /// When is a , a blank (trimmed-empty) column assigns - /// without invoking the underlying parser; otherwise resolution for the underlying T - /// proceeds exactly as for a non-nullable column. When is set (from - /// FixedColumnAttribute.Converter), it takes priority over the fallback. - /// - /// must be a RefAction<TModel, valueType> assigning the parsed value to the member. - public static Utf8ColumnParser Create(Type valueType, Delegate setter, Type? converterType, string memberName, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing, string? format = null) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - { - byte trimByte = Utf8FixedWidthRuntime.ToAsciiByte(trimChar, memberName); - var underlyingType = Nullable.GetUnderlyingType(valueType); - if (underlyingType is not null) - { - return (Utf8ColumnParser)BuildNullableMethod.MakeGenericMethod(typeof(TModel), underlyingType) - .Invoke(null, [setter, converterType, memberName, trimByte, trimMode, format])!; - } - - return ColumnParserFactoryShared.Create>( - valueType, setter, converterType, memberName, trimByte, trimMode, format, typeof(IUtf8FixedWidthConverter<>), - BuildParsableMethod, BuildConverterMethod, BuildDoubleMethod, BuildFloatMethod, BuildString, BuildExact); - } - - // A nullable column (T?): a blank (trimmed-empty) column assigns null without invoking the - // underlying parser. Otherwise, the underlying T resolves exactly as a non-nullable column - // would (converter → double/float → IUtf8SpanParsable fallback), via an adapter setter that - // boxes the result into the T? setter — no duplicated parse logic per type. - private static Utf8ColumnParser BuildNullable(Delegate setter, Type? converterType, string memberName, byte trimChar, Attributes.TrimMode trimMode, string? format) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - where TUnderlying : struct - { - var nullableSetter = (RefAction)setter; - RefAction adapted = (ref TModel model, TUnderlying value) => nullableSetter(ref model, value); - - var inner = ColumnParserFactoryShared.Create>( - typeof(TUnderlying), adapted, converterType, memberName, trimChar, trimMode, format, typeof(IUtf8FixedWidthConverter<>), - BuildParsableMethod, BuildConverterMethod, BuildDoubleMethod, BuildFloatMethod, BuildString, BuildExact); - - return (column, formatProvider, stringPool, ref model) => - { - if (column.Trim(trimChar).Trim((byte)' ').IsEmpty) - { - nullableSetter(ref model, null); - return true; - } - return inner(column, formatProvider, stringPool, ref model); - }; - } - - // Fallback for types with no dedicated builder: call the static-abstract IUtf8SpanParsable.TryParse - // directly inside the column closure. This is a constrained call the JIT devirtualizes to the - // concrete TValue.TryParse (and can inline), saving one indirect call per column and no - // value-parser delegate allocated. - private static Utf8ColumnParser BuildParsable(RefAction setter, byte trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - where TValue : IUtf8SpanParsable - { - return (column, formatProvider, _, ref model) => - { - // Null means invariant (matches Utf8FixedWidthRuntime.TryParse and the double/float - // columns) rather than the BCL's own CurrentCulture default. - if (!TValue.TryParse(Utf8FixedWidthRuntime.TrimColumn(column, trimChar, trimMode), formatProvider ?? CultureInfo.InvariantCulture, out var value)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - - private static Utf8ColumnParser BuildDouble(RefAction setter, byte trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - { - return (column, formatProvider, _, ref model) => - { - if (!CultureHelpers.TryParseDouble(column, formatProvider, out double value, trimChar, trimMode)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - - private static Utf8ColumnParser BuildFloat(RefAction setter, byte trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - { - return (column, formatProvider, _, ref model) => - { - if (!CultureHelpers.TryParseFloat(column, formatProvider, out float value, trimChar, trimMode)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - - // Attribute-driven custom converter (FixedColumnAttribute.Converter): the instance is created - // once (by the caller) and reused for every row, so it must be stateless. - private static Utf8ColumnParser BuildConverter( - RefAction setter, TConverter converter, byte trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - where TConverter : IUtf8FixedWidthConverter - { - return (column, formatProvider, _, ref model) => - { - if (!converter.TryParse(Utf8FixedWidthRuntime.TrimColumn(column, trimChar, trimMode), formatProvider, out var value)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - - private static Utf8ColumnParser BuildString(RefAction setter, object trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - { - byte trim = (byte)trimChar; - return (column, _, stringPool, ref model) => - { - var slice = Utf8FixedWidthRuntime.TrimColumn(column, trim, trimMode); - setter(ref model, stringPool is null ? Encoding.UTF8.GetString(slice) : stringPool.GetOrAdd(slice, Encoding.UTF8)); - return true; - }; - } - - private static Utf8ColumnParser BuildExact(Type valueType, Delegate setter, string format, object trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - { - byte trim = (byte)trimChar; - if (valueType == typeof(DateTime)) - { - return BuildExact((RefAction)setter, format, trim, trimMode, DateTime.TryParseExact); - } - if (valueType == typeof(DateOnly)) - { - return BuildExact((RefAction)setter, format, trim, trimMode, DateOnly.TryParseExact); - } - if (valueType == typeof(TimeOnly)) - { - return BuildExact((RefAction)setter, format, trim, trimMode, TimeOnly.TryParseExact); - } - if (valueType == typeof(DateTimeOffset)) - { - return BuildExact((RefAction)setter, format, trim, trimMode, DateTimeOffset.TryParseExact); - } - throw new InvalidOperationException($"Unsupported TryParseExact type {valueType}"); - } - - private delegate bool ExactParser(ReadOnlySpan value, ReadOnlySpan format, IFormatProvider? formatProvider, DateTimeStyles styles, out TValue result); - - private static Utf8ColumnParser BuildExact(RefAction setter, string format, byte trim, Attributes.TrimMode trimMode, ExactParser parser) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - { - return (column, fp, _, ref model) => - { - var trimmed = Utf8FixedWidthRuntime.TrimColumn(column, trim, trimMode); - Span chars = trimmed.Length <= 128 ? stackalloc char[128] : new char[trimmed.Length]; - int written = Encoding.UTF8.GetChars(trimmed, chars); - if (!parser(chars[..written], format, fp ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out var value)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - } -} diff --git a/src/FixedWidthParser/PublicAPI.Shipped.txt b/src/FixedWidthParser/PublicAPI.Shipped.txt new file mode 100644 index 0000000..7dc5c58 --- /dev/null +++ b/src/FixedWidthParser/PublicAPI.Shipped.txt @@ -0,0 +1 @@ +#nullable enable diff --git a/src/FixedWidthParser/PublicAPI.Unshipped.txt b/src/FixedWidthParser/PublicAPI.Unshipped.txt new file mode 100644 index 0000000..fb87ce5 --- /dev/null +++ b/src/FixedWidthParser/PublicAPI.Unshipped.txt @@ -0,0 +1,173 @@ +#nullable enable +FixedWidthParser.Attributes.Alignment +FixedWidthParser.Attributes.Alignment.Left = 0 -> FixedWidthParser.Attributes.Alignment +FixedWidthParser.Attributes.Alignment.Right = 1 -> FixedWidthParser.Attributes.Alignment +FixedWidthParser.Attributes.FixedColumnAttribute +FixedWidthParser.Attributes.FixedColumnAttribute.Alignment.get -> FixedWidthParser.Attributes.Alignment +FixedWidthParser.Attributes.FixedColumnAttribute.Alignment.set -> void +FixedWidthParser.Attributes.FixedColumnAttribute.Converter.get -> System.Type? +FixedWidthParser.Attributes.FixedColumnAttribute.Converter.set -> void +FixedWidthParser.Attributes.FixedColumnAttribute.FixedColumnAttribute(int start, int length) -> void +FixedWidthParser.Attributes.FixedColumnAttribute.Format.get -> string? +FixedWidthParser.Attributes.FixedColumnAttribute.Format.set -> void +FixedWidthParser.Attributes.FixedColumnAttribute.Length.get -> int +FixedWidthParser.Attributes.FixedColumnAttribute.Overflow.get -> FixedWidthParser.Attributes.OverflowBehavior +FixedWidthParser.Attributes.FixedColumnAttribute.Overflow.set -> void +FixedWidthParser.Attributes.FixedColumnAttribute.Padding.get -> char +FixedWidthParser.Attributes.FixedColumnAttribute.Padding.set -> void +FixedWidthParser.Attributes.FixedColumnAttribute.Start.get -> int +FixedWidthParser.Attributes.FixedColumnAttribute.TrimChar.get -> char +FixedWidthParser.Attributes.FixedColumnAttribute.TrimChar.set -> void +FixedWidthParser.Attributes.FixedColumnAttribute.TrimMode.get -> FixedWidthParser.Attributes.TrimMode +FixedWidthParser.Attributes.FixedColumnAttribute.TrimMode.set -> void +FixedWidthParser.Attributes.OverflowBehavior +FixedWidthParser.Attributes.OverflowBehavior.Default = 0 -> FixedWidthParser.Attributes.OverflowBehavior +FixedWidthParser.Attributes.OverflowBehavior.Throw = 2 -> FixedWidthParser.Attributes.OverflowBehavior +FixedWidthParser.Attributes.OverflowBehavior.Truncate = 1 -> FixedWidthParser.Attributes.OverflowBehavior +FixedWidthParser.Attributes.TrimMode +FixedWidthParser.Attributes.TrimMode.Both = 2 -> FixedWidthParser.Attributes.TrimMode +FixedWidthParser.Attributes.TrimMode.Leading = 1 -> FixedWidthParser.Attributes.TrimMode +FixedWidthParser.Attributes.TrimMode.Trailing = 0 -> FixedWidthParser.Attributes.TrimMode +FixedWidthParser.FixedWidth +FixedWidthParser.FixedWidthContext +FixedWidthParser.FixedWidthContext.Encoding.get -> System.Text.Encoding! +FixedWidthParser.FixedWidthContext.Encoding.init -> void +FixedWidthParser.FixedWidthContext.Equals(FixedWidthParser.FixedWidthContext other) -> bool +FixedWidthParser.FixedWidthContext.FixedWidthContext() -> void +FixedWidthParser.FixedWidthContext.FormatProvider.get -> System.IFormatProvider! +FixedWidthParser.FixedWidthContext.FormatProvider.init -> void +FixedWidthParser.FixedWidthContext.ReportFailedColumn(string! columnName, int start, int length) -> void +FixedWidthParser.FixedWidthContext.ReportShortLine(int requiredLength) -> void +FixedWidthParser.FixedWidthContext.StringPool.get -> CommunityToolkit.HighPerformance.Buffers.StringPool? +FixedWidthParser.FixedWidthContext.StringPool.init -> void +FixedWidthParser.FixedWidthParseException +FixedWidthParser.FixedWidthParseException.ColumnLength.get -> int +FixedWidthParser.FixedWidthParseException.ColumnName.get -> string? +FixedWidthParser.FixedWidthParseException.ColumnStart.get -> int +FixedWidthParser.FixedWidthParseException.FixedWidthParseException() -> void +FixedWidthParser.FixedWidthParseException.FixedWidthParseException(string! message) -> void +FixedWidthParser.FixedWidthParseException.FixedWidthParseException(string! message, System.Exception! innerException) -> void +FixedWidthParser.FixedWidthParseException.LineNumber.get -> long +FixedWidthParser.FixedWidthParseException.RawValue.get -> string? +FixedWidthParser.FixedWidthReaderOptions +FixedWidthParser.FixedWidthReaderOptions.BufferSize.get -> int +FixedWidthParser.FixedWidthReaderOptions.BufferSize.init -> void +FixedWidthParser.FixedWidthReaderOptions.Encoding.get -> System.Text.Encoding? +FixedWidthParser.FixedWidthReaderOptions.Encoding.init -> void +FixedWidthParser.FixedWidthReaderOptions.FixedWidthReaderOptions() -> void +FixedWidthParser.FixedWidthReaderOptions.FormatProvider.get -> System.IFormatProvider? +FixedWidthParser.FixedWidthReaderOptions.FormatProvider.init -> void +FixedWidthParser.FixedWidthReaderOptions.MaxLineLength.get -> int +FixedWidthParser.FixedWidthReaderOptions.MaxLineLength.init -> void +FixedWidthParser.FixedWidthReaderOptions.OnParseError.get -> System.Func? +FixedWidthParser.FixedWidthReaderOptions.OnParseError.init -> void +FixedWidthParser.FixedWidthReaderOptions.StringPool.get -> CommunityToolkit.HighPerformance.Buffers.StringPool? +FixedWidthParser.FixedWidthReaderOptions.StringPool.init -> void +FixedWidthParser.FixedWidthRuntime +FixedWidthParser.FixedWidthWriterOptions +FixedWidthParser.FixedWidthWriterOptions.Encoding.get -> System.Text.Encoding? +FixedWidthParser.FixedWidthWriterOptions.Encoding.init -> void +FixedWidthParser.FixedWidthWriterOptions.FixedWidthWriterOptions() -> void +FixedWidthParser.FixedWidthWriterOptions.FormatProvider.get -> System.IFormatProvider? +FixedWidthParser.FixedWidthWriterOptions.FormatProvider.init -> void +FixedWidthParser.FixedWidthWriterOptions.NewLine.get -> string! +FixedWidthParser.FixedWidthWriterOptions.NewLine.init -> void +FixedWidthParser.Formatters.ColumnFormatOptions +FixedWidthParser.Formatters.ColumnFormatOptions.Alignment.get -> FixedWidthParser.Attributes.Alignment +FixedWidthParser.Formatters.ColumnFormatOptions.Alignment.init -> void +FixedWidthParser.Formatters.ColumnFormatOptions.ColumnFormatOptions() -> void +FixedWidthParser.Formatters.ColumnFormatOptions.ColumnFormatOptions(FixedWidthParser.Attributes.Alignment Alignment, char Padding, string? Format, FixedWidthParser.Attributes.OverflowBehavior Overflow) -> void +FixedWidthParser.Formatters.ColumnFormatOptions.Deconstruct(out FixedWidthParser.Attributes.Alignment Alignment, out char Padding, out string? Format, out FixedWidthParser.Attributes.OverflowBehavior Overflow) -> void +FixedWidthParser.Formatters.ColumnFormatOptions.Equals(FixedWidthParser.Formatters.ColumnFormatOptions other) -> bool +FixedWidthParser.Formatters.ColumnFormatOptions.Format.get -> string? +FixedWidthParser.Formatters.ColumnFormatOptions.Format.init -> void +FixedWidthParser.Formatters.ColumnFormatOptions.Overflow.get -> FixedWidthParser.Attributes.OverflowBehavior +FixedWidthParser.Formatters.ColumnFormatOptions.Overflow.init -> void +FixedWidthParser.Formatters.ColumnFormatOptions.Padding.get -> char +FixedWidthParser.Formatters.ColumnFormatOptions.Padding.init -> void +FixedWidthParser.Formatters.ColumnFormatOptions.WriteInto(System.ReadOnlySpan content, System.Span slice, string! columnName) -> void +FixedWidthParser.IFixedWidthModel +FixedWidthParser.IFixedWidthModel.TryFormat(in TSelf model, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int bytesWritten) -> bool +FixedWidthParser.IFixedWidthModel.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TSelf model) -> bool +FixedWidthParser.Parsers.FixedWidthParser +FixedWidthParser.Parsers.FixedWidthParser.FixedWidthParser() -> void +FixedWidthParser.Parsers.FixedWidthParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool +FixedWidthParser.Processors.IFixedWidthConverter +FixedWidthParser.Processors.IFixedWidthConverter.TryFormat(T value, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int written) -> bool +FixedWidthParser.Processors.IFixedWidthConverter.TryParse(System.ReadOnlySpan field, in FixedWidthParser.FixedWidthContext context, out T value) -> bool +FixedWidthParser.Readers.AsyncRecordEnumerator +FixedWidthParser.Readers.AsyncRecordEnumerator.Current.get -> TModel +FixedWidthParser.Readers.AsyncRecordEnumerator.DisposeAsync() -> System.Threading.Tasks.ValueTask +FixedWidthParser.Readers.AsyncRecordEnumerator.MoveNextAsync() -> System.Threading.Tasks.ValueTask +FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable +FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable.GetAsyncEnumerator(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> FixedWidthParser.Readers.AsyncRecordEnumerator! +FixedWidthParser.Readers.FixedWidthPipeRecordEnumerable +FixedWidthParser.Readers.FixedWidthPipeRecordEnumerable.GetAsyncEnumerator(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> FixedWidthParser.Readers.PipeRecordEnumerator! +FixedWidthParser.Readers.FixedWidthReader +FixedWidthParser.Readers.FixedWidthReader.FixedWidthReader(FixedWidthParser.FixedWidthReaderOptions? options = null) -> void +FixedWidthParser.Readers.FixedWidthReader.Read(System.IO.Stream! stream, bool leaveOpen = false) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! +FixedWidthParser.Readers.FixedWidthReader.ReadAsync(System.IO.Pipelines.PipeReader! reader, bool leaveOpen = false) -> FixedWidthParser.Readers.FixedWidthPipeRecordEnumerable>! +FixedWidthParser.Readers.FixedWidthReader.ReadAsync(System.IO.Stream! stream, bool leaveOpen = false) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! +FixedWidthParser.Readers.FixedWidthReader.ReadFile(string! path) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! +FixedWidthParser.Readers.FixedWidthReader.ReadFileAsync(string! path) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! +FixedWidthParser.Readers.FixedWidthReader.TryParse(System.ReadOnlySpan line, out TModel model) -> bool +FixedWidthParser.Readers.FixedWidthRecordEnumerable +FixedWidthParser.Readers.FixedWidthRecordEnumerable.GetEnumerator() -> FixedWidthParser.Readers.RecordEnumerator +FixedWidthParser.Readers.GeneratedLineParser +FixedWidthParser.Readers.GeneratedLineParser.GeneratedLineParser() -> void +FixedWidthParser.Readers.GeneratedLineParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool +FixedWidthParser.Readers.IRecordLineParser +FixedWidthParser.Readers.IRecordLineParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool +FixedWidthParser.Readers.PipeRecordEnumerator +FixedWidthParser.Readers.PipeRecordEnumerator.Current.get -> TModel +FixedWidthParser.Readers.PipeRecordEnumerator.DisposeAsync() -> System.Threading.Tasks.ValueTask +FixedWidthParser.Readers.PipeRecordEnumerator.MoveNextAsync() -> System.Threading.Tasks.ValueTask +FixedWidthParser.Readers.RecordEnumerator +FixedWidthParser.Readers.RecordEnumerator.Current.get -> TModel +FixedWidthParser.Readers.RecordEnumerator.Dispose() -> void +FixedWidthParser.Readers.RecordEnumerator.MoveNext() -> bool +FixedWidthParser.Readers.RecordEnumerator.RecordEnumerator() -> void +FixedWidthParser.Readers.RecordEnumerator.Reset() -> void +FixedWidthParser.Readers.ReflectionLineParser +FixedWidthParser.Readers.ReflectionLineParser.ReflectionLineParser() -> void +FixedWidthParser.Readers.ReflectionLineParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool +FixedWidthParser.Writers.FixedWidthWriter +FixedWidthParser.Writers.FixedWidthWriter.FixedWidthWriter(FixedWidthParser.FixedWidthWriterOptions? options = null) -> void +FixedWidthParser.Writers.FixedWidthWriter.TryFormat(in TModel model, System.Span destination, out int bytesWritten) -> bool +FixedWidthParser.Writers.FixedWidthWriter.Write(System.IO.Stream! stream, in TModel model) -> void +FixedWidthParser.Writers.FixedWidthWriter.WriteAsync(System.IO.Stream! stream, TModel model, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +FixedWidthParser.Writers.FixedWidthWriter.WriteMany(System.IO.Stream! stream, System.Collections.Generic.IEnumerable! models) -> void +FixedWidthParser.Writers.FixedWidthWriter.WriteMany(System.IO.Stream! stream, System.ReadOnlySpan models) -> void +FixedWidthParser.Writers.FixedWidthWriter.WriteManyAsync(System.IO.Stream! stream, System.Collections.Generic.IEnumerable! models, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +override FixedWidthParser.FixedWidthContext.GetHashCode() -> int +override FixedWidthParser.Formatters.ColumnFormatOptions.GetHashCode() -> int +static FixedWidthParser.FixedWidth.Read(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadAsync(System.IO.Pipelines.PipeReader! reader, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthPipeRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadAsync(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadFile(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadFileAsync(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! +static FixedWidthParser.FixedWidth.TryFormat(in TModel model, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int bytesWritten) -> bool +static FixedWidthParser.FixedWidth.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool +static FixedWidthParser.FixedWidthContext.operator !=(FixedWidthParser.FixedWidthContext left, FixedWidthParser.FixedWidthContext right) -> bool +static FixedWidthParser.FixedWidthContext.operator ==(FixedWidthParser.FixedWidthContext left, FixedWidthParser.FixedWidthContext right) -> bool +static FixedWidthParser.FixedWidthRuntime.FormatConvert(TValue value, System.Span slice, in FixedWidthParser.FixedWidthContext context, TConverter converter, FixedWidthParser.Formatters.ColumnFormatOptions options, string! columnName) -> void +static FixedWidthParser.FixedWidthRuntime.FormatString(string! value, System.Span slice, in FixedWidthParser.FixedWidthContext context, FixedWidthParser.Formatters.ColumnFormatOptions options, string! columnName) -> void +static FixedWidthParser.FixedWidthRuntime.FormatValue(TValue value, System.Span slice, in FixedWidthParser.FixedWidthContext context, FixedWidthParser.Formatters.ColumnFormatOptions options, string! columnName) -> void +static FixedWidthParser.FixedWidthRuntime.IsBlank(System.ReadOnlySpan column, byte trimChar) -> bool +static FixedWidthParser.FixedWidthRuntime.String(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> string! +static FixedWidthParser.FixedWidthRuntime.TrimColumn(System.ReadOnlySpan column, byte trimChar, FixedWidthParser.Attributes.TrimMode trimMode) -> System.ReadOnlySpan +static FixedWidthParser.FixedWidthRuntime.TryConvert(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, TConverter converter, out TValue value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool +static FixedWidthParser.FixedWidthRuntime.TryDateOnlyExact(System.ReadOnlySpan column, string! format, in FixedWidthParser.FixedWidthContext context, out System.DateOnly value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool +static FixedWidthParser.FixedWidthRuntime.TryDateTimeExact(System.ReadOnlySpan column, string! format, in FixedWidthParser.FixedWidthContext context, out System.DateTime value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool +static FixedWidthParser.FixedWidthRuntime.TryDateTimeOffsetExact(System.ReadOnlySpan column, string! format, in FixedWidthParser.FixedWidthContext context, out System.DateTimeOffset value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool +static FixedWidthParser.FixedWidthRuntime.TryDouble(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out double value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool +static FixedWidthParser.FixedWidthRuntime.TryFloat(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out float value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool +static FixedWidthParser.FixedWidthRuntime.TryParse(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out TValue value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool +static FixedWidthParser.FixedWidthRuntime.TryParseDecoded(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out TValue value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool +static FixedWidthParser.FixedWidthRuntime.TryTimeOnlyExact(System.ReadOnlySpan column, string! format, in FixedWidthParser.FixedWidthContext context, out System.TimeOnly value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool +static FixedWidthParser.Formatters.ColumnFormatOptions.operator !=(FixedWidthParser.Formatters.ColumnFormatOptions left, FixedWidthParser.Formatters.ColumnFormatOptions right) -> bool +static FixedWidthParser.Formatters.ColumnFormatOptions.operator ==(FixedWidthParser.Formatters.ColumnFormatOptions left, FixedWidthParser.Formatters.ColumnFormatOptions right) -> bool +~override FixedWidthParser.FixedWidthContext.Equals(object obj) -> bool +~override FixedWidthParser.FixedWidthContext.ToString() -> string +~override FixedWidthParser.Formatters.ColumnFormatOptions.Equals(object obj) -> bool +~override FixedWidthParser.Formatters.ColumnFormatOptions.ToString() -> string diff --git a/src/FixedWidthParser/Readers/AsyncRecordEnumerator.cs b/src/FixedWidthParser/Readers/AsyncRecordEnumerator.cs new file mode 100644 index 0000000..d1cbb69 --- /dev/null +++ b/src/FixedWidthParser/Readers/AsyncRecordEnumerator.cs @@ -0,0 +1,95 @@ +namespace FixedWidthParser.Readers +{ + /// + /// Asynchronous record enumerator over a stream. The line scan stays synchronous; only the refill + /// awaits the stream. + /// + public sealed class AsyncRecordEnumerator : IAsyncEnumerator + where TParser : struct, IRecordLineParser + { + private readonly TParser _parser; + private readonly FixedWidthContext _context; + private readonly Func? _onParseError; + private readonly Stream _stream; + private readonly bool _ownsStream; + private readonly CancellationToken _cancellationToken; + private LineBufferState _lines; + private TModel _current; + private bool _disposed; + + internal AsyncRecordEnumerator(TParser parser, Stream stream, bool ownsStream, FixedWidthReaderOptions options, CancellationToken cancellationToken) + { + _parser = parser; + _context = options.Context; + _onParseError = options.OnParseError; + _stream = stream; + _ownsStream = ownsStream; + _cancellationToken = cancellationToken; + _lines = default; + _lines.Rent(options); + _current = default!; + } + + /// + public TModel Current => _current; + + /// + public ValueTask MoveNextAsync() + { + ObjectDisposedException.ThrowIf(_disposed, nameof(AsyncRecordEnumerator)); + _cancellationToken.ThrowIfCancellationRequested(); + var status = TryReadFromBuffer(); + if (status != LineStatus.NeedData) + { + return new ValueTask(status == LineStatus.Line); + } + + return MoveNextSlowAsync(); + } + + private async ValueTask MoveNextSlowAsync() + { + while (true) + { + _lines.Compact(); + _lines.GrowIfFull(); + _lines.Advance(await _stream.ReadAsync(_lines.Buffer.AsMemory(_lines.End), _cancellationToken).ConfigureAwait(false)); + _cancellationToken.ThrowIfCancellationRequested(); + + var status = TryReadFromBuffer(); + if (status != LineStatus.NeedData) + { + return status == LineStatus.Line; + } + } + } + + private LineStatus TryReadFromBuffer() + { + while (true) + { + var status = _lines.TryGetLine(out var line); + if (status != LineStatus.Line || _parser.TryParse(line, in _context, out _current)) + { + return status; + } + ParseErrors.HandleFailure(_parser, line, _lines.LineNumber, in _context, _onParseError); + } + } + + /// + public ValueTask DisposeAsync() + { + if (_disposed) + { + return ValueTask.CompletedTask; + } + + _lines.Return(); + _disposed = true; +#pragma warning disable IDISP007 // Disposed only when the enumerable was told it owns the stream. + return _ownsStream ? _stream.DisposeAsync() : ValueTask.CompletedTask; +#pragma warning restore IDISP007 + } + } +} diff --git a/src/FixedWidthParser/Readers/AsyncRecordEnumeratorCore.cs b/src/FixedWidthParser/Readers/AsyncRecordEnumeratorCore.cs deleted file mode 100644 index 911e34d..0000000 --- a/src/FixedWidthParser/Readers/AsyncRecordEnumeratorCore.cs +++ /dev/null @@ -1,126 +0,0 @@ -using System.Runtime.CompilerServices; -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser.Readers -{ - /// - /// Shared asynchronous enumerator logic for character and UTF-8 record readers. The span scan - /// remains synchronous; only the source refill awaits a . - /// - public sealed class AsyncRecordEnumeratorCore : IAsyncEnumerator - where T : unmanaged, IEquatable - where TFormat : struct, ILineFormat - where TParser : struct, IRecordLineParser - where TSource : struct, ISource - { - private readonly TParser _strategy; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - private readonly CancellationToken _cancellationToken; - private TSource _source; - private LineBufferState _lines; - private TModel _current; - private bool _disposed; - - internal AsyncRecordEnumeratorCore( - TParser strategy, - TSource source, - IFormatProvider? formatProvider, - StringPool? stringPool, - int bufferSize, - CancellationToken cancellationToken) - { - _strategy = strategy; - _source = source; - _formatProvider = formatProvider; - _stringPool = stringPool; - _cancellationToken = cancellationToken; - _lines = default; - _lines.Rent(bufferSize); - _current = default!; - _disposed = false; - } - - /// - public TModel Current => _current; - - /// - public ValueTask MoveNextAsync() - { - ObjectDisposedException.ThrowIf(_disposed, nameof(AsyncRecordEnumeratorCore)); - _cancellationToken.ThrowIfCancellationRequested(); - var result = TryReadFromBuffer(); - if (result == LineStatus.Line) - { - return new ValueTask(true); - } - if (result == LineStatus.End) - { - return new ValueTask(false); - } - - return MoveNextSlowAsync(); - } - - private async ValueTask MoveNextSlowAsync() - { - while (true) - { - PrepareRefill(); - _lines.Advance(await _source - .ReadAsync(_lines.Buffer.AsMemory(_lines.End, _lines.Buffer.Length - _lines.End), _cancellationToken) - .ConfigureAwait(false)); - _cancellationToken.ThrowIfCancellationRequested(); - - var result = TryReadFromBuffer(); - if (result == LineStatus.Line) - { - return true; - } - if (result == LineStatus.End) - { - return false; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private LineStatus TryReadFromBuffer() - { - var status = _lines.TryGetLine(out var line); - if (status == LineStatus.Line) - { - Parse(line); - } - return status; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void Parse(ReadOnlySpan line) - { - if (!_strategy.TryParse(line, _formatProvider, _stringPool, out _current)) - { - throw new FormatException( - $"Line {_lines.LineNumber} could not be parsed into {typeof(TModel).Name}: \"{TFormat.FormatForException(line)}\"."); - } - } - - private void PrepareRefill() - { - _lines.Compact(); - _lines.GrowIfFull(); - } - - /// - public ValueTask DisposeAsync() - { - if (!_disposed) - { - _lines.Return(); - _source.Dispose(); - _disposed = true; - } - return ValueTask.CompletedTask; - } - } -} diff --git a/src/FixedWidthParser/Readers/FixedWidthAsyncRecordEnumerable.cs b/src/FixedWidthParser/Readers/FixedWidthAsyncRecordEnumerable.cs index 520b003..1be8d89 100644 --- a/src/FixedWidthParser/Readers/FixedWidthAsyncRecordEnumerable.cs +++ b/src/FixedWidthParser/Readers/FixedWidthAsyncRecordEnumerable.cs @@ -1,43 +1,43 @@ -using System.Buffers; -using CommunityToolkit.HighPerformance.Buffers; - namespace FixedWidthParser.Readers { /// - /// Asynchronous version of : reads from a - /// via await foreach, also without allocating a string per - /// line ( buffer + slices). The - /// enumerator logic lives in the shared , - /// specialized by . + /// Asynchronous version of for + /// await foreach. A stream is read once; a file is reopened, with asynchronous I/O, on each + /// enumeration. /// public sealed class FixedWidthAsyncRecordEnumerable : IAsyncEnumerable - where TParser : struct, IRecordLineParser + where TParser : struct, IRecordLineParser { private readonly TParser _parser; - private readonly TextReaderSource _source; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - private readonly int _bufferSize; + private readonly Stream? _stream; + private readonly string? _path; + private readonly bool _ownsStream; + private readonly FixedWidthReaderOptions _options; + + internal FixedWidthAsyncRecordEnumerable(TParser parser, Stream stream, bool ownsStream, FixedWidthReaderOptions options) + { + _parser = parser; + _stream = stream; + _ownsStream = ownsStream; + _options = options; + } - internal FixedWidthAsyncRecordEnumerable( - TParser parser, - TextReaderSource source, - IFormatProvider? formatProvider, - StringPool? stringPool, - int bufferSize) + internal FixedWidthAsyncRecordEnumerable(TParser parser, string path, FixedWidthReaderOptions options) { _parser = parser; - _source = source; - _formatProvider = formatProvider; - _stringPool = stringPool; - _bufferSize = bufferSize; + _path = path; + _ownsStream = true; + _options = options; } #pragma warning disable HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type - public AsyncRecordEnumeratorCore GetAsyncEnumerator(CancellationToken cancellationToken = default) -#pragma warning restore HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type + public AsyncRecordEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) +#pragma warning restore HLQ006 { - return new(_parser, _source.Create(_bufferSize), _formatProvider, _stringPool, _bufferSize, cancellationToken); +#pragma warning disable CA2000, IDISP001 // The enumerator owns and disposes the file stream. + var stream = _stream ?? new FileStream(_path!, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1, FileOptions.Asynchronous | FileOptions.SequentialScan); +#pragma warning restore CA2000, IDISP001 + return new(_parser, stream, _ownsStream, _options, cancellationToken); } IAsyncEnumerator IAsyncEnumerable.GetAsyncEnumerator(CancellationToken cancellationToken) diff --git a/src/FixedWidthParser/Readers/FixedWidthByteReader.cs b/src/FixedWidthParser/Readers/FixedWidthByteReader.cs deleted file mode 100644 index c13caa7..0000000 --- a/src/FixedWidthParser/Readers/FixedWidthByteReader.cs +++ /dev/null @@ -1,101 +0,0 @@ -using System.IO.Pipelines; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Parsers; - -namespace FixedWidthParser.Readers -{ - /// - /// UTF-8 / byte counterpart of . Parses fixed-width records - /// straight from raw spans and streams, avoiding the UTF-8 → UTF-16 transcode - /// that the char reader pays through a . Configured once with an - /// optional culture, an optional and a buffer size; reusable across reads. - /// - /// Column offsets are measured in bytes (see ), - /// which is exact for the single-byte/ASCII payloads typical of flat files. String columns are - /// decoded with ; when a is - /// supplied, repeated values are interned (decoded once, same instance reused). A leading UTF-8 - /// byte-order mark on the stream is skipped. - /// - /// - public sealed class FixedWidthByteReader where TModel : new() - { - private readonly Utf8FixedWidthParser _parser = new(); - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - private readonly int _bufferSize; - - public FixedWidthByteReader(IFormatProvider? formatProvider = null, StringPool? stringPool = null, int bufferSize = 65536) - { - ArgumentOutOfRangeException.ThrowIfLessThan(bufferSize, 1); - _formatProvider = formatProvider; - _stringPool = stringPool; - _bufferSize = bufferSize; - } - - /// - /// Parses a single UTF-8 fixed-width line (a slice of raw bytes, without the line terminator) - /// into using the configured culture and string pool. Returns - /// when the line is too short for the layout or a non-string column - /// fails to parse. - /// - public bool TryParse(ReadOnlySpan line, out TModel model) - { - return _parser.TryParse(line, _formatProvider, _stringPool, out model); - } - - /// - /// Reads from a (single pass) as raw bytes. - /// controls whether the stream is disposed when iteration completes. - /// - public Utf8FixedWidthRecordEnumerable> Read(Stream stream, bool leaveOpen = false) - { - ArgumentNullException.ThrowIfNull(stream); - return new Utf8FixedWidthRecordEnumerable>( - new ReflectionUtf8LineParser(_parser), stream, ownsStream: !leaveOpen, _formatProvider, _stringPool, _bufferSize); - } - - /// Reads from a file as raw bytes. Re-enumerable: each iteration opens the file again. - public Utf8FixedWidthRecordEnumerable> ReadFile(string path) - { - ArgumentException.ThrowIfNullOrEmpty(path); - return new Utf8FixedWidthRecordEnumerable>( - new ReflectionUtf8LineParser(_parser), path, _formatProvider, _stringPool, _bufferSize); - } - - /// - /// Reads from a via await foreach (single pass) as raw bytes. - /// controls whether the stream is disposed when iteration completes. - /// - public Utf8FixedWidthAsyncRecordEnumerable> ReadAsync(Stream stream, bool leaveOpen = false) - { - ArgumentNullException.ThrowIfNull(stream); - return new Utf8FixedWidthAsyncRecordEnumerable>( - new ReflectionUtf8LineParser(_parser), stream, ownsStream: !leaveOpen, _formatProvider, _stringPool, _bufferSize); - } - - /// - /// Reads from a file via await foreach, with true asynchronous I/O (FileStream useAsync). - /// Re-enumerable: each iteration reopens the file. - /// - public Utf8FixedWidthAsyncRecordEnumerable> ReadFileAsync(string path) - { - ArgumentException.ThrowIfNullOrEmpty(path); - return new Utf8FixedWidthAsyncRecordEnumerable>( - new ReflectionUtf8LineParser(_parser), path, _formatProvider, _stringPool, _bufferSize); - } - - /// - /// Reads from a via await foreach (single pass) as raw bytes, - /// letting the pipe own buffering and read-ahead. Use this when the source is already a pipe - /// (e.g. a Kestrel request body, a socket, or an upstream System.IO.Pipelines stage); - /// the configured buffer size does not apply since the pipe manages its own segments. - /// controls whether the reader is completed when iteration ends. - /// - public Utf8FixedWidthPipeRecordEnumerable ReadAsync(PipeReader reader, bool leaveOpen = false) - { - ArgumentNullException.ThrowIfNull(reader); - return new Utf8FixedWidthPipeRecordEnumerable( - _parser, reader, completeReader: !leaveOpen, _formatProvider, _stringPool); - } - } -} diff --git a/src/FixedWidthParser/Readers/FixedWidthPipeRecordEnumerable.cs b/src/FixedWidthParser/Readers/FixedWidthPipeRecordEnumerable.cs new file mode 100644 index 0000000..4d13298 --- /dev/null +++ b/src/FixedWidthParser/Readers/FixedWidthPipeRecordEnumerable.cs @@ -0,0 +1,37 @@ +using System.IO.Pipelines; + +namespace FixedWidthParser.Readers +{ + /// + /// Records read from a via await foreach, letting the pipe own + /// buffering and read-ahead. Single-pass: a pipe is consumed as it is read. + /// + public sealed class FixedWidthPipeRecordEnumerable : IAsyncEnumerable + where TParser : struct, IRecordLineParser + { + private readonly TParser _parser; + private readonly PipeReader _reader; + private readonly bool _completeReader; + private readonly FixedWidthReaderOptions _options; + + internal FixedWidthPipeRecordEnumerable(TParser parser, PipeReader reader, bool completeReader, FixedWidthReaderOptions options) + { + _parser = parser; + _reader = reader; + _completeReader = completeReader; + _options = options; + } + +#pragma warning disable HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type + public PipeRecordEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) +#pragma warning restore HLQ006 + { + return new(_parser, _reader, _completeReader, _options, cancellationToken); + } + + IAsyncEnumerator IAsyncEnumerable.GetAsyncEnumerator(CancellationToken cancellationToken) + { + return GetAsyncEnumerator(cancellationToken); + } + } +} diff --git a/src/FixedWidthParser/Readers/FixedWidthReader.cs b/src/FixedWidthParser/Readers/FixedWidthReader.cs index 9012486..366a263 100644 --- a/src/FixedWidthParser/Readers/FixedWidthReader.cs +++ b/src/FixedWidthParser/Readers/FixedWidthReader.cs @@ -1,88 +1,71 @@ -using System.Text; -using CommunityToolkit.HighPerformance.Buffers; +using System.IO.Pipelines; using FixedWidthParser.Parsers; namespace FixedWidthParser.Readers { /// - /// Reads models from text sources (TextReader, Stream or file) lazily and with low - /// allocation. Configured once with culture, an optional and a - /// buffer size; reusable across multiple reads. + /// Reads attribute-mapped models from a stream, a file or a pipe, lazily and with low allocation, + /// through the runtime (reflection-built) parser. Configured once; reusable across reads. For + /// source-generated models use . + /// Column positions are bytes. /// public sealed class FixedWidthReader where TModel : new() { private readonly FixedWidthParser _parser = new(); - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - private readonly int _bufferSize; + private readonly FixedWidthReaderOptions _options; + private readonly FixedWidthContext _context; - public FixedWidthReader(IFormatProvider? formatProvider = null, StringPool? stringPool = null, int bufferSize = 65536) + /// Creates a reader; options use every default. + public FixedWidthReader(FixedWidthReaderOptions? options = null) { - ArgumentOutOfRangeException.ThrowIfLessThan(bufferSize, 1); - _formatProvider = formatProvider; - _stringPool = stringPool; - _bufferSize = bufferSize; + _options = FixedWidthReaderOptions.Validate(options); + _context = _options.Context; } - /// Reads from an existing (single pass; does not dispose it). - public FixedWidthRecordEnumerable> Read(TextReader reader) + /// Parses a single line, without its line break, using the reader's options. + public bool TryParse(ReadOnlySpan line, out TModel model) { - ArgumentNullException.ThrowIfNull(reader); - return new FixedWidthRecordEnumerable>( - new ReflectionLineParser(_parser), TextReaderSource.FromReader(reader), _formatProvider, _stringPool, _bufferSize); + return _parser.TryParse(line, in _context, out model); } - /// - /// Reads from a (single pass). Disposes the internally created - /// ; controls closing the stream. - /// - public FixedWidthRecordEnumerable> Read(Stream stream, Encoding? encoding = null, bool leaveOpen = false) + /// Reads from a stream, once. controls whether the stream is disposed when iteration ends. + public FixedWidthRecordEnumerable> Read(Stream stream, bool leaveOpen = false) { ArgumentNullException.ThrowIfNull(stream); - var enc = encoding ?? Encoding.UTF8; - return new FixedWidthRecordEnumerable>( - new ReflectionLineParser(_parser), TextReaderSource.FromStream(stream, enc, leaveOpen), _formatProvider, _stringPool, _bufferSize); + return new(new ReflectionLineParser(_parser), stream, ownsStream: !leaveOpen, _options); } - /// Reads from a file. Re-enumerable: each iteration opens the file again. - public FixedWidthRecordEnumerable> ReadFile(string path, Encoding? encoding = null) + /// Reads from a file, reopening it on each enumeration. + public FixedWidthRecordEnumerable> ReadFile(string path) { ArgumentException.ThrowIfNullOrEmpty(path); - var enc = encoding ?? Encoding.UTF8; - return new FixedWidthRecordEnumerable>( - new ReflectionLineParser(_parser), TextReaderSource.FromFile(path, enc, useAsync: false), _formatProvider, _stringPool, _bufferSize); + return new(new ReflectionLineParser(_parser), path, _options); } - /// Reads from an existing via await foreach (does not dispose it). - public FixedWidthAsyncRecordEnumerable> ReadAsync(TextReader reader) + /// Reads from a stream via await foreach, once. controls whether the stream is disposed when iteration ends. + public FixedWidthAsyncRecordEnumerable> ReadAsync(Stream stream, bool leaveOpen = false) { - ArgumentNullException.ThrowIfNull(reader); - return new FixedWidthAsyncRecordEnumerable>( - new ReflectionLineParser(_parser), TextReaderSource.FromReader(reader), _formatProvider, _stringPool, _bufferSize); + ArgumentNullException.ThrowIfNull(stream); + return new(new ReflectionLineParser(_parser), stream, ownsStream: !leaveOpen, _options); } /// - /// Reads from a via await foreach (single pass). Disposes the created - /// ; controls the stream. + /// Reads from a via await foreach, letting the pipe own buffering + /// ( does not apply). Use it when the source is + /// already a pipe, such as a Kestrel request body. controls whether + /// the reader is completed when iteration ends. /// - public FixedWidthAsyncRecordEnumerable> ReadAsync(Stream stream, Encoding? encoding = null, bool leaveOpen = false) + public FixedWidthPipeRecordEnumerable> ReadAsync(PipeReader reader, bool leaveOpen = false) { - ArgumentNullException.ThrowIfNull(stream); - var enc = encoding ?? Encoding.UTF8; - return new FixedWidthAsyncRecordEnumerable>( - new ReflectionLineParser(_parser), TextReaderSource.FromStream(stream, enc, leaveOpen), _formatProvider, _stringPool, _bufferSize); + ArgumentNullException.ThrowIfNull(reader); + return new(new ReflectionLineParser(_parser), reader, completeReader: !leaveOpen, _options); } - /// - /// Reads from a file via await foreach, with true asynchronous I/O (FileStream useAsync). - /// Re-enumerable: each iteration reopens the file. - /// - public FixedWidthAsyncRecordEnumerable> ReadFileAsync(string path, Encoding? encoding = null) + /// Reads from a file via await foreach with asynchronous I/O, reopening it on each enumeration. + public FixedWidthAsyncRecordEnumerable> ReadFileAsync(string path) { ArgumentException.ThrowIfNullOrEmpty(path); - var enc = encoding ?? Encoding.UTF8; - return new FixedWidthAsyncRecordEnumerable>( - new ReflectionLineParser(_parser), TextReaderSource.FromFile(path, enc, useAsync: true), _formatProvider, _stringPool, _bufferSize); + return new(new ReflectionLineParser(_parser), path, _options); } } } diff --git a/src/FixedWidthParser/Readers/FixedWidthRecordEnumerable.cs b/src/FixedWidthParser/Readers/FixedWidthRecordEnumerable.cs index f8be5b3..203eca1 100644 --- a/src/FixedWidthParser/Readers/FixedWidthRecordEnumerable.cs +++ b/src/FixedWidthParser/Readers/FixedWidthRecordEnumerable.cs @@ -1,44 +1,46 @@ -using System.Buffers; using System.Collections; using System.Diagnostics.CodeAnalysis; -using CommunityToolkit.HighPerformance.Buffers; namespace FixedWidthParser.Readers { /// - /// A lazily-read sequence of models from a , without allocating a - /// string per line: lines are read in blocks into a character buffer rented from the - /// and sliced as straight into the - /// parser. Exposes a enumerator for allocation-free iteration in - /// foreach, and implements for LINQ interop. + /// A lazily-read sequence of models from a stream or a file, without a string per line: bytes are + /// read in blocks into a pooled buffer and each line is parsed in place. Exposes a + /// enumerator for allocation-free foreach. A stream is read once; a + /// file is reopened on each enumeration. /// public sealed class FixedWidthRecordEnumerable : IEnumerable - where TParser : struct, IRecordLineParser + where TParser : struct, IRecordLineParser { private readonly TParser _parser; - private readonly TextReaderSource _source; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - private readonly int _bufferSize; + private readonly Stream? _stream; + private readonly string? _path; + private readonly bool _ownsStream; + private readonly FixedWidthReaderOptions _options; - internal FixedWidthRecordEnumerable( - TParser parser, - TextReaderSource source, - IFormatProvider? formatProvider, - StringPool? stringPool, - int bufferSize) + internal FixedWidthRecordEnumerable(TParser parser, Stream stream, bool ownsStream, FixedWidthReaderOptions options) { _parser = parser; - _source = source; - _formatProvider = formatProvider; - _stringPool = stringPool; - _bufferSize = bufferSize; + _stream = stream; + _ownsStream = ownsStream; + _options = options; + } + + internal FixedWidthRecordEnumerable(TParser parser, string path, FixedWidthReaderOptions options) + { + _parser = parser; + _path = path; + _ownsStream = true; + _options = options; } /// Struct enumerator: foreach iteration without heap allocation. - public RecordEnumeratorCore GetEnumerator() + public RecordEnumerator GetEnumerator() { - return new(_parser, _source.Create(_bufferSize), _formatProvider, _stringPool, _bufferSize); +#pragma warning disable CA2000, IDISP001 // The enumerator owns and disposes the file stream. + var stream = _stream ?? new FileStream(_path!, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1, FileOptions.SequentialScan); +#pragma warning restore CA2000, IDISP001 + return new(_parser, stream, _ownsStream, _options); } IEnumerator IEnumerable.GetEnumerator() @@ -51,6 +53,5 @@ IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } - } } diff --git a/src/FixedWidthParser/Readers/GeneratedUtf8FixedWidthPipeRecordEnumerable.cs b/src/FixedWidthParser/Readers/GeneratedUtf8FixedWidthPipeRecordEnumerable.cs deleted file mode 100644 index 3c18063..0000000 --- a/src/FixedWidthParser/Readers/GeneratedUtf8FixedWidthPipeRecordEnumerable.cs +++ /dev/null @@ -1,45 +0,0 @@ -using System.IO.Pipelines; -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser.Readers -{ - /// - /// source for the source-generated UTF-8 path: parses fixed-width records - /// straight off a pipe via await foreach through - /// (no reflection, no delegates). The enumerator logic lives in - /// , specialized with the - /// strategy. Single-pass. - /// - public sealed class GeneratedUtf8FixedWidthPipeRecordEnumerable : IAsyncEnumerable - where TModel : IUtf8FixedWidthModel - { - private readonly PipeReader _reader; - private readonly bool _completeReader; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - - internal GeneratedUtf8FixedWidthPipeRecordEnumerable( - PipeReader reader, - bool completeReader, - IFormatProvider? formatProvider, - StringPool? stringPool) - { - _reader = reader; - _completeReader = completeReader; - _formatProvider = formatProvider; - _stringPool = stringPool; - } - -#pragma warning disable HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type - public Utf8PipeRecordEnumeratorCore> GetAsyncEnumerator(CancellationToken cancellationToken = default) -#pragma warning restore HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type - { - return new(default, _reader, _completeReader, _formatProvider, _stringPool, cancellationToken); - } - - IAsyncEnumerator IAsyncEnumerable.GetAsyncEnumerator(CancellationToken cancellationToken) - { - return GetAsyncEnumerator(cancellationToken); - } - } -} diff --git a/src/FixedWidthParser/Readers/ILineParser.cs b/src/FixedWidthParser/Readers/ILineParser.cs index 66c6e9c..12011f7 100644 --- a/src/FixedWidthParser/Readers/ILineParser.cs +++ b/src/FixedWidthParser/Readers/ILineParser.cs @@ -1,54 +1,47 @@ -using CommunityToolkit.HighPerformance.Buffers; using FixedWidthParser.Parsers; namespace FixedWidthParser.Readers { /// - /// Element-type-generic parser strategy used by the shared record enumerator cores. + /// Parse strategy the record enumerators are specialized by. Implemented by + /// types so the JIT monomorphizes each (TModel, TParser) pair and + /// devirtualizes into a direct call. Public only because it appears in the + /// enumerator types; the library is its only implementer. /// - public interface IRecordLineParser where T : unmanaged + public interface IRecordLineParser { /// Parses a buffered line into a model. - bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model); + bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TModel model); } - /// - /// Strategy that parses a single fixed-width line into a . - /// Implemented by types so the shared enumerator cores can be - /// specialized per strategy: a struct type argument lets the JIT monomorphize each - /// (TModel, TParser) pair and devirtualize into a direct call — - /// no delegate, no boxing, no virtual dispatch on the hot path. - /// - public interface ILineParser - { - /// - bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model); - } - - // Strategy structs are plumbing for the generic enumerator cores; they are never compared, so + // Strategy structs are plumbing for the generic enumerators; they are never compared, so // value equality is meaningless for them. #pragma warning disable CA1815 // Override equals and operator equals on value types - /// - /// Reflection-based parse strategy: forwards to a runtime . - /// - public readonly struct ReflectionLineParser(FixedWidthParser parser) : ILineParser, IRecordLineParser where TModel : new() + /// Reflection-based parse strategy: forwards to a runtime . + public readonly struct ReflectionLineParser : IRecordLineParser where TModel : new() { - public bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model) + private readonly FixedWidthParser _parser; + + internal ReflectionLineParser(FixedWidthParser parser) + { + _parser = parser; + } + + /// + public bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TModel model) { - return parser.TryParse(line, formatProvider, stringPool, out model); + return _parser.TryParse(line, in context, out model); } } - /// - /// Source-generated parse strategy: forwards to the model's static - /// , avoiding reflection and delegates. - /// - public readonly struct GeneratedLineParser : ILineParser, IRecordLineParser where TModel : IFixedWidthModel + /// Source-generated parse strategy: forwards to the model's static . + public readonly struct GeneratedLineParser : IRecordLineParser where TModel : IFixedWidthModel { - public bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model) + /// + public bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TModel model) { - return TModel.TryParse(line, formatProvider, stringPool, out model); + return TModel.TryParse(line, in context, out model); } } -#pragma warning restore CA1815 // Override equals and operator equals on value types +#pragma warning restore CA1815 } diff --git a/src/FixedWidthParser/Readers/ISource.cs b/src/FixedWidthParser/Readers/ISource.cs deleted file mode 100644 index 2955fa0..0000000 --- a/src/FixedWidthParser/Readers/ISource.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace FixedWidthParser.Readers -{ - /// - /// Synchronous and asynchronous source for buffered record reading. Implemented by value types - /// so the reader cores specialize the calls without interface dispatch or allocations. - /// - public interface ISource where T : unmanaged - { - /// Reads buffered elements into . - int Read(Span buffer); - - /// Asynchronously reads buffered elements into . - ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken); - - /// Disposes the source when it owns its underlying resource. - void Dispose(); - } -} diff --git a/src/FixedWidthParser/Readers/IUtf8LineParser.cs b/src/FixedWidthParser/Readers/IUtf8LineParser.cs deleted file mode 100644 index 2e8f3f6..0000000 --- a/src/FixedWidthParser/Readers/IUtf8LineParser.cs +++ /dev/null @@ -1,47 +0,0 @@ -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Parsers; - -namespace FixedWidthParser.Readers -{ - /// - /// UTF-8 / byte counterpart of : a strategy that parses a single - /// fixed-width line of raw s into a . Implemented by - /// types so the shared UTF-8 enumerator cores can be specialized per - /// strategy — a struct type argument lets the JIT monomorphize each (TModel, TParser) - /// pair and devirtualize into a direct call (no delegate, no boxing, no - /// virtual dispatch on the hot path). - /// - public interface IUtf8LineParser - { - /// - bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model); - } - - // Strategy structs are plumbing for the generic enumerator cores; they are never compared, so - // value equality is meaningless for them. -#pragma warning disable CA1815 // Override equals and operator equals on value types - /// - /// Reflection-based parse strategy: forwards to a runtime . - /// - public readonly struct ReflectionUtf8LineParser(Utf8FixedWidthParser parser) : IUtf8LineParser, IRecordLineParser where TModel : new() - { - public bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model) - { - return parser.TryParse(line, formatProvider, stringPool, out model); - } - } - - /// - /// Source-generated parse strategy: forwards to the model's static - /// , avoiding reflection and delegates. - /// - public readonly struct GeneratedUtf8LineParser : IUtf8LineParser, IRecordLineParser where TModel : IUtf8FixedWidthModel - { - public bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model) - { - return TModel.TryParse(line, formatProvider, stringPool, out model); - } - - } -#pragma warning restore CA1815 // Override equals and operator equals on value types -} diff --git a/src/FixedWidthParser/Readers/LineBufferState.cs b/src/FixedWidthParser/Readers/LineBufferState.cs index b4d685a..f471a09 100644 --- a/src/FixedWidthParser/Readers/LineBufferState.cs +++ b/src/FixedWidthParser/Readers/LineBufferState.cs @@ -1,5 +1,4 @@ using System.Buffers; -using System.Text; namespace FixedWidthParser.Readers { @@ -11,111 +10,47 @@ internal enum LineStatus } /// - /// Element-type specifics for : the line-break markers - /// and an optional preamble (BOM) to skip before the first line. Implemented by - /// s so the calls devirtualize inside the generic buffer state, exactly - /// like the TParser strategy on the enumerator cores. + /// Buffers bytes read in blocks from a stream and splits them into lines as + /// slices, with no per-line allocation. Splits on \n, dropping + /// a preceding \r, skips a UTF-8 byte order mark when asked, and refuses a line longer than + /// the maximum, which bounds how far the buffer grows on a source without line breaks. /// - public interface ILineFormat where T : unmanaged + internal struct LineBufferState { - static abstract T Cr { get; } - static abstract T Lf { get; } - - /// - /// Decides how many leading elements to skip before the first line (e.g. a UTF-8 BOM). - /// Returns when more buffered data is needed to decide. - /// - static abstract bool TrySkipPreamble(ReadOnlySpan data, bool eof, out int skip); - - /// Formats a line for a parsing exception. - static abstract string FormatForException(ReadOnlySpan line); - } - - /// Char line format: no preamble — the already handled the BOM. - public readonly struct CharLineFormat : ILineFormat - { - public static char Cr => '\r'; - public static char Lf => '\n'; - - public static bool TrySkipPreamble(ReadOnlySpan data, bool eof, out int skip) - { - skip = 0; - return true; - } - - public static string FormatForException(ReadOnlySpan line) - { - return line.ToString(); - } - } - - /// UTF-8 byte line format: skips a leading byte-order mark (EF BB BF). - public readonly struct Utf8LineFormat : ILineFormat - { - public static byte Cr => (byte)'\r'; - public static byte Lf => (byte)'\n'; - - // Needs at least 3 buffered bytes to decide; otherwise wait for more data (unless the - // stream already ended). - public static bool TrySkipPreamble(ReadOnlySpan data, bool eof, out int skip) - { - if (data.Length >= 3) - { - skip = data is [0xEF, 0xBB, 0xBF, ..] ? 3 : 0; - return true; - } - skip = 0; - return eof; - } - - public static string FormatForException(ReadOnlySpan line) - { - return Encoding.UTF8.GetString(line); - } - } - - /// - /// Buffers elements read in blocks from the underlying source and splits them into lines as - /// , with no string (or per-line buffer) allocated. Splits on - /// , dropping a preceding (CRLF), - /// and skips the format's preamble before the first line. Used as - /// / by the -based cores and - /// as / by the -based UTF-8 cores. - /// - internal struct LineBufferState - where T : unmanaged, IEquatable - where TFormat : struct, ILineFormat - { - internal T[] Buffer; + internal byte[] Buffer; internal int Start; internal int End; internal int ScanFrom; internal bool Eof; internal int LineNumber; + private int _maxLineLength; private bool _preambleChecked; - internal void Rent(int bufferSize) + internal void Rent(FixedWidthReaderOptions options) { - Buffer = ArrayPool.Shared.Rent(bufferSize); + Buffer = ArrayPool.Shared.Rent(options.BufferSize); Start = 0; End = 0; ScanFrom = 0; Eof = false; LineNumber = 0; - _preambleChecked = false; + _maxLineLength = options.MaxLineLength; + _preambleChecked = !options.SkipsByteOrderMark; } - internal LineStatus TryGetLine(out ReadOnlySpan line) + internal LineStatus TryGetLine(out ReadOnlySpan line) { if (!_preambleChecked) { - if (!TFormat.TrySkipPreamble(Buffer.AsSpan(Start, End - Start), Eof, out int skip)) + if (End - Start < 3 && !Eof) { line = default; return LineStatus.NeedData; } - - Start += skip; + if (Buffer.AsSpan(Start, End - Start) is [0xEF, 0xBB, 0xBF, ..]) + { + Start += 3; + } ScanFrom = Start; _preambleChecked = true; } @@ -124,11 +59,10 @@ internal LineStatus TryGetLine(out ReadOnlySpan line) { if (ScanFrom < End) { - int rel = Buffer.AsSpan(ScanFrom, End - ScanFrom).IndexOf(TFormat.Lf); + int rel = Buffer.AsSpan(ScanFrom, End - ScanFrom).IndexOf((byte)'\n'); if (rel >= 0) { int nlIndex = ScanFrom + rel; - // Content ends at the '\n' (trimming a preceding '\r'); resume past the '\n'. if (TrySliceLine(rawEnd: nlIndex, nextStart: nlIndex + 1, out line)) { return LineStatus.Line; @@ -158,19 +92,23 @@ internal LineStatus TryGetLine(out ReadOnlySpan line) } /// - /// Slices the line ending at (dropping a trailing - /// ), advances / to - /// , and counts the line. Returns for an - /// empty line so the caller skips it while still having consumed it. + /// Slices the line ending at (dropping a trailing \r), advances to + /// and counts the line. Returns for an empty + /// line so the caller skips it while still having consumed it. /// - private bool TrySliceLine(int rawEnd, int nextStart, out ReadOnlySpan line) + private bool TrySliceLine(int rawEnd, int nextStart, out ReadOnlySpan line) { int contentEnd = rawEnd; - if (contentEnd > Start && Buffer[contentEnd - 1].Equals(TFormat.Cr)) + if (contentEnd > Start && Buffer[contentEnd - 1] == (byte)'\r') { contentEnd--; } + if (contentEnd - Start > _maxLineLength) + { + throw ParseErrors.LineTooLong(LineNumber + 1L, _maxLineLength); + } + line = Buffer.AsSpan(Start, contentEnd - Start); Start = nextStart; ScanFrom = nextStart; @@ -203,9 +141,16 @@ internal void GrowIfFull() return; } - var bigger = ArrayPool.Shared.Rent(Buffer.Length * 2); + // A full buffer after Compact holds one unterminated line; +1 leaves room for its '\r'. + if (End - Start > _maxLineLength + 1L) + { + throw ParseErrors.LineTooLong(LineNumber + 1L, _maxLineLength); + } + + int size = (int)Math.Min(Buffer.Length * 2L, _maxLineLength + 2L); + var bigger = ArrayPool.Shared.Rent(size); Array.Copy(Buffer, 0, bigger, 0, End); - ArrayPool.Shared.Return(Buffer); + ArrayPool.Shared.Return(Buffer); Buffer = bigger; } @@ -225,7 +170,7 @@ internal void Return() { if (Buffer is not null) { - ArrayPool.Shared.Return(Buffer); + ArrayPool.Shared.Return(Buffer); Buffer = null!; } } diff --git a/src/FixedWidthParser/Readers/ParseFailure.cs b/src/FixedWidthParser/Readers/ParseFailure.cs new file mode 100644 index 0000000..0494f4a --- /dev/null +++ b/src/FixedWidthParser/Readers/ParseFailure.cs @@ -0,0 +1,78 @@ +using System.Runtime.CompilerServices; + +namespace FixedWidthParser.Readers +{ + /// What a parser reported about the line it rejected, filled during a reader's diagnosis re-parse. + internal sealed class ParseFailure + { + internal string? ColumnName { get; private set; } + internal int Start { get; private set; } = -1; + internal int Length { get; private set; } + internal int RequiredLength { get; private set; } + + internal void SetColumn(string columnName, int start, int length) + { + ColumnName = columnName; + Start = start; + Length = length; + } + + internal void SetShortLine(int requiredLength) + { + RequiredLength = requiredLength; + } + } + + /// + /// The slow paths shared by every reader. A line that fails to parse is parsed again with a failure + /// sink, so the hot path pays nothing to learn which column rejected it. + /// + internal static class ParseErrors + { + /// Returns when asks to skip the line; throws otherwise. + [MethodImpl(MethodImplOptions.NoInlining)] + internal static void HandleFailure( + TParser parser, ReadOnlySpan line, long lineNumber, in FixedWidthContext context, Func? onParseError) + where TParser : struct, IRecordLineParser + { + var failure = new ParseFailure(); + var diagnosis = context with { Failure = failure }; + _ = parser.TryParse(line, in diagnosis, out _); + + var exception = Create(line, lineNumber, typeof(TModel).Name, failure, in context); + if (onParseError is null || !onParseError(exception)) + { + throw exception; + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + internal static InvalidDataException LineTooLong(long lineNumber, int maxLineLength) + { + return new InvalidDataException( + $"Line {lineNumber} is longer than {maxLineLength} bytes (FixedWidthReaderOptions.MaxLineLength)."); + } + + private static FixedWidthParseException Create(ReadOnlySpan line, long lineNumber, string modelName, ParseFailure failure, in FixedWidthContext context) + { + if (failure.RequiredLength > line.Length) + { + return new FixedWidthParseException( + $"Line {lineNumber} could not be parsed into {modelName}: it is {line.Length} bytes long and the layout needs {failure.RequiredLength}.", + lineNumber, columnName: null, columnStart: -1, columnLength: 0, rawValue: null); + } + + if (failure.ColumnName is { } columnName && failure.Start >= 0 && failure.Start + failure.Length <= line.Length) + { + string rawValue = context.Encoding.GetString(line.Slice(failure.Start, failure.Length)); + return new FixedWidthParseException( + $"Line {lineNumber} could not be parsed into {modelName}: column '{columnName}' (start {failure.Start}, length {failure.Length}) holds \"{rawValue}\".", + lineNumber, columnName, failure.Start, failure.Length, rawValue); + } + + return new FixedWidthParseException( + $"Line {lineNumber} could not be parsed into {modelName}: \"{context.Encoding.GetString(line)}\".", + lineNumber, columnName: null, columnStart: -1, columnLength: 0, rawValue: null); + } + } +} diff --git a/src/FixedWidthParser/Readers/Utf8PipeRecordEnumeratorCore.cs b/src/FixedWidthParser/Readers/PipeRecordEnumerator.cs similarity index 64% rename from src/FixedWidthParser/Readers/Utf8PipeRecordEnumeratorCore.cs rename to src/FixedWidthParser/Readers/PipeRecordEnumerator.cs index 2d1758b..47a25c8 100644 --- a/src/FixedWidthParser/Readers/Utf8PipeRecordEnumeratorCore.cs +++ b/src/FixedWidthParser/Readers/PipeRecordEnumerator.cs @@ -1,36 +1,27 @@ using System.Buffers; using System.IO.Pipelines; -using System.Text; -using CommunityToolkit.HighPerformance.Buffers; namespace FixedWidthParser.Readers { /// - /// -based counterpart of - /// : parses fixed-width records from a - /// , letting the pipe own buffering and read-ahead instead of the manual - /// pooled the stream cores use. Lines are split out of the - /// with a ; a line is parsed in place - /// from its when it sits in a single segment, and copied - /// into a pooled scratch buffer only when it straddles segment boundaries (the parser needs a - /// contiguous span). Specialized by a strategy - /// so parsing devirtualizes, exactly like the stream cores. - /// - /// Line semantics match the stream path: a leading UTF-8 BOM is skipped once, lines split on - /// \n (a preceding \r is dropped), empty lines are counted but not yielded, and a final - /// line without a trailing newline is yielded. - /// + /// -based counterpart of : + /// lets the pipe own buffering and read-ahead. A line is parsed in place when it sits in a single + /// segment, and copied into a pooled scratch buffer only when it straddles segments. Line semantics + /// match the stream path: a UTF-8 byte order mark is skipped once, lines split on \n (a + /// preceding \r is dropped), empty lines are counted but not yielded, and a final line without + /// a line break is yielded. /// - public sealed class Utf8PipeRecordEnumeratorCore : IAsyncEnumerator - where TParser : struct, IUtf8LineParser + public sealed class PipeRecordEnumerator : IAsyncEnumerator + where TParser : struct, IRecordLineParser { private const byte Cr = (byte)'\r'; private const byte Lf = (byte)'\n'; - private readonly TParser _strategy; + private readonly TParser _parser; private readonly bool _completeReader; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; + private readonly FixedWidthContext _context; + private readonly Func? _onParseError; + private readonly int _maxLineLength; private readonly CancellationToken _cancellationToken; private PipeReader? _reader; private byte[]? _scratch; @@ -38,35 +29,32 @@ public sealed class Utf8PipeRecordEnumeratorCore : IAsyncEnumer private bool _preambleChecked; private TModel _current; - // Stateful PipeReader tracking (PERF-B) private ReadResult _currentResult; private bool _hasResult; private SequencePosition _consumed; private SequencePosition _examined; private long _scannedBytes; - internal Utf8PipeRecordEnumeratorCore( - TParser strategy, - PipeReader reader, - bool completeReader, - IFormatProvider? formatProvider, - StringPool? stringPool, - CancellationToken cancellationToken) + internal PipeRecordEnumerator(TParser parser, PipeReader reader, bool completeReader, FixedWidthReaderOptions options, CancellationToken cancellationToken) { - _strategy = strategy; + _parser = parser; _reader = reader; _completeReader = completeReader; - _formatProvider = formatProvider; - _stringPool = stringPool; + _context = options.Context; + _onParseError = options.OnParseError; + _maxLineLength = options.MaxLineLength; + _preambleChecked = !options.SkipsByteOrderMark; _cancellationToken = cancellationToken; _current = default!; } + /// public TModel Current => _current; + /// public async ValueTask MoveNextAsync() { - ObjectDisposedException.ThrowIf(_reader is null, nameof(Utf8PipeRecordEnumeratorCore<,>)); + ObjectDisposedException.ThrowIf(_reader is null, nameof(PipeRecordEnumerator<,>)); var reader = _reader; try @@ -83,20 +71,16 @@ public async ValueTask MoveNextAsync() throw new OperationCanceledException(_cancellationToken); } _hasResult = true; - - // Reset positions to point to the start of the new buffer (which corresponds to previous _consumed) _consumed = _currentResult.Buffer.Start; _examined = _currentResult.Buffer.Start; } var status = TryReadFromResult(out _consumed, out _examined); - if (status == LineStatus.Line) { return true; } - // Before reading more data or returning false at the end, advance the reader reader.AdvanceTo(_consumed, _examined); _hasResult = false; @@ -128,16 +112,15 @@ private LineStatus TryReadFromResult(out SequencePosition consumed, out Sequence { Span head = stackalloc byte[3]; buffer.Slice(0, 3).CopyTo(head); - int skip = head is [0xEF, 0xBB, 0xBF] ? 3 : 0; - if (skip > 0) + if (head is [0xEF, 0xBB, 0xBF]) { - _consumed = buffer.GetPosition(skip); + _consumed = buffer.GetPosition(3); } _preambleChecked = true; } else if (isCompleted) { - _preambleChecked = true; // too short to hold a BOM; no preamble + _preambleChecked = true; } else { @@ -157,16 +140,14 @@ private LineStatus TryReadFromResult(out SequencePosition consumed, out Sequence { _lineNumber++; var positionOfLf = remaining.GetPosition(seqReader.Consumed - 1, scanStart); - var scopeLine = TrimTrailingCr(remaining.Slice(0, positionOfLf)); - + var line = TrimTrailingCr(remaining.Slice(0, positionOfLf)); var nextPosition = remaining.GetPosition(seqReader.Consumed, scanStart); _consumed = nextPosition; _examined = nextPosition; _scannedBytes = 0; - if (!scopeLine.IsEmpty) + if (!line.IsEmpty && Parse(in line)) { - Parse(in scopeLine); consumed = _consumed; examined = _consumed; return LineStatus.Line; @@ -176,6 +157,11 @@ private LineStatus TryReadFromResult(out SequencePosition consumed, out Sequence if (!isCompleted) { + // An unterminated line longer than the limit (+1 for its '\r') can never become valid. + if (remaining.Length > _maxLineLength + 1L) + { + throw ParseErrors.LineTooLong(_lineNumber + 1, _maxLineLength); + } _scannedBytes = remaining.Length; consumed = _consumed; examined = buffer.End; @@ -184,36 +170,29 @@ private LineStatus TryReadFromResult(out SequencePosition consumed, out Sequence consumed = buffer.End; examined = buffer.End; + _consumed = buffer.End; + _examined = buffer.End; + _scannedBytes = 0; if (remaining.Length <= 0) { - _consumed = buffer.End; - _examined = buffer.End; - _scannedBytes = 0; return LineStatus.End; } _lineNumber++; - var line = TrimTrailingCr(in remaining); - if (!line.IsEmpty) - { - _consumed = buffer.End; - _examined = buffer.End; - _scannedBytes = 0; - Parse(in line); - return LineStatus.Line; - } - _consumed = buffer.End; - _examined = buffer.End; - _scannedBytes = 0; - return LineStatus.End; + var last = TrimTrailingCr(in remaining); + return !last.IsEmpty && Parse(in last) ? LineStatus.Line : LineStatus.End; } } - private void Parse(in ReadOnlySequence line) + // Returns false when the line failed to parse and OnParseError asked to skip it. + private bool Parse(in ReadOnlySequence line) { + if (line.Length > _maxLineLength) + { + throw ParseErrors.LineTooLong(_lineNumber, _maxLineLength); + } if (line.IsSingleSegment) { - ParseSpan(line.FirstSpan); - return; + return ParseSpan(line.FirstSpan); } int length = (int)line.Length; @@ -228,16 +207,17 @@ private void Parse(in ReadOnlySequence line) _scratch = ArrayPool.Shared.Rent(length); } line.CopyTo(_scratch); - ParseSpan(_scratch.AsSpan(0, length)); + return ParseSpan(_scratch.AsSpan(0, length)); } - private void ParseSpan(ReadOnlySpan line) + private bool ParseSpan(ReadOnlySpan line) { - if (!_strategy.TryParse(line, _formatProvider, _stringPool, out _current)) + if (_parser.TryParse(line, in _context, out _current)) { - throw new FormatException( - $"Line {_lineNumber} could not be parsed into {typeof(TModel).Name}: \"{Encoding.UTF8.GetString(line)}\"."); + return true; } + ParseErrors.HandleFailure(_parser, line, _lineNumber, in _context, _onParseError); + return false; } private static ReadOnlySequence TrimTrailingCr(in ReadOnlySequence line) @@ -251,6 +231,7 @@ private static ReadOnlySequence TrimTrailingCr(in ReadOnlySequence l return last == Cr ? line.Slice(0, length - 1) : line; } + /// public async ValueTask DisposeAsync() { if (_scratch is not null) diff --git a/src/FixedWidthParser/Readers/RecordEnumerator.cs b/src/FixedWidthParser/Readers/RecordEnumerator.cs new file mode 100644 index 0000000..cf27cb6 --- /dev/null +++ b/src/FixedWidthParser/Readers/RecordEnumerator.cs @@ -0,0 +1,93 @@ +using System.Collections; +using System.Diagnostics.CodeAnalysis; + +namespace FixedWidthParser.Readers +{ + /// + /// Synchronous record enumerator over a stream. A specialized by + /// , so foreach allocates nothing and parsing devirtualizes. + /// + public struct RecordEnumerator : IEnumerator + where TParser : struct, IRecordLineParser + { + private readonly TParser _parser; + private readonly FixedWidthContext _context; + private readonly Func? _onParseError; + private readonly Stream _stream; + private readonly bool _ownsStream; + private LineBufferState _lines; + private TModel _current; + private bool _disposed; + + internal RecordEnumerator(TParser parser, Stream stream, bool ownsStream, FixedWidthReaderOptions options) + { + _parser = parser; + _context = options.Context; + _onParseError = options.OnParseError; + _stream = stream; + _ownsStream = ownsStream; + _lines = default; + _lines.Rent(options); + _current = default!; + _disposed = false; + } + + /// + public readonly TModel Current => _current; + + [ExcludeFromCodeCoverage] + readonly object IEnumerator.Current => _current!; + + /// + public bool MoveNext() + { + ObjectDisposedException.ThrowIf(_disposed, nameof(RecordEnumerator)); + while (true) + { + var status = _lines.TryGetLine(out var line); + if (status == LineStatus.Line) + { + if (_parser.TryParse(line, in _context, out _current)) + { + return true; + } + ParseErrors.HandleFailure(_parser, line, _lines.LineNumber, in _context, _onParseError); + continue; + } + if (status == LineStatus.End) + { + return false; + } + + _lines.Compact(); + _lines.GrowIfFull(); + _lines.Advance(_stream.Read(_lines.Buffer.AsSpan(_lines.End))); + } + } + + /// + public void Dispose() + { + if (_disposed) + { + return; + } + + _lines.Return(); + if (_ownsStream) + { +#pragma warning disable IDISP007 // Disposed only when the enumerable was told it owns the stream. + _stream.Dispose(); +#pragma warning restore IDISP007 + } + _disposed = true; + } + + /// + [ExcludeFromCodeCoverage] + public readonly void Reset() + { + throw new NotSupportedException("Reading is single-pass."); + } + } +} diff --git a/src/FixedWidthParser/Readers/RecordEnumeratorCore.cs b/src/FixedWidthParser/Readers/RecordEnumeratorCore.cs deleted file mode 100644 index ae04bc2..0000000 --- a/src/FixedWidthParser/Readers/RecordEnumeratorCore.cs +++ /dev/null @@ -1,107 +0,0 @@ -using System.Collections; -using System.Diagnostics.CodeAnalysis; -using System.Runtime.CompilerServices; -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser.Readers -{ - /// - /// Shared synchronous enumerator logic for character and UTF-8 record readers. The element, - /// line format, parser, and source strategies are structs, so the JIT specializes every hot-path - /// call without allocations or interface dispatch. - /// - public struct RecordEnumeratorCore : IEnumerator - where T : unmanaged, IEquatable - where TFormat : struct, ILineFormat - where TParser : struct, IRecordLineParser - where TSource : struct, ISource - { - private readonly TParser _strategy; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - private TSource _source; - private LineBufferState _lines; - private TModel _current; - private bool _disposed; - - internal RecordEnumeratorCore( - TParser strategy, - TSource source, - IFormatProvider? formatProvider, - StringPool? stringPool, - int bufferSize) - { - _strategy = strategy; - _source = source; - _formatProvider = formatProvider; - _stringPool = stringPool; - _lines = default; - _lines.Rent(bufferSize); - _current = default!; - _disposed = false; - } - - /// - public readonly TModel Current => _current; - [ExcludeFromCodeCoverage] - readonly object IEnumerator.Current => _current!; - - /// - public bool MoveNext() - { - ObjectDisposedException.ThrowIf(_disposed, nameof(RecordEnumeratorCore)); - while (true) - { - var status = _lines.TryGetLine(out var line); - if (status == LineStatus.Line) - { - Parse(line); - return true; - } - if (status == LineStatus.End) - { - return false; - } - - Refill(); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void Parse(ReadOnlySpan line) - { - if (!_strategy.TryParse(line, _formatProvider, _stringPool, out _current)) - { - throw new FormatException( - $"Line {_lines.LineNumber} could not be parsed into {typeof(TModel).Name}: \"{TFormat.FormatForException(line)}\"."); - } - } - - private void Refill() - { - _lines.Compact(); - _lines.GrowIfFull(); - _lines.Advance(_source.Read(_lines.Buffer.AsSpan(_lines.End, _lines.Buffer.Length - _lines.End))); - } - - /// - public void Dispose() - { - if (_disposed) - { - return; - } - - _lines.Return(); - _source.Dispose(); - _disposed = true; - } - - [ExcludeFromCodeCoverage] - /// - public readonly void Reset() - { - throw new NotSupportedException("Reading is single-pass."); - } - } -} diff --git a/src/FixedWidthParser/Readers/StreamSource.cs b/src/FixedWidthParser/Readers/StreamSource.cs deleted file mode 100644 index 9443acc..0000000 --- a/src/FixedWidthParser/Readers/StreamSource.cs +++ /dev/null @@ -1,29 +0,0 @@ -namespace FixedWidthParser.Readers -{ - /// Stream-backed byte source for the shared record enumerator cores. - public readonly struct StreamSource(Stream stream, bool ownsStream) : ISource - { - /// - public int Read(Span buffer) - { - return stream.Read(buffer); - } - - /// - public ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) - { - return stream.ReadAsync(buffer, cancellationToken); - } - - /// - public void Dispose() - { -#pragma warning disable IDISP007 // Don't dispose injected - if (ownsStream) - { - stream.Dispose(); - } -#pragma warning restore IDISP007 // Don't dispose injected - } - } -} diff --git a/src/FixedWidthParser/Readers/TextReaderSource.cs b/src/FixedWidthParser/Readers/TextReaderSource.cs deleted file mode 100644 index 290c2b3..0000000 --- a/src/FixedWidthParser/Readers/TextReaderSource.cs +++ /dev/null @@ -1,101 +0,0 @@ -using System.Text; - -namespace FixedWidthParser.Readers -{ - /// - /// A closure-free description of where a char reader gets its from: - /// an existing reader (single-pass, not owned), a wrapped in a - /// (single-pass, owned), or a file path reopened per enumeration - /// (owned). Stored by value on the record enumerables, so a Read call allocates no - /// Func<TextReader> delegate or capture — only the enumerable itself. - /// - public readonly struct TextReaderSource : ISource - { - private readonly TextReader? _reader; - private readonly Stream? _stream; - private readonly string? _path; - private readonly Encoding? _encoding; - private readonly bool _leaveOpen; - private readonly bool _useAsync; - - private TextReaderSource( - TextReader? reader, Stream? stream, string? path, Encoding? encoding, bool leaveOpen, bool useAsync, bool ownsReader) - { - _reader = reader; - _stream = stream; - _path = path; - _encoding = encoding; - _leaveOpen = leaveOpen; - _useAsync = useAsync; - OwnsReader = ownsReader; - } - - /// Whether the enumerator should dispose the reader (true except for an injected reader). - public bool OwnsReader { get; } - - /// Existing reader: single pass, not disposed by the enumerator. - public static TextReaderSource FromReader(TextReader reader) - { - return new(reader, null, null, null, leaveOpen: false, useAsync: false, ownsReader: false); - } - - /// Stream wrapped in a : single pass; controls closing the stream. - public static TextReaderSource FromStream(Stream stream, Encoding encoding, bool leaveOpen) - { - return new(null, stream, null, encoding, leaveOpen, useAsync: false, ownsReader: true); - } - - /// File path reopened per enumeration; opts into asynchronous file I/O. - public static TextReaderSource FromFile(string path, Encoding encoding, bool useAsync) - { - return new(null, null, path, encoding, leaveOpen: false, useAsync, ownsReader: true); - } - - /// - /// Materializes a read source. Returns the injected reader as-is, or allocates a - /// (and, for an async file, a ) owned by the - /// returned source. BOM detection mirrors the original reader behavior. - /// - public TextReaderSource Create(int bufferSize) - { - if (_reader is not null) - { - return new(_reader, null, null, null, leaveOpen: false, useAsync: false, ownsReader: OwnsReader); - } - if (_stream is not null) - { - return new(new StreamReader(_stream, _encoding, detectEncodingFromByteOrderMarks: true, bufferSize: bufferSize, leaveOpen: _leaveOpen), null, null, null, leaveOpen: false, useAsync: false, ownsReader: true); - } - if (_useAsync) - { - return new(new StreamReader( - new FileStream(_path!, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1, FileOptions.Asynchronous | FileOptions.SequentialScan), - _encoding, detectEncodingFromByteOrderMarks: true, bufferSize: bufferSize), null, null, null, leaveOpen: false, useAsync: false, ownsReader: true); - } - return new(new StreamReader(_path!, _encoding, detectEncodingFromByteOrderMarks: true, bufferSize: bufferSize), null, null, null, leaveOpen: false, useAsync: false, ownsReader: true); - } - - /// - public int Read(Span buffer) - { - return _reader!.Read(buffer); - } - - /// - public ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) - { - return _reader!.ReadAsync(buffer, cancellationToken); - } - - /// - public void Dispose() - { -#pragma warning disable IDISP007 // Don't dispose injected - if (OwnsReader) - { - _reader?.Dispose(); - } -#pragma warning restore IDISP007 // Don't dispose injected - } - } -} diff --git a/src/FixedWidthParser/Readers/Utf8FixedWidthAsyncRecordEnumerable.cs b/src/FixedWidthParser/Readers/Utf8FixedWidthAsyncRecordEnumerable.cs deleted file mode 100644 index e2e0261..0000000 --- a/src/FixedWidthParser/Readers/Utf8FixedWidthAsyncRecordEnumerable.cs +++ /dev/null @@ -1,73 +0,0 @@ -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser.Readers -{ - /// - /// Asynchronous version of : reads from a - /// via await foreach, also straight from raw bytes (no - /// , no transcode, no string per line). The enumerator logic lives in - /// . - /// - /// The source is stored as either a fixed (single-pass) or a file path - /// (reopened per enumeration with useAsync) — never a captured delegate, so a - /// ReadAsync call allocates only the enumerable itself, no closure. - /// - /// - public sealed class Utf8FixedWidthAsyncRecordEnumerable : IAsyncEnumerable - where TParser : struct, IRecordLineParser - { - private readonly TParser _parser; - private readonly Stream? _stream; - private readonly string? _path; - private readonly bool _ownsStream; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - private readonly int _bufferSize; - - /// Single-pass source: a fixed stream. - internal Utf8FixedWidthAsyncRecordEnumerable( - TParser parser, - Stream stream, - bool ownsStream, - IFormatProvider? formatProvider, - StringPool? stringPool, - int bufferSize) - { - _parser = parser; - _stream = stream; - _ownsStream = ownsStream; - _formatProvider = formatProvider; - _stringPool = stringPool; - _bufferSize = bufferSize; - } - - /// Re-enumerable source: a file path reopened (with useAsync) on each enumeration. - internal Utf8FixedWidthAsyncRecordEnumerable( - TParser parser, - string path, - IFormatProvider? formatProvider, - StringPool? stringPool, - int bufferSize) - { - _parser = parser; - _path = path; - _ownsStream = true; - _formatProvider = formatProvider; - _stringPool = stringPool; - _bufferSize = bufferSize; - } - -#pragma warning disable HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type - public AsyncRecordEnumeratorCore GetAsyncEnumerator(CancellationToken cancellationToken = default) -#pragma warning restore HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type - { - var stream = _stream ?? new FileStream(_path!, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1, FileOptions.Asynchronous | FileOptions.SequentialScan); - return new(_parser, new StreamSource(stream, _ownsStream), _formatProvider, _stringPool, _bufferSize, cancellationToken); - } - - IAsyncEnumerator IAsyncEnumerable.GetAsyncEnumerator(CancellationToken cancellationToken) - { - return GetAsyncEnumerator(cancellationToken); - } - } -} diff --git a/src/FixedWidthParser/Readers/Utf8FixedWidthPipeRecordEnumerable.cs b/src/FixedWidthParser/Readers/Utf8FixedWidthPipeRecordEnumerable.cs deleted file mode 100644 index 4b63269..0000000 --- a/src/FixedWidthParser/Readers/Utf8FixedWidthPipeRecordEnumerable.cs +++ /dev/null @@ -1,48 +0,0 @@ -using System.IO.Pipelines; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Parsers; - -namespace FixedWidthParser.Readers -{ - /// - /// source for the reflection-based UTF-8 path: parses fixed-width records - /// straight off a pipe via await foreach, letting the pipe own buffering/read-ahead. The - /// enumerator logic lives in , specialized - /// with the strategy. Single-pass (a pipe is consumed - /// as it is read). - /// - public sealed class Utf8FixedWidthPipeRecordEnumerable : IAsyncEnumerable where TModel : new() - { - private readonly Utf8FixedWidthParser _parser; - private readonly PipeReader _reader; - private readonly bool _completeReader; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - - internal Utf8FixedWidthPipeRecordEnumerable( - Utf8FixedWidthParser parser, - PipeReader reader, - bool completeReader, - IFormatProvider? formatProvider, - StringPool? stringPool) - { - _parser = parser; - _reader = reader; - _completeReader = completeReader; - _formatProvider = formatProvider; - _stringPool = stringPool; - } - -#pragma warning disable HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type - public Utf8PipeRecordEnumeratorCore> GetAsyncEnumerator(CancellationToken cancellationToken = default) -#pragma warning restore HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type - { - return new(new ReflectionUtf8LineParser(_parser), _reader, _completeReader, _formatProvider, _stringPool, cancellationToken); - } - - IAsyncEnumerator IAsyncEnumerable.GetAsyncEnumerator(CancellationToken cancellationToken) - { - return GetAsyncEnumerator(cancellationToken); - } - } -} diff --git a/src/FixedWidthParser/Readers/Utf8FixedWidthRecordEnumerable.cs b/src/FixedWidthParser/Readers/Utf8FixedWidthRecordEnumerable.cs deleted file mode 100644 index 3f02d80..0000000 --- a/src/FixedWidthParser/Readers/Utf8FixedWidthRecordEnumerable.cs +++ /dev/null @@ -1,82 +0,0 @@ -using System.Collections; -using System.Diagnostics.CodeAnalysis; -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser.Readers -{ - /// - /// UTF-8 / byte counterpart of : a lazily-read - /// sequence of models read straight from a as raw bytes (no - /// , no transcode, no string per line). Exposes a - /// enumerator for allocation-free foreach and implements for - /// LINQ interop. - /// - /// The source is stored as either a fixed (single-pass) or a file path - /// (reopened per enumeration) — never a captured delegate, so a Read call allocates only the - /// enumerable itself, no closure. - /// - /// - public sealed class Utf8FixedWidthRecordEnumerable : IEnumerable - where TParser : struct, IRecordLineParser - { - private readonly TParser _parser; - private readonly Stream? _stream; - private readonly string? _path; - private readonly bool _ownsStream; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - private readonly int _bufferSize; - - /// Single-pass source: a fixed stream. - internal Utf8FixedWidthRecordEnumerable( - TParser parser, - Stream stream, - bool ownsStream, - IFormatProvider? formatProvider, - StringPool? stringPool, - int bufferSize) - { - _parser = parser; - _stream = stream; - _ownsStream = ownsStream; - _formatProvider = formatProvider; - _stringPool = stringPool; - _bufferSize = bufferSize; - } - - /// Re-enumerable source: a file path reopened on each enumeration. - internal Utf8FixedWidthRecordEnumerable( - TParser parser, - string path, - IFormatProvider? formatProvider, - StringPool? stringPool, - int bufferSize) - { - _parser = parser; - _path = path; - _ownsStream = true; - _formatProvider = formatProvider; - _stringPool = stringPool; - _bufferSize = bufferSize; - } - - /// Struct enumerator: foreach iteration without heap allocation. - public RecordEnumeratorCore GetEnumerator() - { - var stream = _stream ?? new FileStream(_path!, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1, FileOptions.SequentialScan); - return new(_parser, new StreamSource(stream, _ownsStream), _formatProvider, _stringPool, _bufferSize); - } - - IEnumerator IEnumerable.GetEnumerator() - { - return GetEnumerator(); - } - - [ExcludeFromCodeCoverage] - IEnumerator IEnumerable.GetEnumerator() - { - return GetEnumerator(); - } - - } -} diff --git a/src/FixedWidthParser/Utf8FixedWidthRuntime.cs b/src/FixedWidthParser/Utf8FixedWidthRuntime.cs deleted file mode 100644 index 1302cae..0000000 --- a/src/FixedWidthParser/Utf8FixedWidthRuntime.cs +++ /dev/null @@ -1,127 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using System.Globalization; -using System.Runtime.CompilerServices; -using System.Text; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Processors; - -namespace FixedWidthParser -{ - /// - /// UTF-8 / byte counterpart of : low-level helpers called by the - /// source-generated implementations. They - /// centralize the column slicing, UTF-8 string decoding/pooling and numeric parsing so the generated - /// code stays small and matches the runtime byte parser's semantics exactly. - /// Public because the generated code lives in the consumer assembly; not intended for direct use. - /// - public static class Utf8FixedWidthRuntime - { - /// - /// Converts a FixedColumnAttribute.TrimChar (or any other single character configured for - /// the char path) to the single UTF-8 byte it maps to. Only ASCII (<= 0x7F) characters are a - /// single byte in UTF-8; anything else throws rather than silently trimming the wrong byte. - /// - /// The character is not a single ASCII byte. - public static byte ToAsciiByte(char value, string columnName) - { - if (value > '\x7F') - { - throw new NotSupportedException( - $"The trim character '{value}' (U+{(int)value:X4}) for column \"{columnName}\" is not a single ASCII " + - "byte. The UTF-8 byte parser only supports ASCII trim characters; use the char-based parser for this column."); - } - return (byte)value; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ReadOnlySpan TrimColumn(ReadOnlySpan column, byte trimChar, Attributes.TrimMode trimMode) - { - return trimMode switch - { - Attributes.TrimMode.Leading => column.TrimStart(trimChar), - Attributes.TrimMode.Both => column.Trim(trimChar), - _ => column.TrimEnd(trimChar) - }; - } - - /// Materializes a string column: trims spaces and UTF-8 decodes, interning via the pool when supplied. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static string String(ReadOnlySpan column, StringPool? stringPool, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - var trimmed = TrimColumn(column, trimChar, trimMode); - return stringPool is null ? Encoding.UTF8.GetString(trimmed) : stringPool.GetOrAdd(trimmed, Encoding.UTF8); - } - - /// Parses a column, honoring the provider's decimal separator (see ). - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryDouble(ReadOnlySpan column, IFormatProvider? formatProvider, out double value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - return CultureHelpers.TryParseDouble(column, formatProvider, out value, trimChar, trimMode); - } - - /// Parses a column, honoring the provider's decimal separator (see ). - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryFloat(ReadOnlySpan column, IFormatProvider? formatProvider, out float value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - return CultureHelpers.TryParseFloat(column, formatProvider, out value, trimChar, trimMode); - } - - /// - /// Parses any column (int, decimal, DateTime, …), trimming - /// spaces. A means - /// — see - /// for why (same contract on both the char and UTF-8 paths). - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryParse(ReadOnlySpan column, IFormatProvider? formatProvider, [MaybeNullWhen(false)] out TValue value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - where TValue : IUtf8SpanParsable - { - return TValue.TryParse(TrimColumn(column, trimChar, trimMode), formatProvider ?? CultureInfo.InvariantCulture, out value); - } - - /// Parses a column via a FixedColumnAttribute.Converter instance, trimming spaces. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryConvert( - ReadOnlySpan column, IFormatProvider? formatProvider, TConverter converter, [MaybeNullWhen(false)] out TValue value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - where TConverter : IUtf8FixedWidthConverter - { - return converter.TryParse(TrimColumn(column, trimChar, trimMode), formatProvider, out value); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryDateTimeExact(ReadOnlySpan column, string format, IFormatProvider? formatProvider, out DateTime value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - var trimmed = TrimColumn(column, trimChar, trimMode); - Span chars = trimmed.Length <= 128 ? stackalloc char[128] : new char[trimmed.Length]; - int written = Encoding.UTF8.GetChars(trimmed, chars); - return DateTime.TryParseExact(chars[..written], format, formatProvider ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out value); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryDateOnlyExact(ReadOnlySpan column, string format, IFormatProvider? formatProvider, out DateOnly value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - var trimmed = TrimColumn(column, trimChar, trimMode); - Span chars = trimmed.Length <= 128 ? stackalloc char[128] : new char[trimmed.Length]; - int written = Encoding.UTF8.GetChars(trimmed, chars); - return DateOnly.TryParseExact(chars[..written], format, formatProvider ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out value); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryTimeOnlyExact(ReadOnlySpan column, string format, IFormatProvider? formatProvider, out TimeOnly value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - var trimmed = TrimColumn(column, trimChar, trimMode); - Span chars = trimmed.Length <= 128 ? stackalloc char[128] : new char[trimmed.Length]; - int written = Encoding.UTF8.GetChars(trimmed, chars); - return TimeOnly.TryParseExact(chars[..written], format, formatProvider ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out value); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryDateTimeOffsetExact(ReadOnlySpan column, string format, IFormatProvider? formatProvider, out DateTimeOffset value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - var trimmed = TrimColumn(column, trimChar, trimMode); - Span chars = trimmed.Length <= 128 ? stackalloc char[128] : new char[trimmed.Length]; - int written = Encoding.UTF8.GetChars(trimmed, chars); - return DateTimeOffset.TryParseExact(chars[..written], format, formatProvider ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out value); - } - } -} diff --git a/src/FixedWidthParser/Writers/FixedWidthWriter.cs b/src/FixedWidthParser/Writers/FixedWidthWriter.cs index 9a10b3d..4514b4a 100644 --- a/src/FixedWidthParser/Writers/FixedWidthWriter.cs +++ b/src/FixedWidthParser/Writers/FixedWidthWriter.cs @@ -9,305 +9,288 @@ namespace FixedWidthParser.Writers { + /// + /// Writes attribute-mapped models as fixed-width lines of bytes through the runtime + /// (reflection-built) formatter. Batches lines into one pooled buffer, so writing many records + /// allocates nothing per record. For source-generated models use . + /// Column positions are bytes. + /// public sealed class FixedWidthWriter { - private static readonly IColumnFormatter[] _cachedFormatters; - private static readonly int _cachedLineLength; + private const int BatchSize = 64 * 1024; + + private static readonly IColumnFormatter[] _formatters; + private static readonly int _lineLength; private static readonly ExceptionDispatchInfo? _buildError; +#pragma warning disable CA1810, S3963 // The try/catch needs a static constructor. static FixedWidthWriter() { try { var formatters = new List>(); - int maxLen = 0; - + int lineLength = 0; ModelColumns.ForEachColumn(typeof(TModel), (member, attribute) => { - maxLen = Math.Max(maxLen, attribute.Start + attribute.Length); + lineLength = Math.Max(lineLength, attribute.Start + attribute.Length); formatters.Add(CreateFormatter(member, attribute)); }); - - _cachedFormatters = formatters.ToArray(); - _cachedLineLength = maxLen; + _formatters = [.. formatters]; + _lineLength = lineLength; _buildError = null; } +#pragma warning disable CA1031 // Rethrown from the instance constructor. catch (Exception ex) +#pragma warning restore CA1031 { - _cachedFormatters = []; - _cachedLineLength = 0; + _formatters = []; + _lineLength = 0; _buildError = ExceptionDispatchInfo.Capture(ex); } } +#pragma warning restore CA1810, S3963 - private readonly IColumnFormatter[] _formatters; - private readonly int _lineLength; + private readonly FixedWidthContext _context; + private readonly byte[] _newLine; - public FixedWidthWriter() + /// Creates a writer; options use every default. + public FixedWidthWriter(FixedWidthWriterOptions? options = null) { _buildError?.Throw(); - _formatters = _cachedFormatters; - _lineLength = _cachedLineLength; - } - - private static OverflowBehavior DetermineOverflowBehavior(FixedColumnAttribute attribute, bool isString) - { - if (attribute.Overflow != OverflowBehavior.Default) + options ??= FixedWidthWriterOptions.Default; + if (options.NewLine is null) { - return attribute.Overflow; + throw new ArgumentException("NewLine must not be null.", nameof(options)); } + _context = options.Context; + _newLine = _context.Encoding.GetBytes(options.NewLine); + } - if (isString) + private int RecordLength => _lineLength + _newLine.Length; + + /// + /// Formats into , without a line break. + /// Returns only when is shorter than the line. + /// + public bool TryFormat(in TModel model, Span destination, out int bytesWritten) + { + if (destination.Length < _lineLength) { - return OverflowBehavior.Truncate; + bytesWritten = 0; + return false; } - - return OverflowBehavior.Throw; + FormatLine(in model, destination[.._lineLength]); + bytesWritten = _lineLength; + return true; } - private static IColumnFormatter CreateFormatter(MemberInfo member, FixedColumnAttribute attribute) + /// Writes one record and its line break. + [SkipLocalsInit] + public void Write(Stream stream, in TModel model) { - var (memberType, model, access) = ModelColumns.MemberAccess(typeof(TModel), member); - - if (attribute.Overflow == OverflowBehavior.Truncate && memberType != typeof(string)) + ArgumentNullException.ThrowIfNull(stream); + int size = RecordLength; + byte[]? rented = size <= 1024 ? null : ArrayPool.Shared.Rent(size); + Span record = rented is null ? stackalloc byte[1024] : rented; + try { - throw new InvalidOperationException( - $"OverflowBehavior.Truncate is not supported for non-string column '{member.Name}' of type '{memberType.Name}'."); + stream.Write(record[..FormatRecord(in model, record)]); } - - var delegateType = typeof(RefGetter<,>).MakeGenericType(typeof(TModel), memberType); - var getter = Expression.Lambda(delegateType, access, model).Compile(); - // Resolve Default overflow per type: string truncates, numeric (incl. T?) throws. - var overflow = DetermineOverflowBehavior(attribute, memberType == typeof(string)); - var options = new ColumnFormatOptions(attribute.Alignment, attribute.Padding, attribute.Format, overflow); - - var underlyingType = Nullable.GetUnderlyingType(memberType); - if (underlyingType is null) + finally { - return BuildValueFormatter(member, attribute, memberType, options, getter); + if (rented is not null) + { + ArrayPool.Shared.Return(rented); + } } - - // A null value writes as a blank (padding-filled) column instead of formatting - // otherwise the underlying T formats exactly as a non-nullable column would. - var underlyingGetter = BuildUnderlyingGetter(underlyingType, getter); - var inner = BuildValueFormatter(member, attribute, underlyingType, options, underlyingGetter); - var nullableFormatterType = typeof(NullableColumnFormatter<,>).MakeGenericType(typeof(TModel), underlyingType); - return (IColumnFormatter)Activator.CreateInstance( - nullableFormatterType, attribute.Start, attribute.Length, attribute.TrimChar, getter, inner)!; } - // Adapts a RefGetter into a RefGetter (reads .Value - // only ever invoked by NullableColumnFormatter after it has confirmed HasValue). - private static Delegate BuildUnderlyingGetter(Type underlyingType, Delegate nullableGetter) + /// Writes every record, each followed by a line break. + public void WriteMany(Stream stream, IEnumerable models) { - var modelParam = Expression.Parameter(typeof(TModel).MakeByRefType(), "model"); - var invoke = Expression.Invoke(Expression.Constant(nullableGetter), modelParam); - var value = Expression.Property(invoke, "Value"); - var adapterType = typeof(RefGetter<,>).MakeGenericType(typeof(TModel), underlyingType); - return Expression.Lambda(adapterType, value, modelParam).Compile(); - } - - private static IColumnFormatter BuildValueFormatter( - MemberInfo member, FixedColumnAttribute attribute, Type valueType, ColumnFormatOptions options, Delegate getter) - { - if (attribute.Converter is { } converterType) + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(models); + byte[] buffer = RentBatch(); + try { - var requiredInterface = typeof(IFixedWidthConverter<>).MakeGenericType(valueType); - if (!requiredInterface.IsAssignableFrom(converterType)) + int used = 0; + foreach (var model in models) { - throw new InvalidOperationException( - $"Converter '{converterType}' for column '{member.Name}' must implement '{requiredInterface}'."); + used = Append(stream, buffer, used, in model); } - var converterInstance = Activator.CreateInstance(converterType) - ?? throw new InvalidOperationException($"Could not instantiate converter '{converterType}' for column '{member.Name}'."); - var converterFormatterType = typeof(ConverterColumnFormatter<,,>).MakeGenericType(typeof(TModel), valueType, converterType); - return (IColumnFormatter)Activator.CreateInstance( - converterFormatterType, attribute.Start, attribute.Length, options, member.Name, getter, converterInstance)!; + stream.Write(buffer, 0, used); } - - var formatterType = valueType == typeof(string) - ? typeof(StringColumnFormatter<>).MakeGenericType(typeof(TModel)) - : typeof(SpanFormattableColumnFormatter<,>).MakeGenericType(typeof(TModel), valueType); - return (IColumnFormatter)Activator.CreateInstance( - formatterType, attribute.Start, attribute.Length, options, member.Name, getter)!; - } - - /// - /// Formats a single model line into the given buffer: fills it with spaces and applies - /// each column formatter. Shared core used by every write overload. - /// - private void FormatLine(in TModel model, Span lineBuffer, IFormatProvider? formatProvider) - { - lineBuffer.Fill(' '); - foreach (ref readonly var formatter in _formatters.AsSpan()) + finally { - formatter.Format(in model, lineBuffer, formatProvider); + ArrayPool.Shared.Return(buffer); } } - public async Task WriteAsync(StreamWriter writer, TModel model, IFormatProvider? formatProvider = null, CancellationToken cancellationToken = default) + /// Writes every record, each followed by a line break, reading each one by reference. + public void WriteMany(Stream stream, ReadOnlySpan models) { - var lineBuffer = ArrayPool.Shared.Rent(_lineLength); + ArgumentNullException.ThrowIfNull(stream); + byte[] buffer = RentBatch(); try { - FormatLine(in model, lineBuffer.AsSpan(0, _lineLength), formatProvider); - await writer.WriteLineAsync(lineBuffer.AsMemory(0, _lineLength), cancellationToken).ConfigureAwait(false); + int used = 0; + foreach (ref readonly var model in models) + { + used = Append(stream, buffer, used, in model); + } + stream.Write(buffer, 0, used); } finally { - ArrayPool.Shared.Return(lineBuffer); + ArrayPool.Shared.Return(buffer); } } - public async Task WriteAsync(Stream stream, TModel model, IFormatProvider? formatProvider = null, CancellationToken cancellationToken = default) + + /// Writes one record and its line break. + public async Task WriteAsync(Stream stream, TModel model, CancellationToken cancellationToken = default) { - StreamWriter writer = new(stream, leaveOpen: true); - await using (writer.ConfigureAwait(false)) + ArgumentNullException.ThrowIfNull(stream); + byte[] buffer = ArrayPool.Shared.Rent(RecordLength); + try { - await WriteAsync(writer, model, formatProvider, cancellationToken).ConfigureAwait(false); + int written = FormatRecord(in model, buffer); + await stream.WriteAsync(buffer.AsMemory(0, written), cancellationToken).ConfigureAwait(false); } - } - public async Task WriteManyAsync(Stream stream, IEnumerable models, IFormatProvider? formatProvider = null, CancellationToken cancellationToken = default) - { - StreamWriter writer = new(stream, leaveOpen: true); - await using (writer.ConfigureAwait(false)) + finally { - await WriteManyAsync(writer, models, formatProvider, cancellationToken).ConfigureAwait(false); + ArrayPool.Shared.Return(buffer); } } - public async Task WriteManyAsync(StreamWriter writer, IEnumerable models, IFormatProvider? formatProvider = null, CancellationToken cancellationToken = default) + + /// Writes every record, each followed by a line break. + public async Task WriteManyAsync(Stream stream, IEnumerable models, CancellationToken cancellationToken = default) { - var lineBuffer = ArrayPool.Shared.Rent(_lineLength); + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(models); + byte[] buffer = RentBatch(); try { + int used = 0; foreach (var model in models) { - FormatLine(in model, lineBuffer.AsSpan(0, _lineLength), formatProvider); - await writer.WriteLineAsync(lineBuffer.AsMemory(0, _lineLength), cancellationToken).ConfigureAwait(false); + if (used + RecordLength > buffer.Length) + { + await stream.WriteAsync(buffer.AsMemory(0, used), cancellationToken).ConfigureAwait(false); + used = 0; + } + used += FormatRecord(in model, buffer.AsSpan(used)); } + await stream.WriteAsync(buffer.AsMemory(0, used), cancellationToken).ConfigureAwait(false); } finally { - ArrayPool.Shared.Return(lineBuffer); + ArrayPool.Shared.Return(buffer); } } - [SkipLocalsInit] - public void Write(StreamWriter writer, in TModel model, IFormatProvider? formatProvider = null) + + private byte[] RentBatch() { - if (_lineLength <= 1024) - { - Span lineBuffer = stackalloc char[_lineLength]; - FormatLine(in model, lineBuffer, formatProvider); - writer.WriteLine(lineBuffer); - return; - } + return ArrayPool.Shared.Rent(Math.Max(BatchSize, RecordLength)); + } - var rented = ArrayPool.Shared.Rent(_lineLength); - try - { - var lineBuffer = rented.AsSpan(0, _lineLength); - FormatLine(in model, lineBuffer, formatProvider); - writer.WriteLine(lineBuffer); - } - finally + private int Append(Stream stream, byte[] buffer, int used, in TModel model) + { + if (used + RecordLength > buffer.Length) { - ArrayPool.Shared.Return(rented); + stream.Write(buffer, 0, used); + used = 0; } + return used + FormatRecord(in model, buffer.AsSpan(used)); } - /// - /// Writes a single model directly to a Stream. - /// - public void Write(Stream stream, in TModel model, IFormatProvider? formatProvider = null) + private int FormatRecord(in TModel model, Span destination) { - using var writer = new StreamWriter(stream, leaveOpen: true); - Write(writer, in model, formatProvider); + FormatLine(in model, destination[.._lineLength]); + _newLine.CopyTo(destination[_lineLength..]); + return RecordLength; } - /// - /// Writes a collection of models to a Stream continuously. - /// - public void WriteMany(Stream stream, IEnumerable models, IFormatProvider? formatProvider = null) + private void FormatLine(in TModel model, Span line) { - using var writer = new StreamWriter(stream, leaveOpen: true); - WriteMany(writer, models, formatProvider); + line.Fill((byte)' '); + foreach (var formatter in _formatters) + { + formatter.Format(in model, line, in _context); + } } - /// - /// Writes a collection of models reusing an existing StreamWriter - /// (avoids allocating/disposing a StreamWriter per call). - /// - [SkipLocalsInit] - public void WriteMany(StreamWriter writer, IEnumerable models, IFormatProvider? formatProvider = null) + private static IColumnFormatter CreateFormatter(MemberInfo member, FixedColumnAttribute attribute) { - if (_lineLength <= 1024) + var (memberType, model, access) = ModelColumns.MemberAccess(typeof(TModel), member); + FixedWidthRuntime.ToAsciiByte(attribute.Padding, member.Name, nameof(FixedColumnAttribute.Padding)); + byte fill = FixedWidthRuntime.ToAsciiByte(attribute.TrimChar, member.Name, nameof(FixedColumnAttribute.TrimChar)); + + if (attribute.Overflow == OverflowBehavior.Truncate && memberType != typeof(string)) { - Span lineBuffer = stackalloc char[_lineLength]; - foreach (var model in models) - { - FormatLine(in model, lineBuffer, formatProvider); - writer.WriteLine(lineBuffer); - } - return; + throw new InvalidOperationException( + $"OverflowBehavior.Truncate is not supported for non-string column '{member.Name}' of type '{memberType.Name}'."); } - var rented = ArrayPool.Shared.Rent(_lineLength); - try + var getter = Expression.Lambda(typeof(RefGetter<,>).MakeGenericType(typeof(TModel), memberType), access, model).Compile(); + // Default overflow resolves per type: string truncates, everything else throws. + var overflow = attribute.Overflow; + if (overflow == OverflowBehavior.Default) { - var lineBuffer = rented.AsSpan(0, _lineLength); - foreach (var model in models) - { - FormatLine(in model, lineBuffer, formatProvider); - writer.WriteLine(lineBuffer); - } + overflow = memberType == typeof(string) ? OverflowBehavior.Truncate : OverflowBehavior.Throw; } - finally + var options = new ColumnFormatOptions(attribute.Alignment, attribute.Padding, attribute.Format, overflow); + + var underlyingType = Nullable.GetUnderlyingType(memberType); + if (underlyingType is null) { - ArrayPool.Shared.Return(rented); + return BuildValueFormatter(member, attribute, memberType, options, getter); } + + var inner = BuildValueFormatter(member, attribute, underlyingType, options, BuildUnderlyingGetter(underlyingType, getter)); + return (IColumnFormatter)Activator.CreateInstance( + typeof(NullableColumnFormatter<,>).MakeGenericType(typeof(TModel), underlyingType), + attribute.Start, attribute.Length, fill, getter, inner)!; } - /// - /// Writes a contiguous collection of models to a Stream without allocating an enumerator - /// (iterates by ref readonly, avoiding a copy of each struct). - /// - public void WriteMany(Stream stream, ReadOnlySpan models, IFormatProvider? formatProvider = null) + // Adapts a RefGetter into a RefGetter; only called after HasValue is confirmed. + private static Delegate BuildUnderlyingGetter(Type underlyingType, Delegate nullableGetter) { - using var writer = new StreamWriter(stream, leaveOpen: true); - WriteMany(writer, models, formatProvider); + var modelParam = Expression.Parameter(typeof(TModel).MakeByRefType(), "model"); + var value = Expression.Property(Expression.Invoke(Expression.Constant(nullableGetter), modelParam), "Value"); + return Expression.Lambda(typeof(RefGetter<,>).MakeGenericType(typeof(TModel), underlyingType), value, modelParam).Compile(); } - /// - /// Writes a contiguous collection of models reusing an existing StreamWriter, without - /// allocating an enumerator (iterates by ref readonly, avoiding a copy of each struct). - /// - [SkipLocalsInit] - public void WriteMany(StreamWriter writer, ReadOnlySpan models, IFormatProvider? formatProvider = null) + private static IColumnFormatter BuildValueFormatter( + MemberInfo member, FixedColumnAttribute attribute, Type valueType, ColumnFormatOptions options, Delegate getter) { - if (_lineLength <= 1024) + if (attribute.Converter is { } converterType) { - Span lineBuffer = stackalloc char[_lineLength]; - foreach (ref readonly var model in models) + var requiredInterface = typeof(IFixedWidthConverter<>).MakeGenericType(valueType); + if (!requiredInterface.IsAssignableFrom(converterType)) { - FormatLine(in model, lineBuffer, formatProvider); - writer.WriteLine(lineBuffer); + throw new InvalidOperationException( + $"Converter '{converterType}' for column '{member.Name}' must implement '{requiredInterface}'."); } - return; + var converter = Activator.CreateInstance(converterType) + ?? throw new InvalidOperationException($"Could not instantiate converter '{converterType}' for column '{member.Name}'."); + return (IColumnFormatter)Activator.CreateInstance( + typeof(ConverterColumnFormatter<,,>).MakeGenericType(typeof(TModel), valueType, converterType), + attribute.Start, attribute.Length, options, member.Name, getter, converter)!; } - var rented = ArrayPool.Shared.Rent(_lineLength); - try + if (valueType == typeof(string)) { - var lineBuffer = rented.AsSpan(0, _lineLength); - foreach (ref readonly var model in models) - { - FormatLine(in model, lineBuffer, formatProvider); - writer.WriteLine(lineBuffer); - } + return new StringColumnFormatter(attribute.Start, attribute.Length, options, member.Name, (RefGetter)getter); } - finally + + if (!typeof(ISpanFormattable).IsAssignableFrom(valueType)) { - ArrayPool.Shared.Return(rented); + throw new InvalidOperationException( + $"Column '{member.Name}' has type '{valueType}', which is not string or ISpanFormattable; set a Converter for it."); } + return (IColumnFormatter)Activator.CreateInstance( + typeof(SpanFormattableColumnFormatter<,>).MakeGenericType(typeof(TModel), valueType), + attribute.Start, attribute.Length, options, member.Name, getter)!; } } } diff --git a/tests/Benchmarks/Perf/AsyncReaderBenchmarks.cs b/tests/Benchmarks/Perf/AsyncReaderBenchmarks.cs index f89aa8f..fdda07e 100644 --- a/tests/Benchmarks/Perf/AsyncReaderBenchmarks.cs +++ b/tests/Benchmarks/Perf/AsyncReaderBenchmarks.cs @@ -1,28 +1,24 @@ -using System.Globalization; using System.Text; using BenchmarkDotNet.Attributes; using CommunityToolkit.HighPerformance.Buffers; using FixedWidthParser; using FixedWidthParser.Parsers; using FixedWidthParser.Readers; -using FixedWidthParser.Writers; namespace Benchmarks.Perf { /// - /// Asynchronous reading: compares the span-based await foreach (FixedWidthReader.ReadAsync) - /// with the naive baseline ReadLineAsync()+TryParse. In-memory source (StringReader, whose - /// ReadAsync completes synchronously) to isolate the async state-machine overhead + buffering. + /// Asynchronous reading from an in-memory stream: await foreach over the readers against a + /// baseline of plus parsing the re-encoded line. /// [Config(typeof(RegressionConfig))] public class AsyncReaderBenchmarks { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; private readonly FixedWidthParser _parser = new(); - private readonly FixedWidthReader _reader = new(CultureInfo.InvariantCulture); - private readonly FixedWidthReader _pooledReader = new(CultureInfo.InvariantCulture, new StringPool()); - private readonly StringPool _generatedStringPool = new(); - private string _text = string.Empty; + private readonly FixedWidthReader _reader = new(); + private readonly FixedWidthReader _pooledReader = new(new FixedWidthReaderOptions { StringPool = new StringPool() }); + private readonly FixedWidthReaderOptions _generatedPooled = new() { StringPool = new StringPool() }; + private byte[] _bytes = []; [Params(100, 1000)] public int Count { get; set; } @@ -30,27 +26,18 @@ public class AsyncReaderBenchmarks [GlobalSetup] public void Setup() { - var models = new SampleModel[Count]; - for (int i = 0; i < Count; i++) - { - models[i] = new SampleModel { Name = "Name" + (i % 100), Age = 20 + (i % 50), Salary = 1000.0 + i }; - } - var writer = new FixedWidthWriter(); - using var ms = new MemoryStream(); - writer.WriteMany(ms, models.AsSpan(), Culture); - _text = Encoding.UTF8.GetString(ms.ToArray()); + _bytes = SampleData.Bytes(Count); } - /// ReadLineAsync() allocates a string per line; then TryParse. Baseline. [Benchmark(Baseline = true)] public async Task Naive_ReadLineAsync() { - using var reader = new StringReader(_text); + using var reader = new StreamReader(new MemoryStream(_bytes, writable: false), Encoding.UTF8); int sum = 0; string? line; while ((line = await reader.ReadLineAsync().ConfigureAwait(false)) is not null) { - if (_parser.TryParse(line, Culture, null, out var model)) + if (_parser.TryParse(Encoding.UTF8.GetBytes(line), default, out var model)) { sum += model.Age; } @@ -58,59 +45,47 @@ public async Task Naive_ReadLineAsync() return sum; } - /// Span-based await foreach: slices each line from the buffer, no string per line. [Benchmark] - public async Task SpanReader_ReadAsync() + public async Task Reader_ReadAsync() { - using var reader = new StringReader(_text); int sum = 0; - await foreach (var model in _reader.ReadAsync(reader).ConfigureAwait(false)) + await foreach (var model in _reader.ReadAsync(new MemoryStream(_bytes, writable: false)).ConfigureAwait(false)) { sum += model.Age; } - return sum; } - /// Span-based await foreach + StringPool: also interns the string columns. [Benchmark] - public async Task SpanReader_ReadAsync_Pooled() + public async Task Reader_ReadAsync_Pooled() { - using var reader = new StringReader(_text); int sum = 0; - await foreach (var model in _pooledReader.ReadAsync(reader).ConfigureAwait(false)) + await foreach (var model in _pooledReader.ReadAsync(new MemoryStream(_bytes, writable: false)).ConfigureAwait(false)) { sum += model.Age; } - return sum; } - /// Source-generated async reader: same scanner, static generated TryParse, no reflection. [Benchmark] public async Task GeneratedReader_ReadAsync() { - using var reader = new StringReader(_text); int sum = 0; - await foreach (var model in FixedWidth.ReadAsync(reader, formatProvider: Culture).ConfigureAwait(false)) + await foreach (var model in FixedWidth.ReadAsync(new MemoryStream(_bytes, writable: false)).ConfigureAwait(false)) { sum += model.Age; } - return sum; } - /// Source-generated async reader + StringPool: static TryParse plus interned string columns. [Benchmark] public async Task GeneratedReader_ReadAsync_Pooled() { - using var reader = new StringReader(_text); int sum = 0; - await foreach (var model in FixedWidth.ReadAsync(reader, formatProvider: Culture, stringPool: _generatedStringPool).ConfigureAwait(false)) + await foreach (var model in FixedWidth.ReadAsync(new MemoryStream(_bytes, writable: false), options: _generatedPooled).ConfigureAwait(false)) { sum += model.Age; } - return sum; } } diff --git a/tests/Benchmarks/Perf/ByteReaderBenchmarks.cs b/tests/Benchmarks/Perf/ByteReaderBenchmarks.cs deleted file mode 100644 index c180b46..0000000 --- a/tests/Benchmarks/Perf/ByteReaderBenchmarks.cs +++ /dev/null @@ -1,108 +0,0 @@ -using System.Globalization; -using System.Text; -using BenchmarkDotNet.Attributes; -using FixedWidthParser.Parsers; -using FixedWidthParser.Writers; - -namespace Benchmarks.Perf -{ - /// - /// UTF-8 / byte parsing vs the char path. Isolates the per-line parse from I/O: lines are - /// pre-split in into char[] and UTF-8 byte[] forms (ASCII fixtures, so byte - /// offsets equal char offsets). Three lanes: - /// - /// baseline: parse lines that are already char (no transcode at all); - /// char-after-decode: UTF-8 → string per line, then parse — the cost a StreamReader-backed - /// char reader effectively pays on a UTF-8 source; - /// byte: parse straight from UTF-8 bytes with — - /// the transcode is skipped entirely. - /// - /// This measures the parse core only; covers the - /// end-to-end stream read (StreamReader-backed char reader vs the raw-byte reader). - /// - [Config(typeof(RegressionConfig))] - public class ByteReaderBenchmarks - { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; - private readonly FixedWidthParser _charParser = new(); - private readonly Utf8FixedWidthParser _byteParser = new(); - - private string[] _charLines = []; - private byte[][] _byteLines = []; - - [Params(100, 1000)] - public int Count { get; set; } - - [GlobalSetup] - public void Setup() - { - var models = new SampleModel[Count]; - for (int i = 0; i < Count; i++) - { - models[i] = new SampleModel { Name = "Name" + (i % 100), Age = 20 + (i % 50), Salary = 1000.0 + i }; - } - - var writer = new FixedWidthWriter(); - using var ms = new MemoryStream(); - writer.WriteMany(ms, models.AsSpan(), Culture); - - var text = Encoding.UTF8.GetString(ms.ToArray()); - _charLines = text.Split('\n', StringSplitOptions.RemoveEmptyEntries); - foreach (ref var v in _charLines.AsSpan()) - { - v = v.TrimEnd('\r'); - } - _byteLines = new byte[_charLines.Length][]; - for (int i = 0; i < _charLines.Length; i++) - { - _byteLines[i] = Encoding.UTF8.GetBytes(_charLines[i]); - } - } - - /// Char parse over lines that are already UTF-16: no transcode. Baseline. - [Benchmark(Baseline = true)] - public int CharParser_Parse() - { - int sum = 0; - foreach (var line in _charLines) - { - if (_charParser.TryParse(line, Culture, null, out var model)) - { - sum += model.Age; - } - } - return sum; - } - - /// UTF-8 → string per line, then char parse: the transcode a char reader pays on a UTF-8 source. - [Benchmark] - public int CharParser_Parse_AfterUtf8Decode() - { - int sum = 0; - foreach (var bytes in _byteLines) - { - var line = Encoding.UTF8.GetString(bytes); - if (_charParser.TryParse(line, Culture, null, out var model)) - { - sum += model.Age; - } - } - return sum; - } - - /// Byte parse straight from UTF-8: no transcode, no string per line. - [Benchmark] - public int ByteParser_Parse() - { - int sum = 0; - foreach (var bytes in _byteLines) - { - if (_byteParser.TryParse(bytes, Culture, null, out var model)) - { - sum += model.Age; - } - } - return sum; - } - } -} diff --git a/tests/Benchmarks/Perf/ByteReaderPoolingBenchmarks.cs b/tests/Benchmarks/Perf/ByteReaderPoolingBenchmarks.cs deleted file mode 100644 index f357bac..0000000 --- a/tests/Benchmarks/Perf/ByteReaderPoolingBenchmarks.cs +++ /dev/null @@ -1,101 +0,0 @@ -using System.Globalization; -using BenchmarkDotNet.Attributes; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Readers; -using FixedWidthParser.Writers; - -namespace Benchmarks.Perf -{ - /// - /// Where the byte reader's per-line allocation actually comes from — the string column — and the - /// two ways to remove it: a (intern repeated values) or a model with no - /// string at all. End-to-end stream read over an in-memory . The pool is - /// reused across invocations, so after warmup the repeated names ("Name0".."Name99") are already - /// interned and the pooled lane allocates ~nothing per line. - /// - /// WithString_NoPool: , a fresh string decoded per line (baseline); - /// WithString_Pooled: same model + StringPool — the string is interned, not re-decoded; - /// NoString: — no string column, nothing to decode or pool. - /// - /// - [Config(typeof(RegressionConfig))] - public class ByteReaderPoolingBenchmarks - { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; - private readonly FixedWidthByteReader _reader = new(CultureInfo.InvariantCulture); - private readonly FixedWidthByteReader _pooledReader = new(CultureInfo.InvariantCulture, new StringPool()); - private readonly FixedWidthByteReader _numericReader = new(CultureInfo.InvariantCulture); - private MemoryStream _stream = new(); - private MemoryStream _numericStream = new(); - - [Params(100, 1000)] - public int Count { get; set; } - - [GlobalSetup] - public void Setup() - { - var models = new SampleModel[Count]; - var numeric = new NumericModel[Count]; - for (int i = 0; i < Count; i++) - { - models[i] = new SampleModel { Name = "Name" + (i % 100), Age = 20 + (i % 50), Salary = 1000.0 + i }; - numeric[i] = new NumericModel { Age = 20 + (i % 50), Salary = 1000.0 + i }; - } - - _stream = new MemoryStream(); - new FixedWidthWriter().WriteMany(_stream, models.AsSpan(), Culture); - - _numericStream = new MemoryStream(); - new FixedWidthWriter().WriteMany(_numericStream, numeric.AsSpan(), Culture); - } - - [GlobalCleanup] - public void Cleanup() - { - _stream.Dispose(); - _numericStream.Dispose(); - } - - /// Byte reader, string model, no pool: a fresh string decoded per line. Baseline. - [Benchmark(Baseline = true)] - public int WithString_NoPool() - { - _stream.Position = 0; - int sum = 0; - foreach (var model in _reader.Read(_stream, leaveOpen: true)) - { - sum += model.Age; - } - - return sum; - } - - /// Byte reader, string model, StringPool: repeated values interned, not re-decoded. - [Benchmark] - public int WithString_Pooled() - { - _stream.Position = 0; - int sum = 0; - foreach (var model in _pooledReader.Read(_stream, leaveOpen: true)) - { - sum += model.Age; - } - - return sum; - } - - /// Byte reader, numeric-only model: no string column, nothing to decode or pool. - [Benchmark] - public int NoString() - { - _numericStream.Position = 0; - int sum = 0; - foreach (var model in _numericReader.Read(_numericStream, leaveOpen: true)) - { - sum += model.Age; - } - - return sum; - } - } -} diff --git a/tests/Benchmarks/Perf/ByteReaderStreamBenchmarks.cs b/tests/Benchmarks/Perf/ByteReaderStreamBenchmarks.cs deleted file mode 100644 index b597ff3..0000000 --- a/tests/Benchmarks/Perf/ByteReaderStreamBenchmarks.cs +++ /dev/null @@ -1,74 +0,0 @@ -using System.Globalization; -using BenchmarkDotNet.Attributes; -using FixedWidthParser.Readers; -using FixedWidthParser.Writers; - -namespace Benchmarks.Perf -{ - /// - /// End-to-end streaming read: the char (which decodes the - /// stream UTF-8 → UTF-16 through a ) versus the raw-byte - /// (which slices lines straight from the byte buffer, - /// no transcode). The source is an in-memory so the comparison isolates - /// buffering+decoding+parsing from disk I/O. Both readers are rewound per invocation. - /// - [Config(typeof(RegressionConfig))] - public class ByteReaderStreamBenchmarks - { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; - private readonly FixedWidthReader _charReader = new(CultureInfo.InvariantCulture); - private readonly FixedWidthByteReader _byteReader = new(CultureInfo.InvariantCulture); - private MemoryStream _stream = new(); - - [Params(100, 1000)] - public int Count { get; set; } - - [GlobalSetup] - public void Setup() - { - var models = new SampleModel[Count]; - for (int i = 0; i < Count; i++) - { - models[i] = new SampleModel { Name = "Name" + (i % 100), Age = 20 + (i % 50), Salary = 1000.0 + i }; - } - - var writer = new FixedWidthWriter(); - _stream = new MemoryStream(); - writer.WriteMany(_stream, models.AsSpan(), Culture); - } - - [GlobalCleanup] - public void Cleanup() - { - _stream.Dispose(); - } - - /// Char reader: StreamReader decodes UTF-8 → UTF-16, then span slicing. Baseline. - [Benchmark(Baseline = true)] - public int CharReader_Read() - { - _stream.Position = 0; - int sum = 0; - foreach (var model in _charReader.Read(_stream, leaveOpen: true)) - { - sum += model.Age; - } - - return sum; - } - - /// Byte reader: slices lines straight from the byte buffer, no transcode, no string per line. - [Benchmark] - public int ByteReader_Read() - { - _stream.Position = 0; - int sum = 0; - foreach (var model in _byteReader.Read(_stream, leaveOpen: true)) - { - sum += model.Age; - } - - return sum; - } - } -} diff --git a/tests/Benchmarks/Perf/ComparisonBenchmarks.cs b/tests/Benchmarks/Perf/ComparisonBenchmarks.cs index f551241..96f3460 100644 --- a/tests/Benchmarks/Perf/ComparisonBenchmarks.cs +++ b/tests/Benchmarks/Perf/ComparisonBenchmarks.cs @@ -1,11 +1,9 @@ -using System.Globalization; using System.Text; using BenchmarkDotNet.Attributes; using FileHelpers; using FlatFiles; using FixedWidthParser; using FixedWidthParser.Readers; -using FixedWidthParser.Writers; using RecordParser.Builders.Reader; using RecordParser.Extensions; @@ -28,18 +26,19 @@ public sealed class FileHelpersRecord #pragma warning restore CA1051 /// - /// Cross-library comparison: FileHelpers and FlatFiles against FixedWidthParser on the same - /// in-memory fixed-width data (25 chars/line: 10 string + 5 int + 10 double). + /// Cross-library comparison on the same in-memory fixed-width data (25 bytes/line: 10 string + 5 + /// int + 10 double). The other libraries read text, so they get the decoded string; FixedWidthParser + /// reads the file's bytes, which is what a caller holding the file would hand each of them. /// [Config(typeof(RegressionConfig))] public class ComparisonBenchmarks { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; - private readonly FixedWidthReader _reader = new(CultureInfo.InvariantCulture); + private readonly FixedWidthReader _reader = new(); private readonly FileHelperEngine _engine = new(); private readonly FixedLengthSchema _schema = CreateSchema(); private readonly FixedLengthReaderOptions _options = CreateReader(); private string _text = string.Empty; + private byte[] _bytes = []; [Params(100, 1000)] public int Count { get; set; } @@ -66,15 +65,8 @@ private static FixedLengthReaderOptions CreateReader() [GlobalSetup] public void Setup() { - var models = new SampleModel[Count]; - for (int i = 0; i < Count; i++) - { - models[i] = new SampleModel { Name = "Name" + (i % 100), Age = 20 + (i % 50), Salary = 1000.0 + i }; - } - var writer = new FixedWidthWriter(); - using var ms = new MemoryStream(); - writer.WriteMany(ms, models.AsSpan(), Culture); - _text = Encoding.UTF8.GetString(ms.ToArray()); + _bytes = SampleData.Bytes(Count); + _text = Encoding.UTF8.GetString(_bytes); } [Benchmark(Baseline = true)] @@ -119,9 +111,8 @@ public int RecordParser_Read() [Benchmark] public int FixedWidthParser_Read() { - using var stringReader = new StringReader(_text); int sum = 0; - foreach (var model in _reader.Read(stringReader)) + foreach (var model in _reader.Read(new MemoryStream(_bytes, writable: false))) { sum += model.Age; } @@ -132,9 +123,8 @@ public int FixedWidthParser_Read() [Benchmark] public int FixedWidthParser_Generated() { - using var stringReader = new StringReader(_text); int sum = 0; - foreach (var model in FixedWidth.Read(stringReader, formatProvider: Culture)) + foreach (var model in FixedWidth.Read(new MemoryStream(_bytes, writable: false))) { sum += model.Age; } diff --git a/tests/Benchmarks/Perf/ComparisonWriterBenchmarks.cs b/tests/Benchmarks/Perf/ComparisonWriterBenchmarks.cs index 2705e5d..b1de550 100644 --- a/tests/Benchmarks/Perf/ComparisonWriterBenchmarks.cs +++ b/tests/Benchmarks/Perf/ComparisonWriterBenchmarks.cs @@ -1,4 +1,3 @@ -using System.Globalization; using BenchmarkDotNet.Attributes; using FileHelpers; using FlatFiles; @@ -19,12 +18,12 @@ public sealed class FlatFilesWriteModel /// /// Cross-library comparison of fixed-width writing: FileHelpers and FlatFiles against - /// FixedWidthParser. All writes target Stream.Null to isolate formatting from I/O. + /// FixedWidthParser. All writes target Stream.Null to isolate formatting from I/O; the others go + /// through a StreamWriter because they write text, FixedWidthParser writes bytes straight to the stream. /// [Config(typeof(RegressionConfig))] public class ComparisonWriterBenchmarks : IDisposable { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; private static readonly FixedLengthOptions FlatFilesOptions = new(); private readonly FixedWidthWriter _writer = new(); private readonly FileHelperEngine _engine = new(); @@ -101,7 +100,7 @@ public void RecordParser_Write() [Benchmark] public void FixedWidthParser_Write() { - _writer.WriteMany(_sink, _models.AsSpan(), Culture); + _writer.WriteMany(Stream.Null, _models.AsSpan()); } public virtual void Dispose() diff --git a/tests/Benchmarks/Perf/ParserBenchmarks.cs b/tests/Benchmarks/Perf/ParserBenchmarks.cs index c560d9d..c80a285 100644 --- a/tests/Benchmarks/Perf/ParserBenchmarks.cs +++ b/tests/Benchmarks/Perf/ParserBenchmarks.cs @@ -1,6 +1,7 @@ -using System.Globalization; +using System.Text; using BenchmarkDotNet.Attributes; using CommunityToolkit.HighPerformance.Buffers; +using FixedWidthParser; using FixedWidthParser.Parsers; namespace Benchmarks.Perf @@ -12,9 +13,9 @@ namespace Benchmarks.Perf [Config(typeof(RegressionConfig))] public class ParserBenchmarks { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; private readonly FixedWidthParser _parser = new(); - private StringPool _stringPool = null!; + private FixedWidthContext _pooled; + private byte[] _line = []; [Params( "John Doe 30 60000.00 ", @@ -24,20 +25,21 @@ public class ParserBenchmarks [GlobalSetup] public void Setup() { - _stringPool = new StringPool(); + _line = Encoding.UTF8.GetBytes(Line); + _pooled = new FixedWidthContext { StringPool = new StringPool() }; } [Benchmark(Baseline = true)] public SampleModel Parse_NoPool() { - _parser.TryParse(Line, Culture, null, out var model); + _parser.TryParse(_line, default, out var model); return model; } [Benchmark] public SampleModel Parse_WithStringPool() { - _parser.TryParse(Line, Culture, _stringPool, out var model); + _parser.TryParse(_line, _pooled, out var model); return model; } } diff --git a/tests/Benchmarks/Perf/PipeReaderBenchmarks.cs b/tests/Benchmarks/Perf/PipeReaderBenchmarks.cs index f074ca7..dd72651 100644 --- a/tests/Benchmarks/Perf/PipeReaderBenchmarks.cs +++ b/tests/Benchmarks/Perf/PipeReaderBenchmarks.cs @@ -1,90 +1,62 @@ -using System.Globalization; using System.IO.Pipelines; using BenchmarkDotNet.Attributes; using FixedWidthParser.Readers; -using FixedWidthParser.Writers; namespace Benchmarks.Perf { /// - /// Async byte reading from a (the existing pooled LineBufferState core) - /// versus a (the pipe owns buffering/read-ahead). The source is an in-memory - /// , rewound per invocation, so the comparison isolates buffering + line - /// splitting + parsing from disk I/O. forces a tiny pipe - /// buffer so most records straddle segment boundaries, exercising the pooled scratch-copy path. + /// Reading the same bytes through a (the reader owns buffering) versus a + /// (the pipe owns buffering/read-ahead). The source is in memory, so the + /// comparison isolates buffering and line splitting from I/O. /// [Config(typeof(RegressionConfig))] public class PipeReaderBenchmarks { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; - private readonly FixedWidthByteReader _byteReader = new(CultureInfo.InvariantCulture); - private MemoryStream _stream = new(); + private readonly FixedWidthReader _reader = new(); + private byte[] _bytes = []; - [Params(100, 1000)] + [Params(1000)] public int Count { get; set; } [GlobalSetup] public void Setup() { - var models = new SampleModel[Count]; - for (int i = 0; i < Count; i++) - { - models[i] = new SampleModel { Name = "Name" + (i % 100), Age = 20 + (i % 50), Salary = 1000.0 + i }; - } - - var writer = new FixedWidthWriter(); - _stream = new MemoryStream(); - writer.WriteMany(_stream, models.AsSpan(), Culture); + _bytes = SampleData.Bytes(Count); } - [GlobalCleanup] - public void Cleanup() - { - _stream.Dispose(); - } - - /// Stream async path: manual pooled buffer with compaction/growth. Baseline. [Benchmark(Baseline = true)] public async Task Stream_ReadAsync() { - _stream.Position = 0; int sum = 0; - await foreach (var model in _byteReader.ReadAsync(_stream, leaveOpen: true).ConfigureAwait(false)) + await foreach (var model in _reader.ReadAsync(new MemoryStream(_bytes, writable: false)).ConfigureAwait(false)) { sum += model.Age; } - return sum; } - /// PipeReader path with default segment sizing: lines are typically contiguous (parsed in place). + /// Default segment sizing: lines are typically contiguous (parsed in place). [Benchmark] public async Task Pipe_ReadAsync() { - _stream.Position = 0; - // leaveOpen on the pipe options keeps the reused MemoryStream alive; the reader completes the pipe. - var pipe = PipeReader.Create(_stream, new StreamPipeReaderOptions(leaveOpen: true)); int sum = 0; - await foreach (var model in _byteReader.ReadAsync(pipe).ConfigureAwait(false)) + await foreach (var model in _reader.ReadAsync(PipeReader.Create(new MemoryStream(_bytes, writable: false))).ConfigureAwait(false)) { sum += model.Age; } - return sum; } - /// PipeReader path with a tiny buffer: most lines span segments → pooled scratch copy. + /// A tiny buffer: most lines span segments, so they go through the pooled scratch copy. [Benchmark] public async Task Pipe_ReadAsync_SmallSegments() { - _stream.Position = 0; - var pipe = PipeReader.Create(_stream, new StreamPipeReaderOptions(bufferSize: 16, leaveOpen: true)); int sum = 0; - await foreach (var model in _byteReader.ReadAsync(pipe).ConfigureAwait(false)) + var pipe = PipeReader.Create(new MemoryStream(_bytes, writable: false), new StreamPipeReaderOptions(bufferSize: 16)); + await foreach (var model in _reader.ReadAsync(pipe).ConfigureAwait(false)) { sum += model.Age; } - return sum; } } diff --git a/tests/Benchmarks/Perf/ReaderBenchmarks.cs b/tests/Benchmarks/Perf/ReaderBenchmarks.cs index d215c46..dca3b25 100644 --- a/tests/Benchmarks/Perf/ReaderBenchmarks.cs +++ b/tests/Benchmarks/Perf/ReaderBenchmarks.cs @@ -1,28 +1,25 @@ -using System.Globalization; using System.Text; using BenchmarkDotNet.Attributes; using CommunityToolkit.HighPerformance.Buffers; using FixedWidthParser; using FixedWidthParser.Parsers; using FixedWidthParser.Readers; -using FixedWidthParser.Writers; namespace Benchmarks.Perf { /// - /// Synchronous reading: compares the span-based path (FixedWidthReader, no string per line) - /// with the naive baseline ReadLine()+TryParse (one string allocated per line). The source is - /// an in-memory string (StringReader) to isolate parsing+buffering from the cost of disk I/O. + /// Synchronous reading from an in-memory stream, to isolate parsing and buffering from disk I/O. + /// The baseline reads a string per line with a and encodes it back to + /// bytes before parsing: what a caller would do by hand. /// [Config(typeof(RegressionConfig))] public class ReaderBenchmarks { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; private readonly FixedWidthParser _parser = new(); - private readonly FixedWidthReader _reader = new(CultureInfo.InvariantCulture); - private readonly FixedWidthReader _pooledReader = new(CultureInfo.InvariantCulture, new StringPool()); - private readonly StringPool _generatedStringPool = new(); - private string _text = string.Empty; + private readonly FixedWidthReader _reader = new(); + private readonly FixedWidthReader _pooledReader = new(new FixedWidthReaderOptions { StringPool = new StringPool() }); + private readonly FixedWidthReaderOptions _generatedPooled = new() { StringPool = new StringPool() }; + private byte[] _bytes = []; [Params(100, 1000)] public int Count { get; set; } @@ -30,27 +27,18 @@ public class ReaderBenchmarks [GlobalSetup] public void Setup() { - var models = new SampleModel[Count]; - for (int i = 0; i < Count; i++) - { - models[i] = new SampleModel { Name = "Name" + (i % 100), Age = 20 + (i % 50), Salary = 1000.0 + i }; - } - var writer = new FixedWidthWriter(); - using var ms = new MemoryStream(); - writer.WriteMany(ms, models.AsSpan(), Culture); - _text = Encoding.UTF8.GetString(ms.ToArray()); + _bytes = SampleData.Bytes(Count); } - /// ReadLine() allocates a string per line; then TryParse over it. Baseline. [Benchmark(Baseline = true)] public int Naive_ReadLine() { - using var reader = new StringReader(_text); + using var reader = new StreamReader(new MemoryStream(_bytes, writable: false), Encoding.UTF8); int sum = 0; string? line; while ((line = reader.ReadLine()) is not null) { - if (_parser.TryParse(line, Culture, null, out var model)) + if (_parser.TryParse(Encoding.UTF8.GetBytes(line), default, out var model)) { sum += model.Age; } @@ -58,59 +46,47 @@ public int Naive_ReadLine() return sum; } - /// Span-based reading: slices each line from the buffer, no string per line. [Benchmark] - public int SpanReader_Read() + public int Reader_Read() { - using var reader = new StringReader(_text); int sum = 0; - foreach (var model in _reader.Read(reader)) + foreach (var model in _reader.Read(new MemoryStream(_bytes, writable: false))) { sum += model.Age; } - return sum; } - /// Span-based + StringPool: also interns the string columns (tends to zero-alloc). [Benchmark] - public int SpanReader_Read_Pooled() + public int Reader_Read_Pooled() { - using var reader = new StringReader(_text); int sum = 0; - foreach (var model in _pooledReader.Read(reader)) + foreach (var model in _pooledReader.Read(new MemoryStream(_bytes, writable: false))) { sum += model.Age; } - return sum; } - /// Source-generated reader: same scanner, static generated TryParse, no reflection. [Benchmark] public int GeneratedReader_Read() { - using var reader = new StringReader(_text); int sum = 0; - foreach (var model in FixedWidth.Read(reader, formatProvider: Culture)) + foreach (var model in FixedWidth.Read(new MemoryStream(_bytes, writable: false))) { sum += model.Age; } - return sum; } - /// Source-generated reader + StringPool: static TryParse plus interned string columns. [Benchmark] public int GeneratedReader_Read_Pooled() { - using var reader = new StringReader(_text); int sum = 0; - foreach (var model in FixedWidth.Read(reader, formatProvider: Culture, stringPool: _generatedStringPool)) + foreach (var model in FixedWidth.Read(new MemoryStream(_bytes, writable: false), options: _generatedPooled)) { sum += model.Age; } - return sum; } } diff --git a/tests/Benchmarks/Perf/ReaderPoolingBenchmarks.cs b/tests/Benchmarks/Perf/ReaderPoolingBenchmarks.cs new file mode 100644 index 0000000..d3086c9 --- /dev/null +++ b/tests/Benchmarks/Perf/ReaderPoolingBenchmarks.cs @@ -0,0 +1,72 @@ +using BenchmarkDotNet.Attributes; +using CommunityToolkit.HighPerformance.Buffers; +using FixedWidthParser; +using FixedWidthParser.Readers; +using FixedWidthParser.Writers; + +namespace Benchmarks.Perf +{ + /// + /// Where the reader's per-line allocation comes from — the string column — and what a + /// saves. The pool is reused across invocations, so after warmup the + /// repeated names ("Name0".."Name99") are already interned. + /// + [Config(typeof(RegressionConfig))] + public class ReaderPoolingBenchmarks + { + private readonly FixedWidthReader _reader = new(); + private readonly FixedWidthReader _pooledReader = new(new FixedWidthReaderOptions { StringPool = new StringPool() }); + private readonly FixedWidthReader _numericReader = new(); + private byte[] _bytes = []; + private byte[] _numericBytes = []; + + [Params(1000)] + public int Count { get; set; } + + [GlobalSetup] + public void Setup() + { + _bytes = SampleData.Bytes(Count); + var numeric = SampleData.Models(Count).Select(m => new NumericModel { Age = m.Age, Salary = m.Salary }).ToArray(); + using var ms = new MemoryStream(); + new FixedWidthWriter().WriteMany(ms, numeric.AsSpan()); + _numericBytes = ms.ToArray(); + } + + /// String model, no pool: a fresh string decoded per line. Baseline. + [Benchmark(Baseline = true)] + public int WithString_NoPool() + { + int sum = 0; + foreach (var model in _reader.Read(new MemoryStream(_bytes, writable: false))) + { + sum += model.Age; + } + return sum; + } + + /// String model, StringPool: repeated values interned, not re-decoded. + [Benchmark] + public int WithString_Pooled() + { + int sum = 0; + foreach (var model in _pooledReader.Read(new MemoryStream(_bytes, writable: false))) + { + sum += model.Age; + } + return sum; + } + + /// Numeric-only model: no string column, nothing to decode or pool. + [Benchmark] + public int NoString() + { + int sum = 0; + foreach (var model in _numericReader.Read(new MemoryStream(_numericBytes, writable: false))) + { + sum += model.Age; + } + return sum; + } + } +} diff --git a/tests/Benchmarks/Perf/ReaderStartupBenchmarks.cs b/tests/Benchmarks/Perf/ReaderStartupBenchmarks.cs index b059311..a71f18d 100644 --- a/tests/Benchmarks/Perf/ReaderStartupBenchmarks.cs +++ b/tests/Benchmarks/Perf/ReaderStartupBenchmarks.cs @@ -1,4 +1,3 @@ -using System.Globalization; using BenchmarkDotNet.Attributes; using FixedWidthParser; using FixedWidthParser.Readers; @@ -13,16 +12,13 @@ namespace Benchmarks.Perf [Config(typeof(RegressionConfig))] public class ReaderStartupBenchmarks { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; - private const string Text = "John Doe 30 60000.00 \n"; + private static readonly byte[] Text = "John Doe 30 60000.00 \n"u8.ToArray(); [Benchmark(Baseline = true)] public int ReflectionReader_ConstructAndReadOne() { - var fixedWidthReader = new FixedWidthReader(Culture); - using var reader = new StringReader(Text); int sum = 0; - foreach (var model in fixedWidthReader.Read(reader)) + foreach (var model in new FixedWidthReader().Read(new MemoryStream(Text, writable: false))) { sum += model.Age; } @@ -32,9 +28,8 @@ public int ReflectionReader_ConstructAndReadOne() [Benchmark] public int GeneratedReader_ReadOne() { - using var reader = new StringReader(Text); int sum = 0; - foreach (var model in FixedWidth.Read(reader, formatProvider: Culture)) + foreach (var model in FixedWidth.Read(new MemoryStream(Text, writable: false))) { sum += model.Age; } diff --git a/tests/Benchmarks/Perf/SourceGenParserBenchmarks.cs b/tests/Benchmarks/Perf/SourceGenParserBenchmarks.cs index 9916e07..dfbe5ae 100644 --- a/tests/Benchmarks/Perf/SourceGenParserBenchmarks.cs +++ b/tests/Benchmarks/Perf/SourceGenParserBenchmarks.cs @@ -1,4 +1,4 @@ -using System.Globalization; +using System.Text; using BenchmarkDotNet.Attributes; using FixedWidthParser; using FixedWidthParser.Attributes; @@ -21,23 +21,29 @@ namespace Benchmarks.Perf [Config(typeof(RegressionConfig))] public class SourceGenParserBenchmarks { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; private readonly FixedWidthParser _reflection = new(); + private byte[] _line = []; [Params("John Doe 30 6000000.00")] public string Line { get; set; } = string.Empty; + [GlobalSetup] + public void Setup() + { + _line = Encoding.UTF8.GetBytes(Line); + } + [Benchmark(Baseline = true)] public int Reflection() { - _reflection.TryParse(Line, Culture, null, out var model); + _reflection.TryParse(_line, default, out var model); return model.Age; } [Benchmark] public int Generated() { - FixedWidth.TryParse(Line, Culture, null, out var model); + FixedWidth.TryParse(_line, default, out var model); return model.Age; } } diff --git a/tests/Benchmarks/Perf/WriterBenchmarks.cs b/tests/Benchmarks/Perf/WriterBenchmarks.cs index d9dedb3..bbd38ff 100644 --- a/tests/Benchmarks/Perf/WriterBenchmarks.cs +++ b/tests/Benchmarks/Perf/WriterBenchmarks.cs @@ -1,21 +1,17 @@ -using System.Globalization; using BenchmarkDotNet.Attributes; using FixedWidthParser.Writers; namespace Benchmarks.Perf { /// - /// Measures the hot path of fixed-width writing. Writes to Stream.Null to isolate formatting + - /// encoding from the cost of disk I/O. WriteMany_NewStream is the baseline. + /// Measures the hot path of fixed-width writing. Writes to Stream.Null to isolate formatting from + /// the cost of disk I/O. WriteMany_Enumerable is the baseline. /// [Config(typeof(RegressionConfig))] - public class WriterBenchmarks : IDisposable + public class WriterBenchmarks { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; private readonly FixedWidthWriter _writer = new(); - private StreamWriter _sink = null!; private SampleModel[] _models = []; - private bool _disposed; [Params(1, 100, 1000)] public int Count { get; set; } @@ -23,76 +19,27 @@ public class WriterBenchmarks : IDisposable [GlobalSetup] public void Setup() { - _sink?.Dispose(); - _sink = new StreamWriter(Stream.Null); - _models = new SampleModel[Count]; - for (int i = 0; i < Count; i++) - { - _models[i] = new SampleModel - { - Name = "Name" + i, - Age = 20 + (i % 50), - Salary = 1000.0 + i - }; - } + _models = SampleData.Models(Count); } - [GlobalCleanup] - public void Cleanup() - { - _sink.Dispose(); - } - - /// Creates/disposes a StreamWriter per call (over the Stream). Baseline. [Benchmark(Baseline = true)] - public void WriteMany_NewStream() - { - _writer.WriteMany(Stream.Null, _models, Culture); - } - - /// Reuses an existing StreamWriter (new overload). Should eliminate the fixed allocation. - [Benchmark] - public void WriteMany_ReuseWriter() + public void WriteMany_Enumerable() { - _writer.WriteMany(_sink, _models, Culture); + _writer.WriteMany(Stream.Null, (IEnumerable)_models); } - /// Writer reuse + ReadOnlySpan: no StreamWriter per call and no enumerator. Should be zero-alloc. + /// ReadOnlySpan: no enumerator, each record read by reference. Should be zero-alloc. [Benchmark] - public void WriteMany_ReuseWriterSpan() + public void WriteMany_Span() { - _writer.WriteMany(_sink, _models.AsSpan(), Culture); + _writer.WriteMany(Stream.Null, _models.AsSpan()); } - /// - /// Async creating a StreamWriter per call (Stream overload). Mixes the cost of the new - /// StreamWriter with the async state-machine overhead — comparable to NewStream. - /// + /// The async overload: the difference from WriteMany_Enumerable is the async state machine. [Benchmark] - public Task WriteMany_AsyncNewStream() - { - return _writer.WriteManyAsync(Stream.Null, _models, Culture); - } - - /// - /// Async reusing the StreamWriter (StreamWriter overload). Isolates the pure async - /// overhead: compared to WriteMany_ReuseWriter, the difference is only the async. - /// - [Benchmark] - public Task WriteMany_AsyncReuseWriter() - { - return _writer.WriteManyAsync(_sink, _models, Culture); - } - - public virtual void Dispose() + public Task WriteManyAsync() { - if (_disposed) - { - return; - } - _disposed = true; - _sink.Dispose(); - GC.SuppressFinalize(this); + return _writer.WriteManyAsync(Stream.Null, _models); } } } diff --git a/tests/Benchmarks/SampleData.cs b/tests/Benchmarks/SampleData.cs new file mode 100644 index 0000000..983956b --- /dev/null +++ b/tests/Benchmarks/SampleData.cs @@ -0,0 +1,25 @@ +using FixedWidthParser.Writers; + +namespace Benchmarks +{ + /// The fixed-width input every read benchmark parses: lines. + internal static class SampleData + { + public static SampleModel[] Models(int count) + { + var models = new SampleModel[count]; + for (int i = 0; i < count; i++) + { + models[i] = new SampleModel { Name = "Name" + (i % 100), Age = 20 + (i % 50), Salary = 1000.0 + i }; + } + return models; + } + + public static byte[] Bytes(int count) + { + using var ms = new MemoryStream(); + new FixedWidthWriter().WriteMany(ms, Models(count).AsSpan()); + return ms.ToArray(); + } + } +} diff --git a/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs b/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs index 4f72cd1..8965600 100644 --- a/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs +++ b/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs @@ -21,18 +21,16 @@ public void ValidModel_GeneratesTryParse() """); Assert.Empty(result.GeneratorDiagnostics); + Assert.Empty(result.CompilationErrors); var generated = Assert.Single(result.GeneratedSources); - Assert.Contains("public static bool TryParse", generated, StringComparison.Ordinal); - Assert.Contains("if (line.Length < 18) { model = default!; return false; }", generated, StringComparison.Ordinal); - Assert.Contains("FixedWidthRuntime.String(line.Slice(0, 5), stringPool)", generated, StringComparison.Ordinal); - Assert.Contains("FixedWidthRuntime.TryParse(line.Slice(5, 3), formatProvider", generated, StringComparison.Ordinal); - Assert.Contains("FixedWidthRuntime.TryDouble(line.Slice(8, 5), formatProvider", generated, StringComparison.Ordinal); - Assert.Contains("FixedWidthRuntime.TryFloat(line.Slice(13, 5), formatProvider", generated, StringComparison.Ordinal); - Assert.DoesNotContain("global::System.Math.Min", generated, StringComparison.Ordinal); - Assert.DoesNotContain("FixedWidthRuntime.Column", generated, StringComparison.Ordinal); + Assert.Contains("TryParse(global::System.ReadOnlySpan line, in global::FixedWidthParser.FixedWidthContext context", generated, StringComparison.Ordinal); + Assert.Contains("if (line.Length < 18) { context.ReportShortLine(18); model = default!; return false; }", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.String(line.Slice(0, 5), context)", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.TryParse(line.Slice(5, 3), context, out int __v1)) { context.ReportFailedColumn(\"Quantity\", 5, 3); model = default!; return false; }", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.TryDouble(line.Slice(8, 5), context", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.TryFloat(line.Slice(13, 5), context", generated, StringComparison.Ordinal); + Assert.DoesNotContain("ReadOnlySpan", generated, StringComparison.Ordinal); Assert.Contains("Code = __v0", generated, StringComparison.Ordinal); - Assert.Contains("Quantity = __v1", generated, StringComparison.Ordinal); - Assert.Contains("Price = __v2", generated, StringComparison.Ordinal); Assert.Contains("Discount = __v3", generated, StringComparison.Ordinal); } @@ -48,134 +46,124 @@ public void ValidModel_GeneratesTryFormat() """); Assert.Empty(result.GeneratorDiagnostics); + Assert.Empty(result.CompilationErrors); var generated = Assert.Single(result.GeneratedSources); - Assert.Contains("public static bool TryFormat", generated, StringComparison.Ordinal); - Assert.Contains("if (destination.Length < 8) { charsWritten = 0; return false; }", generated, StringComparison.Ordinal); + Assert.Contains("global::System.Span destination, in global::FixedWidthParser.FixedWidthContext context, out int bytesWritten", generated, StringComparison.Ordinal); + Assert.Contains("if (destination.Length < 8) { bytesWritten = 0; return false; }", generated, StringComparison.Ordinal); Assert.Contains("FixedWidthRuntime.FormatString(model.Code,", generated, StringComparison.Ordinal); Assert.Contains("FixedWidthRuntime.FormatValue(model.Quantity,", generated, StringComparison.Ordinal); - Assert.Contains("ColumnFormatOptions __options0", generated, StringComparison.Ordinal); Assert.Contains("ColumnFormatOptions __options1", generated, StringComparison.Ordinal); } [Fact] - public void CharModel_ColumnNotFormattable_EmitsParseOnly_AndReportsFwp010() + public void EveryColumnKind_CompilesCleanly() { - // A column type that is ISpanParsable but NOT ISpanFormattable: the read side is unaffected - // (FWP010 only blocks the write method), so TryParse is still emitted but TryFormat is not. var result = Run(""" - public readonly struct ParsableNotFormattable : System.ISpanParsable + public readonly record struct Cents(long Value); + + public sealed class CentsConverter : FixedWidthParser.Processors.IFixedWidthConverter { - public static ParsableNotFormattable Parse(System.ReadOnlySpan s, System.IFormatProvider? p) => default; - public static bool TryParse(System.ReadOnlySpan s, System.IFormatProvider? p, out ParsableNotFormattable r) { r = default; return true; } - public static ParsableNotFormattable Parse(string s, System.IFormatProvider? p) => default; - public static bool TryParse(string? s, System.IFormatProvider? p, out ParsableNotFormattable r) { r = default; return true; } + public bool TryParse(System.ReadOnlySpan field, in FixedWidthContext context, out Cents value) { value = default; return true; } + public bool TryFormat(Cents value, System.Span destination, in FixedWidthContext context, out int written) { written = 0; return true; } } - public readonly partial record struct WriteBadModel : IFixedWidthModel + public readonly partial record struct AllKinds : IFixedWidthModel { - [FixedColumn(0, 5)] public ParsableNotFormattable Value { get; init; } + [FixedColumn(0, 5)] public string Text { get; init; } + [FixedColumn(5, 5)] public int? Number { get; init; } + [FixedColumn(10, 8, Format = "yyyyMMdd")] public System.DateOnly Day { get; init; } + [FixedColumn(18, 8, Converter = typeof(CentsConverter))] public Cents? Amount { get; init; } + [FixedColumn(26, 5, TrimChar = '0', TrimMode = TrimMode.Leading)] public decimal Rate { get; init; } + [FixedColumn(31, 36)] public System.Guid Id { get; init; } } """); - Assert.Contains(result.GeneratorDiagnostics, d => string.Equals(d.Id, "FWP010", StringComparison.Ordinal)); + Assert.Empty(result.GeneratorDiagnostics); + Assert.Empty(result.CompilationErrors); var generated = Assert.Single(result.GeneratedSources); - Assert.Contains("public static bool TryParse", generated, StringComparison.Ordinal); - Assert.DoesNotContain("public static bool TryFormat", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.IsBlank(line.Slice(5, 5), (byte)' ')", generated, StringComparison.Ordinal); + Assert.Contains("TryDateOnlyExact(line.Slice(10, 8), \"yyyyMMdd\", context", generated, StringComparison.Ordinal); + Assert.Contains("(byte)'0', global::FixedWidthParser.Attributes.TrimMode.Leading", generated, StringComparison.Ordinal); } [Fact] - public void DualMarkerModel_GeneratesBothCharAndByteTryParse() + public void CharOnlyParsableColumn_ParsesThroughDecodedText() { var result = Run(""" - public readonly partial record struct DualModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly struct OnlyCharParsable : System.ISpanParsable, System.ISpanFormattable { - [FixedColumn(0, 5)] public string Code { get; init; } - [FixedColumn(5, 3)] public int Quantity { get; init; } - [FixedColumn(8, 5)] public double Price { get; init; } - [FixedColumn(13, 5)] public float Discount { get; init; } + public static OnlyCharParsable Parse(System.ReadOnlySpan s, System.IFormatProvider? p) => default; + public static bool TryParse(System.ReadOnlySpan s, System.IFormatProvider? p, out OnlyCharParsable r) { r = default; return true; } + public static OnlyCharParsable Parse(string s, System.IFormatProvider? p) => default; + public static bool TryParse(string? s, System.IFormatProvider? p, out OnlyCharParsable r) { r = default; return true; } + public bool TryFormat(System.Span d, out int w, System.ReadOnlySpan f, System.IFormatProvider? p) { w = 0; return true; } + public string ToString(string? f, System.IFormatProvider? p) => ""; } - """); - Assert.Empty(result.GeneratorDiagnostics); - // Both overloads live in one generated file. - var generated = Assert.Single(result.GeneratedSources); - Assert.Contains("global::System.ReadOnlySpan line", generated, StringComparison.Ordinal); - Assert.Contains("global::System.ReadOnlySpan line", generated, StringComparison.Ordinal); - // Char method still routes through FixedWidthRuntime. - Assert.Contains("FixedWidthParser.FixedWidthRuntime.String(line.Slice(0, 5), stringPool)", generated, StringComparison.Ordinal); - // Byte method routes through Utf8FixedWidthRuntime. - Assert.Contains("Utf8FixedWidthRuntime.String(line.Slice(0, 5), stringPool)", generated, StringComparison.Ordinal); - Assert.Contains("Utf8FixedWidthRuntime.TryParse(line.Slice(5, 3), formatProvider", generated, StringComparison.Ordinal); - Assert.Contains("Utf8FixedWidthRuntime.TryDouble(line.Slice(8, 5), formatProvider", generated, StringComparison.Ordinal); - Assert.Contains("Utf8FixedWidthRuntime.TryFloat(line.Slice(13, 5), formatProvider", generated, StringComparison.Ordinal); - } - - [Fact] - public void Utf8OnlyMarkerModel_GeneratesByteTryParseOnly() - { - var result = Run(""" - public readonly partial record struct Utf8OnlyModel : IUtf8FixedWidthModel + public readonly partial record struct DecodedModel : IFixedWidthModel { - [FixedColumn(0, 5)] public string Code { get; init; } - [FixedColumn(5, 3)] public int Quantity { get; init; } + [FixedColumn(0, 5)] public OnlyCharParsable Value { get; init; } } """); Assert.Empty(result.GeneratorDiagnostics); - var generated = Assert.Single(result.GeneratedSources); - Assert.Contains("global::System.ReadOnlySpan line", generated, StringComparison.Ordinal); - Assert.DoesNotContain("global::System.ReadOnlySpan line", generated, StringComparison.Ordinal); - Assert.Contains("Utf8FixedWidthRuntime.String(line.Slice(0, 5), stringPool)", generated, StringComparison.Ordinal); + Assert.Empty(result.CompilationErrors); + Assert.Contains("FixedWidthRuntime.TryParseDecoded", Assert.Single(result.GeneratedSources), StringComparison.Ordinal); } [Fact] - public void Utf8Model_ColumnOnlyCharParsable_ReportsFwp007() + public void ColumnNotFormattable_ReportsFwp010_AndEmitsNothing() { - // A column type that is ISpanParsable but NOT IUtf8SpanParsable cannot be handled by the - // generated byte parser, so a UTF-8-marked model must report FWP007 and emit no source. + // The interface needs TryFormat too, so a column the writer can't handle blocks the whole emission. var result = Run(""" - public readonly struct OnlyCharParsable : System.ISpanParsable + public readonly struct ParsableNotFormattable : System.ISpanParsable { - public static OnlyCharParsable Parse(System.ReadOnlySpan s, System.IFormatProvider? p) => default; - public static bool TryParse(System.ReadOnlySpan s, System.IFormatProvider? p, out OnlyCharParsable r) { r = default; return true; } - public static OnlyCharParsable Parse(string s, System.IFormatProvider? p) => default; - public static bool TryParse(string? s, System.IFormatProvider? p, out OnlyCharParsable r) { r = default; return true; } + public static ParsableNotFormattable Parse(System.ReadOnlySpan s, System.IFormatProvider? p) => default; + public static bool TryParse(System.ReadOnlySpan s, System.IFormatProvider? p, out ParsableNotFormattable r) { r = default; return true; } + public static ParsableNotFormattable Parse(string s, System.IFormatProvider? p) => default; + public static bool TryParse(string? s, System.IFormatProvider? p, out ParsableNotFormattable r) { r = default; return true; } } - public readonly partial record struct Utf8BadModel : IUtf8FixedWidthModel + public readonly partial record struct WriteBadModel : IFixedWidthModel { - [FixedColumn(0, 5)] public OnlyCharParsable Value { get; init; } + [FixedColumn(0, 5)] public ParsableNotFormattable Value { get; init; } } """); - Assert.Contains(result.GeneratorDiagnostics, d => string.Equals(d.Id, "FWP007", StringComparison.Ordinal)); + Assert.Contains(result.GeneratorDiagnostics, d => string.Equals(d.Id, "FWP010", StringComparison.Ordinal)); Assert.Empty(result.GeneratedSources); } [Fact] - public void DualMarker_ColumnOnlyCharParsable_EmitsCharMethodOnly_AndReportsFwp007() + public void ConverterForAnotherType_ReportsFwp008() { - // A dual-marked model whose column is ISpanParsable but not IUtf8SpanParsable: FWP007 blocks - // ONLY the byte path, so the char TryParse is still emitted (partial emit). var result = Run(""" - public readonly struct OnlyCharParsable : System.ISpanParsable + public sealed class IntConverter : FixedWidthParser.Processors.IFixedWidthConverter { - public static OnlyCharParsable Parse(System.ReadOnlySpan s, System.IFormatProvider? p) => default; - public static bool TryParse(System.ReadOnlySpan s, System.IFormatProvider? p, out OnlyCharParsable r) { r = default; return true; } - public static OnlyCharParsable Parse(string s, System.IFormatProvider? p) => default; - public static bool TryParse(string? s, System.IFormatProvider? p, out OnlyCharParsable r) { r = default; return true; } + public bool TryParse(System.ReadOnlySpan field, in FixedWidthContext context, out int value) { value = 0; return true; } + public bool TryFormat(int value, System.Span destination, in FixedWidthContext context, out int written) { written = 0; return true; } } - public readonly partial record struct DualPartialModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct MismatchModel : IFixedWidthModel { - [FixedColumn(0, 5)] public OnlyCharParsable Value { get; init; } + [FixedColumn(0, 5, Converter = typeof(IntConverter))] public string Value { get; init; } } """); - Assert.Contains(result.GeneratorDiagnostics, d => string.Equals(d.Id, "FWP007", StringComparison.Ordinal)); - var generated = Assert.Single(result.GeneratedSources); - Assert.Contains("global::System.ReadOnlySpan line", generated, StringComparison.Ordinal); - Assert.DoesNotContain("global::System.ReadOnlySpan line", generated, StringComparison.Ordinal); + Assert.Contains(result.GeneratorDiagnostics, d => string.Equals(d.Id, "FWP008", StringComparison.Ordinal)); + Assert.Empty(result.GeneratedSources); + } + + [Theory] + [InlineData("[FixedColumn(0, 5, Alignment = Alignment.Right, Padding = '\u00e9')] public int Value { get; init; }", "Padding")] + [InlineData("[FixedColumn(0, 5, TrimChar = '\u00a0')] public int Value { get; init; }", "TrimChar")] + public void NonAsciiPaddingOrTrimChar_ReportsFwp012(string column, string setting) + { + var result = Run("public readonly partial record struct NonAsciiModel : IFixedWidthModel { " + column + " }"); + + var diagnostic = Assert.Single(result.GeneratorDiagnostics, d => string.Equals(d.Id, "FWP012", StringComparison.Ordinal)); + Assert.Contains(setting, diagnostic.GetMessage(System.Globalization.CultureInfo.InvariantCulture), StringComparison.Ordinal); + Assert.Empty(result.GeneratedSources); } [Fact] @@ -420,7 +408,7 @@ public void IncrementalCaching_ColumnChange_RegeneratesOutput() var second = driver.GetRunResult().Results[0].GeneratedSources.Single().SourceText.ToString(); Assert.NotEqual(first, second); - Assert.Contains("line.Length < 8", second, StringComparison.Ordinal); + Assert.Contains("line.Length < 8)", second, StringComparison.Ordinal); } private static CSharpCompilation CreateCompilation(string modelSource, CSharpParseOptions parseOptions) @@ -464,7 +452,7 @@ namespace GeneratorSmoke; GeneratorDriver driver = CSharpGeneratorDriver.Create([generator], parseOptions: parseOptions); driver = driver.RunGeneratorsAndUpdateCompilation( compilation, - out _, + out var updated, out var generatorDiagnostics, TestContext.Current.CancellationToken); @@ -472,8 +460,11 @@ namespace GeneratorSmoke; var generatedSources = runResult.GeneratedTrees .Select(t => t.GetText(TestContext.Current.CancellationToken).ToString()) .ToImmutableArray(); + var compilationErrors = updated.GetDiagnostics(TestContext.Current.CancellationToken) + .Where(d => d.Severity == DiagnosticSeverity.Error) + .ToImmutableArray(); - return new GeneratorRunResult(generatorDiagnostics, generatedSources); + return new GeneratorRunResult(generatorDiagnostics, generatedSources, compilationErrors); } private static readonly Lazy> References = new(CreateReferences); @@ -508,10 +499,12 @@ void Add(string path) private sealed class GeneratorRunResult( ImmutableArray generatorDiagnostics, - ImmutableArray generatedSources) + ImmutableArray generatedSources, + ImmutableArray compilationErrors) { public ImmutableArray GeneratorDiagnostics { get; } = generatorDiagnostics; public ImmutableArray GeneratedSources { get; } = generatedSources; + public ImmutableArray CompilationErrors { get; } = compilationErrors; } } } diff --git a/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj b/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj index d349b3b..68d9094 100644 --- a/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj +++ b/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj @@ -3,7 +3,7 @@ true true - net8.0;net10.0 + net10.0 enable enable false diff --git a/tests/FixedWidthParser.Tests/AllocationGuardTests.cs b/tests/FixedWidthParser.Tests/AllocationGuardTests.cs index 18dcf92..ae7e200 100644 --- a/tests/FixedWidthParser.Tests/AllocationGuardTests.cs +++ b/tests/FixedWidthParser.Tests/AllocationGuardTests.cs @@ -1,4 +1,3 @@ -using System.Globalization; using System.Text; using CommunityToolkit.HighPerformance.Buffers; using FixedWidthParser.Readers; @@ -7,176 +6,96 @@ namespace FixedWidthParser.Tests { /// /// Fast allocation-regression guard (runs in milliseconds, unlike the BenchmarkDotNet suite). - /// Proves the pooled read hot path allocates nothing per line: with a - /// and a repeated value, the only would-be per-line allocation (the string column) is interned - /// away, and the struct enumerator adds no heap per iteration. We read two different line counts - /// and assert the marginal per-line cost is ~0 — subtracting the two cancels the fixed - /// per-call overhead (enumerable, closure, buffer rent) and JIT/warmup noise, so the test is - /// robust without pinning exact byte counts. + /// Proves the pooled read hot path allocates nothing per line: with a and a + /// repeated value, the string column is interned away and the struct enumerator adds no heap per + /// iteration. Reading two line counts and subtracting cancels the fixed per-call overhead. /// public class AllocationGuardTests { - /// - /// stores the stream - /// directly instead of a Func<Stream>, so a call allocates only the enumerable - /// object — no captured-variable display class and no delegate. Measured at 64 B/call; a - /// regressed closure would add a delegate (~56 B+) and push it well past the threshold. - /// - [Fact] - public void ByteReaderRead_AllocatesEnumerableOnly_NoClosure() - { - var reader = new FixedWidthByteReader(); - using var stream = new MemoryStream(); - _ = reader.Read(stream); // warmup/JIT - - const int n = 10_000; - long before = GC.GetAllocatedBytesForCurrentThread(); - for (int i = 0; i < n; i++) - { - _ = reader.Read(stream); - } - - double perCall = (GC.GetAllocatedBytesForCurrentThread() - before) / (double)n; - - Assert.True(perCall <= 80, $"Read allocated {perCall} B/call (expected ~64 B: enumerable only, no closure)."); - } + private const string Line = "ABC"; // CodeModel/GenCodeModel: [FixedColumn(0, 3)] Code + private const int Small = 2_000; + private const int Large = 6_000; /// - /// stores the source in a - /// by-value TextReaderSource struct instead of a Func<TextReader>, so a call - /// allocates only the enumerable object — no captured-variable display class and no delegate. - /// Measured at 88 B/call; a regressed closure would add a delegate (~56 B+) past the threshold. + /// A read call stores the stream directly instead of a Func<Stream>, so it allocates + /// only the enumerable — no display class and no delegate (~56 B more if one sneaks back in). /// [Fact] - public void CharReaderRead_AllocatesEnumerableOnly_NoClosure() + public void Read_AllocatesEnumerableOnly_NoClosure() { var reader = new FixedWidthReader(); - using var tr = new StringReader(""); - _ = reader.Read(tr); // warmup/JIT - - const int n = 10_000; - long before = GC.GetAllocatedBytesForCurrentThread(); - for (int i = 0; i < n; i++) - { - _ = reader.Read(tr); - } - - double perCall = (GC.GetAllocatedBytesForCurrentThread() - before) / (double)n; - - Assert.True(perCall <= 104, $"Read allocated {perCall} B/call (expected ~88 B: enumerable only, no closure)."); - } - - /// - /// - /// stores the stream directly (like the reflection byte reader), so a call allocates only the - /// enumerable object — no captured-variable display class and no delegate. - /// - [Fact] - public void GeneratedUtf8Read_AllocatesEnumerableOnly_NoClosure() - { using var stream = new MemoryStream(); - _ = FixedWidthUtf8.Read(stream); // warmup/JIT - - const int n = 10_000; - long before = GC.GetAllocatedBytesForCurrentThread(); - for (int i = 0; i < n; i++) - { - _ = FixedWidthUtf8.Read(stream); - } - - double perCall = (GC.GetAllocatedBytesForCurrentThread() - before) / (double)n; + _ = reader.Read(stream); + _ = FixedWidth.Read(stream); - Assert.True(perCall <= 80, $"Read allocated {perCall} B/call (expected ~64 B: enumerable only, no closure)."); - } - - private const string Line = "ABC"; // CodeModel/GenCodeModel: [FixedColumn(0, 3)] Code - private const int Small = 2_000; - private const int Large = 6_000; - - private static string Repeat(int lines) - { - var sb = new StringBuilder((Line.Length + 1) * lines); - for (int i = 0; i < lines; i++) - { - sb.Append(Line).Append('\n'); - } - - return sb.ToString(); + Assert.True(PerCall(() => reader.Read(stream)) <= 72, "Reflection Read allocated more than the enumerable."); + Assert.True(PerCall(() => FixedWidth.Read(stream)) <= 72, "Generated Read allocated more than the enumerable."); } [Fact] public void ReflectionPooledReader_AllocatesNothingPerLine() { - var reader = new FixedWidthReader(CultureInfo.InvariantCulture, new StringPool()); - string small = Repeat(Small), large = Repeat(Large); - - // Warmup: JIT every path and intern "ABC" into the pool. + var reader = new FixedWidthReader(new FixedWidthReaderOptions { StringPool = new StringPool() }); + byte[] small = Repeat(Small), large = Repeat(Large); SumReflection(reader, small); SumReflection(reader, large); double perLine = MarginalPerLine(() => SumReflection(reader, small), () => SumReflection(reader, large)); + Assert.True(perLine <= 1, $"Reflection pooled reader regressed to {perLine} B/line (expected ~0)."); } [Fact] public void GeneratedPooledReader_AllocatesNothingPerLine() { - var pool = new StringPool(); - string small = Repeat(Small), large = Repeat(Large); + var options = new FixedWidthReaderOptions { StringPool = new StringPool() }; + byte[] small = Repeat(Small), large = Repeat(Large); + SumGenerated(options, small); + SumGenerated(options, large); - SumGenerated(pool, small); - SumGenerated(pool, large); + double perLine = MarginalPerLine(() => SumGenerated(options, small), () => SumGenerated(options, large)); - double perLine = MarginalPerLine(() => SumGenerated(pool, small), () => SumGenerated(pool, large)); Assert.True(perLine <= 1, $"Generated pooled reader regressed to {perLine} B/line (expected ~0)."); } - [Fact] - public void GeneratedUtf8PooledReader_AllocatesNothingPerLine() + private static double PerCall(Func read) { - var pool = new StringPool(); - byte[] small = Encoding.UTF8.GetBytes(Repeat(Small)), large = Encoding.UTF8.GetBytes(Repeat(Large)); - - SumGeneratedUtf8(pool, small); - SumGeneratedUtf8(pool, large); - - double perLine = MarginalPerLine(() => SumGeneratedUtf8(pool, small), () => SumGeneratedUtf8(pool, large)); - Assert.True(perLine <= 1, $"Generated UTF-8 pooled reader regressed to {perLine} B/line (expected ~0)."); + const int n = 10_000; + long before = GC.GetAllocatedBytesForCurrentThread(); + for (int i = 0; i < n; i++) + { + _ = read(); + } + return (GC.GetAllocatedBytesForCurrentThread() - before) / (double)n; } - private static int SumReflection(FixedWidthReader reader, string text) + private static byte[] Repeat(int lines) { - int total = 0; - using var sr = new StringReader(text); - foreach (var m in reader.Read(sr)) + var sb = new StringBuilder((Line.Length + 1) * lines); + for (int i = 0; i < lines; i++) { - total += m.Code.Length; + sb.Append(Line).Append('\n'); } - - return total; + return Encoding.UTF8.GetBytes(sb.ToString()); } - private static int SumGenerated(StringPool pool, string text) + private static int SumReflection(FixedWidthReader reader, byte[] bytes) { int total = 0; - using var sr = new StringReader(text); - foreach (var m in FixedWidth.Read(sr, CultureInfo.InvariantCulture, pool)) + foreach (var m in reader.Read(new MemoryStream(bytes, writable: false))) { total += m.Code.Length; } - return total; } - private static int SumGeneratedUtf8(StringPool pool, byte[] bytes) + private static int SumGenerated(FixedWidthReaderOptions options, byte[] bytes) { int total = 0; - using var ms = new MemoryStream(bytes, writable: false); - foreach (var m in FixedWidthUtf8.Read(ms, formatProvider: CultureInfo.InvariantCulture, stringPool: pool)) + foreach (var m in FixedWidth.Read(new MemoryStream(bytes, writable: false), options: options)) { total += m.Code.Length; } - return total; } diff --git a/tests/FixedWidthParser.Tests/ByteReaderStreamTests.cs b/tests/FixedWidthParser.Tests/ByteReaderStreamTests.cs deleted file mode 100644 index c9d184c..0000000 --- a/tests/FixedWidthParser.Tests/ByteReaderStreamTests.cs +++ /dev/null @@ -1,342 +0,0 @@ -using System.Text; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Readers; -using static FixedWidthParser.Tests.TestHelpers; - -namespace FixedWidthParser.Tests -{ - /// - /// Covers the streaming Stream/file overloads of (sync - /// and async): raw-byte reading with no transcode, line-ending handling, BOM skipping, buffer - /// growth, error reporting, leaveOpen behavior, and byte/char parity. - /// - public class ByteReaderStreamTests - { - private const string TwoPeople = - "John Doe 30 60000.00 \n" + - "Jane 28 55000.00 "; - - private static MemoryStream Utf8(string text) - { - return new(Encoding.UTF8.GetBytes(text)); - } - - // ----------------------------- Synchronous ----------------------------- - - [Fact] - public void Read_Stream_ParsesAll() - { - var reader = new FixedWidthByteReader(Inv); - using var stream = Utf8(TwoPeople); - - var people = reader.Read(stream).ToList(); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(30, people[0].Age); - Assert.Equal(60000.00, people[0].Salary, 2); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Theory] - [InlineData("ABC\nDEF\n")] // LF + trailing newline - [InlineData("ABC\r\nDEF\r\n")] // CRLF - [InlineData("ABC\nDEF")] // no trailing newline on the last line - public void Read_HandlesLineEndingsAndTrailingNewline(string text) - { - var reader = new FixedWidthByteReader(); - using var stream = Utf8(text); - - var codes = reader.Read(stream).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public void Read_SkipsEmptyLines() - { - var reader = new FixedWidthByteReader(); - using var stream = Utf8("ABC\n\n\nDEF\n"); - - var codes = reader.Read(stream).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public void Read_SkipsLeadingUtf8Bom() - { - var reader = new FixedWidthByteReader(); - byte[] bytes = [0xEF, 0xBB, 0xBF, .. "ABC\nDEF\n"u8]; - using var stream = new MemoryStream(bytes); - - var codes = reader.Read(stream).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public void Read_LineLongerThanBuffer_StillParses() - { - // A tiny bufferSize forces buffer compaction and growth (25-byte lines). - var reader = new FixedWidthByteReader(Inv, bufferSize: 4); - using var stream = Utf8(TwoPeople); - - var people = reader.Read(stream).ToList(); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public void Read_InvalidLine_ThrowsWithLineNumber() - { - var reader = new FixedWidthByteReader(Inv); - using var stream = Utf8("John Doe 30 60000.00 \nJane XX 55000.00 "); - - var ex = Assert.Throws(() => reader.Read(stream).ToList()); - - Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); - } - - [Fact] - public void Read_LeaveOpenTrue_KeepsStreamOpen() - { - var reader = new FixedWidthByteReader(); - using var stream = Utf8("ABC\nDEF\n"); - - _ = reader.Read(stream, leaveOpen: true).ToList(); - - Assert.True(stream.CanRead); - } - - [Fact] - public void Read_LeaveOpenFalse_DisposesStream() - { - var reader = new FixedWidthByteReader(); - var stream = Utf8("ABC\nDEF\n"); - - _ = reader.Read(stream).ToList(); // default leaveOpen: false - - Assert.False(stream.CanRead); - } - - [Fact] - public void Read_NullStream_Throws() - { - var reader = new FixedWidthByteReader(); - - Assert.Throws(() => reader.Read(null!)); - } - - [Fact] - public void GetEnumerator_IsStruct_ForAllocationFreeForeach() - { - Assert.True(typeof(RecordEnumeratorCore, StreamSource>).IsValueType); - - var reader = new FixedWidthByteReader(); - using var stream = Utf8("ABC\nDEF\n"); - int count = 0; - foreach (var _ in reader.Read(stream)) - { - count++; - } - - Assert.Equal(2, count); - } - - [Fact] - public void ReadFile_IsReEnumerable() - { - string path = Path.GetTempFileName(); - try - { - File.WriteAllText(path, "ABC\nDEF\n"); - var reader = new FixedWidthByteReader(); - - var first = reader.ReadFile(path).Select(m => m.Code).ToList(); - var second = reader.ReadFile(path).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], first); - Assert.Equal(first, second); - } - finally - { - File.Delete(path); - } - } - - // ----------------------------- Asynchronous ----------------------------- - - [Fact] - public async Task ReadAsync_Stream_ParsesAll() - { - var reader = new FixedWidthByteReader(Inv); - await using var stream = Utf8(TwoPeople); - - var people = await reader.ReadAsync(stream).ToListAsync().ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(30, people[0].Age); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public async Task ReadAsync_LineLongerThanBuffer_StillParses() - { - var reader = new FixedWidthByteReader(Inv, bufferSize: 4); - await using var stream = Utf8(TwoPeople); - - var people = await reader.ReadAsync(stream).ToListAsync().ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public async Task ReadAsync_SkipsLeadingUtf8Bom() - { - var reader = new FixedWidthByteReader(); - byte[] bytes = [0xEF, 0xBB, 0xBF, .. "ABC\nDEF\n"u8]; - await using var stream = new MemoryStream(bytes); - - var codes = await reader.ReadAsync(stream).Select(m => m.Code).ToListAsync().ConfigureAwait(true); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public async Task ReadAsync_InvalidLine_ThrowsWithLineNumber() - { - var reader = new FixedWidthByteReader(Inv); - await using var stream = Utf8("John Doe 30 60000.00 \nJane XX 55000.00 "); - - var ex = await Assert.ThrowsAsync(async () => - { - await foreach (var _ in reader.ReadAsync(stream).ConfigureAwait(false)) { } - }).ConfigureAwait(true); - - Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); - } - - [Fact] - public async Task ReadAsync_CancelledToken_Throws() - { - var reader = new FixedWidthByteReader(); - await using var stream = Utf8("ABC\nDEF\n"); - using var cts = new CancellationTokenSource(); - await cts.CancelAsync().ConfigureAwait(true); - - await Assert.ThrowsAnyAsync(async () => - { - await foreach (var _ in reader.ReadAsync(stream).WithCancellation(cts.Token).ConfigureAwait(false)) { } - }).ConfigureAwait(true); - } - - [Fact] - public async Task ReadAsync_LeaveOpenTrue_KeepsStreamOpen() - { - var reader = new FixedWidthByteReader(); - await using var stream = Utf8("ABC\nDEF\n"); - - _ = await reader.ReadAsync(stream, leaveOpen: true).ToListAsync().ConfigureAwait(true); - - Assert.True(stream.CanRead); - } - - [Fact] - public async Task ReadAsync_LeaveOpenFalse_DisposesStream() - { - var reader = new FixedWidthByteReader(); - var stream = Utf8("ABC\nDEF\n"); - - _ = await reader.ReadAsync(stream).ToListAsync().ConfigureAwait(true); - - Assert.False(stream.CanRead); - } - - [Fact] - public void ReadAsync_NullStream_Throws() - { - var reader = new FixedWidthByteReader(); - - Assert.Throws(() => reader.ReadAsync((Stream)null!)); - } - - [Fact] - public async Task ReadFileAsync_MatchesSyncReadFile() - { - string path = Path.GetTempFileName(); - try - { - await File.WriteAllTextAsync(path, "ABC\nDEF\nGHI\n").ConfigureAwait(true); - var reader = new FixedWidthByteReader(); - - var asyncCodes = await reader.ReadFileAsync(path).Select(m => m.Code).ToListAsync().ConfigureAwait(true); - var syncCodes = reader.ReadFile(path).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF", "GHI"], asyncCodes); - Assert.Equal(syncCodes, asyncCodes); - } - finally - { - File.Delete(path); - } - } - - // ----------------------------- StringPool ----------------------------- - - [Fact] - public void Read_Stream_WithStringPool_InternsRepeatedValues() - { - var pool = new StringPool(); - var reader = new FixedWidthByteReader(stringPool: pool); - using var stream = Utf8("ABC\nABC\n"); - - var codes = reader.Read(stream).ToList(); - - Assert.Equal("ABC", codes[0].Code); - Assert.Same(codes[0].Code, codes[1].Code); - } - - [Fact] - public async Task ReadAsync_Stream_WithStringPool_InternsRepeatedValues() - { - var pool = new StringPool(); - var reader = new FixedWidthByteReader(stringPool: pool); - await using var stream = Utf8("ABC\nABC\n"); - - var codes = await reader.ReadAsync(stream).ToListAsync().ConfigureAwait(true); - - Assert.Equal("ABC", codes[0].Code); - Assert.Same(codes[0].Code, codes[1].Code); - } - - // ----------------------------- char/byte parity ----------------------------- - - [Fact] - public void ByteReader_MatchesCharReader_ForAsciiStream() - { - var charReader = new FixedWidthReader(Inv); - var byteReader = new FixedWidthByteReader(Inv); - - var fromChars = charReader.Read(new StringReader(TwoPeople)).ToList(); - using var stream = Utf8(TwoPeople); - var fromBytes = byteReader.Read(stream).ToList(); - - Assert.Equal(fromChars.Count, fromBytes.Count); - for (int i = 0; i < fromChars.Count; i++) - { - Assert.Equal(fromChars[i].Name, fromBytes[i].Name); - Assert.Equal(fromChars[i].Age, fromBytes[i].Age); - Assert.Equal(fromChars[i].Salary, fromBytes[i].Salary, 2); - } - } - } -} diff --git a/tests/FixedWidthParser.Tests/ByteReaderTests.cs b/tests/FixedWidthParser.Tests/ByteReaderTests.cs deleted file mode 100644 index 13d77d4..0000000 --- a/tests/FixedWidthParser.Tests/ByteReaderTests.cs +++ /dev/null @@ -1,373 +0,0 @@ -using System.Globalization; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Parsers; -using FixedWidthParser.Readers; -using static FixedWidthParser.Tests.TestHelpers; - -namespace FixedWidthParser.Tests -{ - /// - /// Covers the UTF-8 / byte parse path: and the - /// facade. Column offsets are measured in bytes; - /// all fixtures here are ASCII, so byte offsets equal character offsets. Mirrors the assertions - /// in / to prove byte/char parity. - /// - public class ByteReaderTests - { - private static readonly CultureInfo DeDe = CultureInfo.GetCultureInfo("de-DE"); - - // ----------------------- Core parsing ----------------------- - - [Fact] - public void Parser_ParsesAllColumns() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("John Doe 30 60000.00 "u8, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal("John Doe", model.Name); - Assert.Equal(30, model.Age); - Assert.Equal(60000.00, model.Salary, 2); - } - - [Fact] - public void Parser_StringColumn_TrimsTrailingSpacesAndDecodes() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("ABC"u8, null, null, out var model); - - Assert.True(ok); - Assert.Equal("ABC", model.Code); - } - - [Fact] - public void Parser_DecimalColumn_UsesUtf8SpanParsableFallback() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("1234.56 "u8, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal(1234.56m, model.Amount); - } - - [Fact] - public void Parser_FloatColumn_UsesCsFastFloat() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("3.14 "u8, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal(3.14f, model.Value, 2); - } - - [Fact] - public void Parser_FieldBasedModel_Parses() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("ABCDE0042"u8, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal("ABCDE", model.Code); - Assert.Equal(42, model.Quantity); - } - -#if NET9_0_OR_GREATER - [Fact] - public void Parser_SupportsRefStructModel() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("John Doe 30 60000.00 "u8, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal("John Doe", model.Name); - Assert.Equal(30, model.Age); - Assert.Equal(60000.00, model.Salary, 2); - } -#endif - - // ----------------------- Rejection paths ----------------------- - - [Fact] - public void Parser_LineShorterThanLayout_ReturnsFalse() - { - var parser = new Utf8FixedWidthParser(); - - // Layout requires 25 bytes; supply fewer. - bool ok = parser.TryParse("John Doe 30"u8, Inv, null, out _); - - Assert.False(ok); - } - - [Fact] - public void Parser_InvalidNumericColumn_ReturnsFalse() - { - var parser = new Utf8FixedWidthParser(); - - // "XX" in the Age column [10,5) is not an integer. - bool ok = parser.TryParse("John Doe XX 60000.00 "u8, Inv, null, out _); - - Assert.False(ok); - } - - // ----------------------- Culture handling ----------------------- - - [Fact] - public void Parser_DoubleColumn_HonorsCommaCulture() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 1234,50 "u8, DeDe, null, out var model); - - Assert.True(ok); - Assert.Equal(1234.50, model.Salary, 2); - } - - [Fact] - public void Parser_DoubleColumn_NullProvider_DefaultsToDot() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 1234.50 "u8, null, null, out var model); - - Assert.True(ok); - Assert.Equal(1234.50, model.Salary, 2); - } - - [Fact] - public void Parser_DecimalColumn_HonorsCommaCulture() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("1234,56 "u8, DeDe, null, out var model); - - Assert.True(ok); - Assert.Equal(1234.56m, model.Amount); - } - - [Fact] - public void Parser_FloatColumn_HonorsCommaCulture() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("3,14 "u8, DeDe, null, out var model); - - Assert.True(ok); - Assert.Equal(3.14f, model.Value, 2); - } - - [Fact] - public void Parser_DoubleColumn_DeDeThousandsSeparator_ParsesFullValue() - { - // de-DE: '.' groups thousands, ',' is the decimal separator — "1.234,50" means 1234.50. - // Regression coverage for a bug where csFastFloat silently truncated at the '.' it wasn't - // told about and returned 1.0 instead of failing or parsing the full value. - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 1.234,50 "u8, DeDe, null, out var model); - - Assert.True(ok); - Assert.Equal(1234.50, model.Salary, 2); - } - - [Fact] - public void Parser_DoubleColumn_DotCultureThousandsSeparator_ParsesFullValue() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 1,234.50 "u8, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal(1234.50, model.Salary, 2); - } - - // ----- Null provider means invariant for every column type on the byte path too (see - // CultureTests.Parse_DecimalColumn_NullProvider_DefaultsToInvariant for the char-path version and - // rationale). ----- - - [Fact] - public void Parser_DecimalColumn_NullProvider_DefaultsToInvariant() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("1234.56 "u8, null, null, out var model); - - Assert.True(ok); - Assert.Equal(1234.56m, model.Amount); - } - - [Fact] - public void Parser_DoubleColumn_TrailingGarbageAfterNumber_Fails() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 12x "u8, Inv, null, out _); - - Assert.False(ok); - } - - // ----- Right-aligned (leading-space-padded) numeric columns: same regression as - // CultureTests' char-path equivalents — csFastFloat's characters_consumed did not reliably - // include skipped leading whitespace across its double/float overloads. ----- - - [Fact] - public void Parser_DoubleColumn_RightAligned_DotCulture_Parses() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 123.45"u8, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal(123.45, model.Salary, 2); - } - - [Fact] - public void Parser_FloatColumn_RightAligned_DotCulture_Parses() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse(" 3.14"u8, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal(3.14f, model.Value, 2); - } - - [Fact] - public void Parser_DoubleColumn_RightAligned_CommaCulture_Parses() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 123,45"u8, DeDe, null, out var model); - - Assert.True(ok); - Assert.Equal(123.45, model.Salary, 2); - } - - [Fact] - public void Parser_FloatColumn_RightAligned_CommaCulture_Parses() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse(" 3,14"u8, DeDe, null, out var model); - - Assert.True(ok); - Assert.Equal(3.14f, model.Value, 2); - } - - [Fact] - public void Parser_DoubleColumn_NonAsciiSeparatorCulture_Throws() - { - // The byte parser matches the decimal separator against raw UTF-8 bytes, so a non-ASCII - // separator (here U+066B ARABIC DECIMAL SEPARATOR) cannot be represented as a single byte. - // It must be rejected with a clear error rather than silently truncated to the wrong byte. - var culture = (CultureInfo)CultureInfo.InvariantCulture.Clone(); - culture.NumberFormat.NumberDecimalSeparator = "٫"; - - var parser = new Utf8FixedWidthParser(); - - var ex = Assert.Throws( - () => parser.TryParse("Bob 30 1234.50 "u8, culture, null, out _)); - Assert.Contains("ASCII", ex.Message, StringComparison.Ordinal); - } - - // ----------------------- Facade ----------------------- - - [Fact] - public void ByteReader_TryParse_UsesConfiguredCulture() - { - var reader = new FixedWidthByteReader(Inv); - - bool ok = reader.TryParse("Jane 28 55000.00 "u8, out var model); - - Assert.True(ok); - Assert.Equal("Jane", model.Name); - Assert.Equal(28, model.Age); - Assert.Equal(55000.00, model.Salary, 2); - } - - [Fact] - public void ByteReader_TryParse_CommaCulture() - { - var reader = new FixedWidthByteReader(DeDe); - - bool ok = reader.TryParse("1234,56 "u8, out var model); - - Assert.True(ok); - Assert.Equal(1234.56m, model.Amount); - } - - [Fact] - public void ByteReader_TryParse_TooShort_ReturnsFalse() - { - var reader = new FixedWidthByteReader(Inv); - - bool ok = reader.TryParse("short"u8, out _); - - Assert.False(ok); - } - - // ----------------------- char/byte parity ----------------------- - - [Fact] - public void ByteParser_MatchesCharParser_ForAsciiInput() - { - const string line = "John Doe 30 60000.00 "; - var charParser = new FixedWidthParser(); - var byteParser = new Utf8FixedWidthParser(); - - bool charOk = charParser.TryParse(line, Inv, null, out var fromChars); - bool byteOk = byteParser.TryParse("John Doe 30 60000.00 "u8, Inv, null, out var fromBytes); - - Assert.True(charOk); - Assert.True(byteOk); - Assert.Equal(fromChars.Name, fromBytes.Name); - Assert.Equal(fromChars.Age, fromBytes.Age); - Assert.Equal(fromChars.Salary, fromBytes.Salary, 2); - } - - // ----------------------- StringPool interning ----------------------- - - [Fact] - public void Parser_WithStringPool_InternsRepeatedValues() - { - var pool = new StringPool(); - var parser = new Utf8FixedWidthParser(); - - Assert.True(parser.TryParse("ABC"u8, null, pool, out var first)); - Assert.True(parser.TryParse("ABC"u8, null, pool, out var second)); - - Assert.Equal("ABC", first.Code); - Assert.Same(first.Code, second.Code); - } - - [Fact] - public void Parser_WithStringPool_MatchesNonPooledDecode() - { - var pool = new StringPool(); - var parser = new Utf8FixedWidthParser(); - - Assert.True(parser.TryParse("ABC"u8, null, null, out var nonPooled)); - Assert.True(parser.TryParse("ABC"u8, null, pool, out var pooled)); - - Assert.Equal(nonPooled.Code, pooled.Code); - } - - [Fact] - public void ByteReader_TryParse_WithStringPool_InternsRepeatedValues() - { - var pool = new StringPool(); - var reader = new FixedWidthByteReader(stringPool: pool); - - Assert.True(reader.TryParse("ABC"u8, out var first)); - Assert.True(reader.TryParse("ABC"u8, out var second)); - - Assert.Same(first.Code, second.Code); - } - - } -} diff --git a/tests/FixedWidthParser.Tests/ColumnValidationTests.cs b/tests/FixedWidthParser.Tests/ColumnValidationTests.cs index 32808e2..5d2d5d8 100644 --- a/tests/FixedWidthParser.Tests/ColumnValidationTests.cs +++ b/tests/FixedWidthParser.Tests/ColumnValidationTests.cs @@ -45,10 +45,24 @@ public void Parser_NoColumns_ThrowsOnConstruction() } [Fact] - public void Utf8Parser_NoColumns_ThrowsOnConstruction() + public void Parser_NonAsciiTrimChar_ThrowsOnConstruction() { - var ex = Assert.Throws(() => new Utf8FixedWidthParser()); - Assert.Contains("no [FixedColumn]", ex.Message, StringComparison.Ordinal); + var ex = Assert.Throws(() => new FixedWidthParser()); + Assert.Contains("TrimChar", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Writer_NonAsciiPadding_ThrowsOnConstruction() + { + var ex = Assert.Throws(() => new FixedWidthWriter()); + Assert.Contains("Padding", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Parser_UnsupportedColumnType_ThrowsOnConstruction() + { + var ex = Assert.Throws(() => new FixedWidthParser()); + Assert.Contains("Converter", ex.Message, StringComparison.Ordinal); } [Fact] diff --git a/tests/FixedWidthParser.Tests/ConverterAttributeTests.cs b/tests/FixedWidthParser.Tests/ConverterAttributeTests.cs index b3a8026..4352e4e 100644 --- a/tests/FixedWidthParser.Tests/ConverterAttributeTests.cs +++ b/tests/FixedWidthParser.Tests/ConverterAttributeTests.cs @@ -1,4 +1,3 @@ -using System.Text; using FixedWidthParser.Parsers; using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; @@ -6,72 +5,39 @@ namespace FixedWidthParser.Tests { /// - /// Covers : a per-property converter that - /// takes priority over the built-in ISpanParsable fallback, across all four combinations of - /// reflection/generated and char/UTF-8, plus the write side and the build-time mismatch check. + /// : a per-property converter that takes + /// priority over the built-in parsers, on the reflection and generated paths, for parse and write, + /// plus the build-time mismatch check. /// public class ConverterAttributeTests { [Fact] - public void Reflection_Char_RoundTrips() + public void Reflection_RoundTrips() { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); var original = new CentsConverterModel { Amount = new CentsValue(12345) }; - string line = WriteOne(writer, original); - bool ok = parser.TryParse(line, Inv, null, out var parsed); + string line = WriteOne(new FixedWidthWriter(), original); - Assert.True(ok); + Assert.True(new FixedWidthParser().TryParse(Bytes(line), Ctx, out var parsed)); Assert.Equal(original.Amount, parsed.Amount); } [Fact] - public void Reflection_Utf8_Parses() + public void Generated_MatchesReflection() { - var parser = new Utf8FixedWidthParser(); - byte[] bytes = Encoding.UTF8.GetBytes("12345 "); + bool okR = new FixedWidthParser().TryParse("12345 "u8, Ctx, out var r); + bool okG = FixedWidth.TryParse("12345 "u8, Ctx, out var g); - bool ok = parser.TryParse(bytes, Inv, null, out var parsed); - - Assert.True(ok); - Assert.Equal(new CentsValue(12345), parsed.Amount); - } - - [Fact] - public void Generated_Char_MatchesReflection() - { - var reflection = new FixedWidthParser(); - const string line = "12345 "; - - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); - - Assert.Equal(okR, okG); - Assert.Equal(r.Amount, g.Amount); - } - - [Fact] - public void Generated_Utf8_MatchesReflection() - { - byte[] bytes = Encoding.UTF8.GetBytes("12345 "); - var reflection = new Utf8FixedWidthParser(); - - bool okR = reflection.TryParse(bytes, Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse(bytes, Inv, null, out var g); - - Assert.Equal(okR, okG); + Assert.True(okR); + Assert.True(okG); Assert.Equal(r.Amount, g.Amount); } [Fact] public void Converter_RejectingInput_ReturnsFalse() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("notanum ", Inv, null, out _); - - Assert.False(ok); + Assert.False(new FixedWidthParser().TryParse("notanum "u8, Ctx, out _)); + Assert.False(FixedWidth.TryParse("notanum "u8, Ctx, out _)); } [Fact] diff --git a/tests/FixedWidthParser.Tests/CorrectnessTests.cs b/tests/FixedWidthParser.Tests/CorrectnessTests.cs index e8a4d24..b33b0c9 100644 --- a/tests/FixedWidthParser.Tests/CorrectnessTests.cs +++ b/tests/FixedWidthParser.Tests/CorrectnessTests.cs @@ -1,4 +1,3 @@ -using System.Text; using FixedWidthParser.Parsers; using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; @@ -8,156 +7,77 @@ namespace FixedWidthParser.Tests public class CorrectnessTests { [Fact] - public void DateTimeExact_RoundTrip_AllPaths() + public void DateTimeExact_RoundTrip_BothPaths() { var date = new DateTime(2026, 7, 16, 0, 0, 0, DateTimeKind.Unspecified); var dateOnly = new DateOnly(2026, 7, 16); const string input = "2026071620260716"; - // 1. Reflection Char - var pChar = new FixedWidthParser(); - Assert.True(pChar.TryParse(input, Inv, null, out var model1)); - Assert.Equal(date, model1.Date); - Assert.Equal(dateOnly, model1.DateOnlyVal); + var parser = new FixedWidthParser(); + Assert.True(parser.TryParse(Bytes(input), Ctx, out var reflected)); + Assert.Equal(date, reflected.Date); + Assert.Equal(dateOnly, reflected.DateOnlyVal); + Assert.Equal(input, WriteOne(new FixedWidthWriter(), reflected)); - var wChar = new FixedWidthWriter(); - using (var ms = new MemoryStream()) - using (var sw = new StreamWriter(ms, new UTF8Encoding(false), leaveOpen: true)) - { - wChar.Write(sw, in model1, Inv); - sw.Flush(); - Assert.Equal(input + Environment.NewLine, Encoding.UTF8.GetString(ms.ToArray())); - } - - // 2. Reflection Utf8 - var pUtf8 = new Utf8FixedWidthParser(); - var bytes = "2026071620260716"u8; - Assert.True(pUtf8.TryParse(bytes, Inv, null, out var model2)); - Assert.Equal(date, model2.Date); - Assert.Equal(dateOnly, model2.DateOnlyVal); - - // 3. Generated Char - Assert.True(FixedWidth.TryParse(input, Inv, null, out var model3)); - Assert.Equal(date, model3.Date); - Assert.Equal(dateOnly, model3.DateOnlyVal); - - Span charDest = stackalloc char[16]; - Assert.True(FixedWidth.TryFormat(model3, charDest, Inv, out int cw)); - Assert.Equal(16, cw); - Assert.Equal(input, charDest.ToString()); - - // 4. Generated Utf8 - Assert.True(FixedWidthUtf8.TryParse(bytes, Inv, null, out var model4)); - Assert.Equal(date, model4.Date); - Assert.Equal(dateOnly, model4.DateOnlyVal); + Assert.True(FixedWidth.TryParse(Bytes(input), Ctx, out var generated)); + Assert.Equal(date, generated.Date); + Assert.Equal(dateOnly, generated.DateOnlyVal); + Assert.Equal(input, Format(generated)); } [Fact] public void ZeroPadded_Leading_TrimMode() { - const string input = "0004203.14"; - var bytes = "0004203.14"u8; - - // 1. Reflection Char - var pChar = new FixedWidthParser(); - Assert.True(pChar.TryParse(input, Inv, null, out var model1)); - Assert.Equal(42, model1.Value); - Assert.Equal(3.14, model1.DoubleValue); - - // 2. Reflection Utf8 - var pUtf8 = new Utf8FixedWidthParser(); - Assert.True(pUtf8.TryParse(bytes, Inv, null, out var model2)); - Assert.Equal(42, model2.Value); - Assert.Equal(3.14, model2.DoubleValue); - - // 3. Generated Char - Assert.True(FixedWidth.TryParse(input, Inv, null, out var model3)); - Assert.Equal(42, model3.Value); - Assert.Equal(3.14, model3.DoubleValue); - - // 4. Generated Utf8 - Assert.True(FixedWidthUtf8.TryParse(bytes, Inv, null, out var model4)); - Assert.Equal(42, model4.Value); - Assert.Equal(3.14, model4.DoubleValue); + var parser = new FixedWidthParser(); + Assert.True(parser.TryParse("0004203.14"u8, Ctx, out var reflected)); + Assert.Equal(42, reflected.Value); + Assert.Equal(3.14, reflected.DoubleValue); + + Assert.True(FixedWidth.TryParse("0004203.14"u8, Ctx, out var generated)); + Assert.Equal(42, generated.Value); + Assert.Equal(3.14, generated.DoubleValue); } [Fact] public void ZeroPadded_Both_TrimMode() { - const string input = "00420"; - // 1. Reflection Char - var pChar = new FixedWidthParser(); - Assert.True(pChar.TryParse(input, Inv, null, out var model1)); - Assert.Equal(42, model1.Value); + Assert.True(new FixedWidthParser().TryParse("00420"u8, Ctx, out var reflected)); + Assert.Equal(42, reflected.Value); - // 2. Generated Char - Assert.True(FixedWidth.TryParse(input, Inv, null, out var model3)); - Assert.Equal(42, model3.Value); + Assert.True(FixedWidth.TryParse("00420"u8, Ctx, out var generated)); + Assert.Equal(42, generated.Value); } [Fact] public void Nullable_BlankDetection_WithCustomTrimChar() { - // Nullable model with custom TrimChar: should detect spaces or custom TrimChar as null - const string inputSpace = " "; - const string inputZeroAsterisk = "00000*****"; - - // 1. Reflection Char - var pChar = new FixedWidthParser(); - Assert.True(pChar.TryParse(inputSpace, Inv, null, out var model1)); - Assert.Null(model1.Value); - Assert.Null(model1.DoubleValue); - - Assert.True(pChar.TryParse(inputZeroAsterisk, Inv, null, out var model2)); - Assert.Null(model2.Value); - Assert.Null(model2.DoubleValue); - - var wChar = new FixedWidthWriter(); - using (var ms = new MemoryStream()) - using (var sw = new StreamWriter(ms, new UTF8Encoding(false), leaveOpen: true)) - { - wChar.Write(sw, in model2, Inv); - sw.Flush(); - Assert.Equal(inputZeroAsterisk + Environment.NewLine, Encoding.UTF8.GetString(ms.ToArray())); - } - - // 2. Generated Char - Assert.True(FixedWidth.TryParse(inputSpace, Inv, null, out var model3)); - Assert.Null(model3.Value); - Assert.Null(model3.DoubleValue); - - Assert.True(FixedWidth.TryParse(inputZeroAsterisk, Inv, null, out var model4)); - Assert.Null(model4.Value); - Assert.Null(model4.DoubleValue); - - Span charDest = stackalloc char[10]; - Assert.True(FixedWidth.TryFormat(model4, charDest, Inv, out int cw)); - Assert.Equal(10, cw); - Assert.Equal(inputZeroAsterisk, charDest.ToString()); + const string spaces = " "; + const string padded = "00000*****"; + + var parser = new FixedWidthParser(); + Assert.True(parser.TryParse(Bytes(spaces), Ctx, out var fromSpaces)); + Assert.Null(fromSpaces.Value); + Assert.Null(fromSpaces.DoubleValue); + Assert.True(parser.TryParse(Bytes(padded), Ctx, out var fromPadding)); + Assert.Null(fromPadding.Value); + Assert.Null(fromPadding.DoubleValue); + Assert.Equal(padded, WriteOne(new FixedWidthWriter(), fromPadding)); + + Assert.True(FixedWidth.TryParse(Bytes(spaces), Ctx, out var generatedSpaces)); + Assert.Null(generatedSpaces.Value); + Assert.Null(generatedSpaces.DoubleValue); + Assert.True(FixedWidth.TryParse(Bytes(padded), Ctx, out var generatedPadding)); + Assert.Null(generatedPadding.Value); + Assert.Equal(padded, Format(generatedPadding)); } [Fact] public void SignAware_ZeroPadding() { - var modelNegative = new SignPaddedModel { Value = -42 }; - var modelPositive = new SignPaddedModel { Value = 42 }; - var writer = new FixedWidthWriter(); - using (var ms = new MemoryStream()) - using (var sw = new StreamWriter(ms, new UTF8Encoding(false), leaveOpen: true)) - { - writer.Write(sw, in modelNegative, Inv); - sw.Flush(); - Assert.Equal("-00042" + Environment.NewLine, Encoding.UTF8.GetString(ms.ToArray())); - } - using (var ms = new MemoryStream()) - using (var sw = new StreamWriter(ms, new UTF8Encoding(false), leaveOpen: true)) - { - writer.Write(sw, in modelPositive, Inv); - sw.Flush(); - Assert.Equal("000042" + Environment.NewLine, Encoding.UTF8.GetString(ms.ToArray())); // Default int doesn't format with +, so just zero pad - } + Assert.Equal("-00042", WriteOne(writer, new SignPaddedModel { Value = -42 })); + Assert.Equal("000042", WriteOne(writer, new SignPaddedModel { Value = 42 })); } [Fact] diff --git a/tests/FixedWidthParser.Tests/CoverageTests.cs b/tests/FixedWidthParser.Tests/CoverageTests.cs index 6b26749..82b93a7 100644 --- a/tests/FixedWidthParser.Tests/CoverageTests.cs +++ b/tests/FixedWidthParser.Tests/CoverageTests.cs @@ -1,6 +1,5 @@ using System.Globalization; using System.Text; -using FixedWidthParser; using FixedWidthParser.Attributes; using FixedWidthParser.Formatters; using FixedWidthParser.Parsers; @@ -9,111 +8,71 @@ namespace FixedWidthParser.Tests { /// - /// Fills coverage gaps left by the type-focused suites: the TimeOnly/DateTimeOffset exact-parser - /// branches (reflection factories and generated runtime helpers), the UTF-8 culture parser's - /// wide-field ArrayPool transcode branch, and the converter formatter's ArrayPool growth/exhaustion. + /// Branches the type-focused suites leave out: the TimeOnly/DateTimeOffset exact parsers, the + /// decode fallback for wide numeric fields, and the converter formatter's ArrayPool growth/exhaustion. /// public class CoverageTests { - private static readonly CultureInfo DeDe = CultureInfo.GetCultureInfo("de-DE"); - - // ----- TimeOnly + DateTimeOffset exact parsing (both factories + both generated runtimes) ----- - [Fact] - public void TimeExact_RoundTrip_AllPaths() + public void TimeExact_RoundTrip_BothPaths() { var time = new TimeOnly(13, 45, 30); var timestamp = new DateTimeOffset(2026, 7, 16, 13, 45, 30, TimeSpan.FromHours(3)); - const string input = "13453020260716134530+03:00"; - var bytes = Encoding.UTF8.GetBytes(input); - - // Reflection char (ColumnParserFactory.BuildExact: TimeOnly + DateTimeOffset branches) - var pChar = new FixedWidthParser(); - Assert.True(pChar.TryParse(input, Inv, null, out var m1)); - Assert.Equal(time, m1.Time); - Assert.Equal(timestamp, m1.Timestamp); - - // Reflection UTF-8 (Utf8ColumnParserFactory.BuildExact) - var pUtf8 = new Utf8FixedWidthParser(); - Assert.True(pUtf8.TryParse(bytes, Inv, null, out var m2)); - Assert.Equal(time, m2.Time); - Assert.Equal(timestamp, m2.Timestamp); + byte[] input = Bytes("13453020260716134530+03:00"); - // Generated char (FixedWidthRuntime.TryTimeOnlyExact / TryDateTimeOffsetExact) - Assert.True(FixedWidth.TryParse(input, Inv, null, out var m3)); - Assert.Equal(time, m3.Time); - Assert.Equal(timestamp, m3.Timestamp); + Assert.True(new FixedWidthParser().TryParse(input, Ctx, out var reflected)); + Assert.Equal(time, reflected.Time); + Assert.Equal(timestamp, reflected.Timestamp); - // Generated UTF-8 (Utf8FixedWidthRuntime.TryTimeOnlyExact / TryDateTimeOffsetExact) - Assert.True(FixedWidthUtf8.TryParse(bytes, Inv, null, out var m4)); - Assert.Equal(time, m4.Time); - Assert.Equal(timestamp, m4.Timestamp); + Assert.True(FixedWidth.TryParse(input, Ctx, out var generated)); + Assert.Equal(time, generated.Time); + Assert.Equal(timestamp, generated.Timestamp); } [Fact] public void TimeExact_InvalidField_ReturnsFalse() { - // A non-time first column makes the exact parser return false (the failure branch in both factories). - const string bad = "xxxxxx20260716134530+03:00"; - var badBytes = Encoding.UTF8.GetBytes(bad); + byte[] bad = Bytes("xxxxxx20260716134530+03:00"); - Assert.False(new FixedWidthParser().TryParse(bad, Inv, null, out _)); - Assert.False(new Utf8FixedWidthParser().TryParse(badBytes, Inv, null, out _)); + Assert.False(new FixedWidthParser().TryParse(bad, Ctx, out _)); + Assert.False(FixedWidth.TryParse(bad, Ctx, out _)); } [Fact] - public void Utf8_ConverterRejectsField_ReturnsFalse() + public void WideNumericField_CommaCulture_DecodesThroughArrayPool() { - // A non-numeric field makes CentsConverter.TryParse (byte) fail — the converter-failure - // branch in Utf8ColumnParserFactory.BuildConverter. - Assert.False(new Utf8FixedWidthParser().TryParse("notanum!"u8, Inv, null, out _)); - } - - // ----- UTF-8 culture parser: wide field forces the ArrayPool transcode branch ----- - - [Fact] - public void Utf8_WideNumericField_CommaCulture_UsesTranscodeArrayPoolBranch() - { - // Fields longer than the 128-char stack buffer, and a non-'.' separator so the parser takes - // the transcode path (TryParseDoubleViaTranscode / TryParseFloatViaTranscode) — its long branch. + // Wider than the 128-char stack buffer, with a ',' separator so the BCL fallback runs. string field = ("0," + new string('1', 150)).PadRight(200); - var bytes = Encoding.UTF8.GetBytes(field + field); - bool ok = new Utf8FixedWidthParser().TryParse(bytes, DeDe, null, out var m); + bool ok = new FixedWidthParser().TryParse(Bytes(field + field), CtxWith(CultureInfo.GetCultureInfo("de-DE")), out var m); Assert.True(ok); Assert.InRange(m.D, 0.11, 0.12); Assert.InRange(m.F, 0.11f, 0.12f); } - // ----- Converter formatter: ArrayPool growth loop and exhaustion ----- - [Fact] public void FormatConvert_OutputLargerThanStackBuffer_UsesArrayPoolGrowth() { var options = new ColumnFormatOptions(Alignment.Left, ' ', null, OverflowBehavior.Truncate); - Span slice = stackalloc char[200]; + byte[] slice = new byte[400]; - // 100 chars > the 64-char stack buffer → the ArrayPool growth loop writes it instead. - FixedWidthRuntime.FormatConvert(new BigValue(100), slice, Inv, new BigConverter(), options, "Col"); + // 300 bytes > the 256-byte stack buffer, so the ArrayPool growth loop writes it instead. + FixedWidthRuntime.FormatConvert(new BigValue(300), slice, Ctx, new BigConverter(), options, "Col"); - Assert.Equal(new string('x', 100) + new string(' ', 100), slice.ToString()); + Assert.Equal(new string('x', 300) + new string(' ', 100), Encoding.ASCII.GetString(slice)); } [Fact] public void FormatConvert_OutputNeverFits_Throws() - { - var ex = Record.Exception(FormatConvertThatNeverFits); - var iop = Assert.IsType(ex); - Assert.Contains("Col", iop.Message, StringComparison.Ordinal); - } - - // Separate method: a Span can't be captured by the Record.Exception lambda. - private static void FormatConvertThatNeverFits() { var options = new ColumnFormatOptions(Alignment.Left, ' ', null, OverflowBehavior.Truncate); - Span slice = stackalloc char[8]; // maxSize = max(1024, 8*16) = 1024 - FixedWidthRuntime.FormatConvert(new BigValue(2000), slice, Inv, new BigConverter(), options, "Col"); + byte[] slice = new byte[8]; + + var ex = Assert.Throws( + () => FixedWidthRuntime.FormatConvert(new BigValue(10_000), slice, Ctx, new BigConverter(), options, "Col")); + + Assert.Contains("Col", ex.Message, StringComparison.Ordinal); } } } diff --git a/tests/FixedWidthParser.Tests/CultureTests.cs b/tests/FixedWidthParser.Tests/CultureTests.cs index 35bf73b..f6feded 100644 --- a/tests/FixedWidthParser.Tests/CultureTests.cs +++ b/tests/FixedWidthParser.Tests/CultureTests.cs @@ -6,232 +6,121 @@ namespace FixedWidthParser.Tests { /// - /// Covers culture handling. The generic path (ISpanParsable, e.g. decimal) and writing via - /// ISpanFormattable honor the IFormatProvider. The double/float processors derive the decimal - /// separator from the culture and pass it to csFastFloat, so they also honor the - /// IFormatProvider (with '.' as the default when null). + /// Culture handling. The format provider comes from ; the default + /// context means the invariant culture for every column type, on parse and on write. /// public class CultureTests { private static readonly CultureInfo DeDe = CultureInfo.GetCultureInfo("de-DE"); - - // ----- Generic path (decimal): honors the culture ----- + private static readonly FixedWidthContext German = CtxWith(DeDe); [Fact] public void Parse_DecimalColumn_HonorsCommaCulture() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("1234,56 ", DeDe, null, out var model); - - Assert.True(ok); + Assert.True(new FixedWidthParser().TryParse("1234,56 "u8, German, out var model)); Assert.Equal(1234.56m, model.Amount); } [Fact] - public void Parse_DecimalColumn_HonorsDotCulture() + public void Parse_DecimalColumn_DefaultContext_IsInvariant() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("1234.56 ", CultureInfo.InvariantCulture, null, out var model); - - Assert.True(ok); + Assert.True(new FixedWidthParser().TryParse("1234.56 "u8, Ctx, out var model)); Assert.Equal(1234.56m, model.Amount); } - // ----- Double path: decimal separator derived from the culture ----- - [Fact] public void Parse_DoubleColumn_HonorsCommaCulture() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 1234,50 ", DeDe, null, out var model); - - Assert.True(ok); + Assert.True(new FixedWidthParser().TryParse("Bob 30 1234,50 "u8, German, out var model)); Assert.Equal(1234.50, model.Salary, 2); } [Fact] - public void Parse_DoubleColumn_HonorsDotCulture() + public void Parse_DoubleColumn_DefaultContext_UsesDot() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 1234.50 ", CultureInfo.InvariantCulture, null, out var model); - - Assert.True(ok); - Assert.Equal(1234.50, model.Salary, 2); - } - - [Fact] - public void Parse_DoubleColumn_NullProvider_DefaultsToDot() - { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 1234.50 ", null, null, out var model); - - Assert.True(ok); + Assert.True(new FixedWidthParser().TryParse("Bob 30 1234.50 "u8, Ctx, out var model)); Assert.Equal(1234.50, model.Salary, 2); } [Fact] public void Parse_FloatColumn_HonorsCommaCulture() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("3,14 ", DeDe, null, out var model); - - Assert.True(ok); + Assert.True(new FixedWidthParser().TryParse("3,14 "u8, German, out var model)); Assert.Equal(3.14f, model.Value, 2); } - // ----- Null provider means invariant for EVERY column type, not just double/float: before this - // fix, ISpanParsable columns (decimal, int, DateTime, …) fell through to the BCL's own default - // of CurrentCulture on a null provider, while double/float were hard-coded to invariant — so the - // same line could parse two different ways for two different column types under one null-provider - // call. Pinning both to invariant also makes this test's outcome independent of whatever culture - // happens to be ambient when the suite runs. ----- - - [Fact] - public void Parse_DecimalColumn_NullProvider_DefaultsToInvariant() - { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("1234.56 ", null, null, out var model); - - Assert.True(ok); - Assert.Equal(1234.56m, model.Amount); - } - - // ----- Round-trip under the default (null) provider must preserve the value regardless of the - // machine's ambient culture — before this fix, write (CurrentCulture via ISpanFormattable.TryFormat) - // and parse (invariant via CultureHelpers) disagreed on what null means, so a comma-decimal machine - // would write "1234,5" and then fail to parse its own output back with the same null provider. ----- - - [Fact] - public void RoundTrip_DoubleColumn_NullProvider_PreservesValue() - { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); - var original = new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }; - - string line = WriteOne(writer, original); - bool ok = parser.TryParse(line, null, null, out var parsed); - - Assert.True(ok); - Assert.Equal(original.Salary, parsed.Salary, 2); - } - - // ----- Thousands separators under the invariant '.' fast path: csFastFloat's single - // decimal_separator override can't express a thousands separator, so it used to fail the - // full-consumption check and reject the whole line even though the value is valid; the BCL - // fallback now retried on that failure accepts it (and still rejects real garbage, see - // Parse_DoubleColumn_TrailingGarbageAfterNumber_Fails above). ----- - + // csFastFloat's single decimal_separator override can't express a thousands separator; the + // BCL fallback accepts the value and still rejects real garbage. [Fact] public void Parse_DoubleColumn_DotCultureThousandsSeparator_ParsesFullValue() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 1,234.50 ", Inv, null, out var model); - - Assert.True(ok); + Assert.True(new FixedWidthParser().TryParse("Bob 30 1,234.50 "u8, Ctx, out var model)); Assert.Equal(1234.50, model.Salary, 2); } - // ----- Thousands separators: csFastFloat's decimal_separator override silently truncates at - // the first character it doesn't recognize (e.g. a thousands separator) instead of failing - // only real NumberFormatInfo-aware parsing (used whenever the separator isn't '.') gets this - // right. Regression coverage for a bug where "1.234,50" under de-DE silently became 1.0. ----- - + // Regression: "1.234,50" under de-DE used to come back as 1.0 from csFastFloat. [Fact] public void Parse_DoubleColumn_DeDeThousandsSeparator_ParsesFullValue() { - var parser = new FixedWidthParser(); - - // de-DE: '.' groups thousands, ',' is the decimal separator — "1.234,50" means 1234.50. - bool ok = parser.TryParse("Bob 30 1.234,50 ", DeDe, null, out var model); - - Assert.True(ok); + Assert.True(new FixedWidthParser().TryParse("Bob 30 1.234,50 "u8, German, out var model)); Assert.Equal(1234.50, model.Salary, 2); } - // ----- Trailing garbage after a dot-separated number: the fast path must reject it instead of - // silently truncating to the leading digits and reporting success. ----- - [Fact] public void Parse_DoubleColumn_TrailingGarbageAfterNumber_Fails() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 12x ", Inv, null, out _); - - Assert.False(ok); + Assert.False(new FixedWidthParser().TryParse("Bob 30 12x "u8, Ctx, out _)); } [Fact] public void Parse_FloatColumn_TrailingGarbageAfterNumber_Fails() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("12x ", Inv, null, out _); - - Assert.False(ok); + Assert.False(new FixedWidthParser().TryParse("12x "u8, Ctx, out _)); } - // ----- Right-aligned (leading-space-padded) numeric columns: csFastFloat's characters_consumed - // does not consistently include skipped leading whitespace across its double/float overloads - // (FastDoubleParser counted it, FastFloatParser didn't, and vice versa on the UTF-8 overloads), - // so a right-aligned column could fail the full-consumption check even under the invariant '.' - // fast path — the exact opposite of a version that should always accept it. ----- - + // csFastFloat's characters_consumed does not consistently include skipped leading whitespace, + // so right-aligned columns must be trimmed before the fast path. [Fact] public void Parse_DoubleColumn_RightAligned_DotCulture_Parses() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 123.45", Inv, null, out var model); - - Assert.True(ok); + Assert.True(new FixedWidthParser().TryParse("Bob 30 123.45"u8, Ctx, out var model)); Assert.Equal(123.45, model.Salary, 2); } [Fact] public void Parse_FloatColumn_RightAligned_DotCulture_Parses() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse(" 3.14", Inv, null, out var model); - - Assert.True(ok); + Assert.True(new FixedWidthParser().TryParse(" 3.14"u8, Ctx, out var model)); Assert.Equal(3.14f, model.Value, 2); } [Fact] public void Parse_DoubleColumn_RightAligned_CommaCulture_Parses() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 123,45", DeDe, null, out var model); - - Assert.True(ok); + Assert.True(new FixedWidthParser().TryParse("Bob 30 123,45"u8, German, out var model)); Assert.Equal(123.45, model.Salary, 2); } [Fact] public void Parse_FloatColumn_RightAligned_CommaCulture_Parses() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse(" 3,14", DeDe, null, out var model); - - Assert.True(ok); + Assert.True(new FixedWidthParser().TryParse(" 3,14"u8, German, out var model)); Assert.Equal(3.14f, model.Value, 2); } - // ----- Regression guard for the old single-entry decimal-separator memo: alternating between - // two providers must not thrash into wrong results for either one. ----- + // A separator that is not one byte (U+066B in UTF-8) used to be refused; the value is now + // decoded and parsed by the BCL with the real culture. + [Fact] + public void Parse_DoubleColumn_NonAsciiSeparatorCulture_Parses() + { + var culture = (CultureInfo)CultureInfo.InvariantCulture.Clone(); + culture.NumberFormat.NumberDecimalSeparator = "٫"; + + Assert.True(new FixedWidthParser().TryParse(Bytes("Bob 30 1234٫50 "), CtxWith(culture), out var model)); + Assert.Equal(1234.50, model.Salary, 2); + } + // Regression guard for the old single-entry decimal-separator memo. [Fact] public void Parse_DoubleColumn_AlternatingProviders_BothStayCorrect() { @@ -239,41 +128,44 @@ public void Parse_DoubleColumn_AlternatingProviders_BothStayCorrect() for (int i = 0; i < 5; i++) { - Assert.True(parser.TryParse("Bob 30 1234.50 ", Inv, null, out var invModel)); - Assert.Equal(1234.50, invModel.Salary, 2); + Assert.True(parser.TryParse("Bob 30 1234.50 "u8, Ctx, out var invariant)); + Assert.Equal(1234.50, invariant.Salary, 2); - Assert.True(parser.TryParse("Bob 30 1234,50 ", DeDe, null, out var deModel)); - Assert.Equal(1234.50, deModel.Salary, 2); + Assert.True(parser.TryParse("Bob 30 1234,50 "u8, German, out var german)); + Assert.Equal(1234.50, german.Salary, 2); } } - // ----- Writing: honors the culture (ISpanFormattable) ----- - [Fact] public void Write_DoubleColumn_UsesCultureDecimalSeparator() { - var writer = new FixedWidthWriter(); + var writer = new FixedWidthWriter(new FixedWidthWriterOptions { FormatProvider = DeDe }); - string line = WriteOne(writer, new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }, DeDe); + string line = WriteOne(writer, new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }); - // Salary column at [15,25): formatted with a comma under de-DE. Assert.Equal("1234,5", line[15..21]); } - // ----- Round-trip under a comma culture now preserves the value ----- + [Fact] + public void RoundTrip_DoubleColumn_DefaultOptions_PreservesValue() + { + var original = new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }; + + string line = WriteOne(new FixedWidthWriter(), original); + + Assert.True(new FixedWidthParser().TryParse(Bytes(line), Ctx, out var parsed)); + Assert.Equal(original.Salary, parsed.Salary, 2); + } [Fact] public void RoundTrip_DoubleColumn_UnderCommaCulture_PreservesValue() { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); + var writer = new FixedWidthWriter(new FixedWidthWriterOptions { FormatProvider = DeDe }); var original = new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }; - // Writing produces "1234,5"; reading uses the same culture separator → same value. - string line = WriteOne(writer, original, DeDe); - bool ok = parser.TryParse(line, DeDe, null, out var parsed); + string line = WriteOne(writer, original); - Assert.True(ok); + Assert.True(new FixedWidthParser().TryParse(Bytes(line), German, out var parsed)); Assert.Equal(original.Salary, parsed.Salary, 2); } } diff --git a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj index 653b783..e77ad86 100644 --- a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj +++ b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj @@ -1,9 +1,9 @@ - + true true - net8.0;net10.0 + net10.0 latest enable enable @@ -20,7 +20,6 @@ - diff --git a/tests/FixedWidthParser.Tests/GeneratedModels.cs b/tests/FixedWidthParser.Tests/GeneratedModels.cs index 080f67a..e843452 100644 --- a/tests/FixedWidthParser.Tests/GeneratedModels.cs +++ b/tests/FixedWidthParser.Tests/GeneratedModels.cs @@ -2,14 +2,12 @@ namespace FixedWidthParser.Tests { - // Models that opt into the source-generated parser. Each declares BOTH IFixedWidthModel - // (char) and IUtf8FixedWidthModel (byte), so the generator emits both TryParse overloads - // into the same partial — exercising marker coexistence. Each mirrors a reflection-based - // counterpart in TestModels.cs so the parity tests can compare the generated TryParse against the - // runtime FixedWidthParser / Utf8FixedWidthParser over the same lines. + // Models that opt into the source-generated parser and writer. Each mirrors a reflection-based + // counterpart in TestModels.cs so the parity tests can compare the generated TryParse/TryFormat + // against the runtime FixedWidthParser/FixedWidthWriter over the same lines. /// Property-based model (string + int + double) — mirrors . - public readonly partial record struct GenPersonModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenPersonModel : IFixedWidthModel { [FixedColumn(0, 10)] public string Name { get; init; } [FixedColumn(10, 5)] public int Age { get; init; } @@ -17,141 +15,145 @@ namespace FixedWidthParser.Tests } /// Public-field model (exercises the field path) — mirrors . - public partial struct GenProductModel : IFixedWidthModel, IUtf8FixedWidthModel + public partial struct GenProductModel : IFixedWidthModel { [FixedColumn(0, 5)] public string Code; [FixedColumn(5, 4)] public int Quantity; } /// Float column — mirrors . - public readonly partial record struct GenMeasurementModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenMeasurementModel : IFixedWidthModel { [FixedColumn(0, 8)] public float Value { get; init; } } /// Trailing string column — mirrors . - public readonly partial record struct GenTrailingStringModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenTrailingStringModel : IFixedWidthModel { [FixedColumn(0, 5)] public int Id { get; init; } [FixedColumn(5, 10)] public string Note { get; init; } } - /// Decimal column (ISpanParsable / IUtf8SpanParsable path) — mirrors . - public readonly partial record struct GenDecimalModel : IFixedWidthModel, IUtf8FixedWidthModel + /// Decimal column (IUtf8SpanParsable path) — mirrors . + public readonly partial record struct GenDecimalModel : IFixedWidthModel { [FixedColumn(0, 12)] public decimal Amount { get; init; } } /// Single-column generated model — mirrors for reader tests. - public readonly partial record struct GenCodeModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenCodeModel : IFixedWidthModel { [FixedColumn(0, 3)] public string Code { get; init; } } /// Custom-converter column () — mirrors . - public readonly partial record struct GenCentsConverterModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenCentsConverterModel : IFixedWidthModel { [FixedColumn(0, 8, Converter = typeof(CentsConverter))] public CentsValue Amount { get; init; } } /// Nullable value-type columns (T?) — mirrors . - public readonly partial record struct GenNullableModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenNullableModel : IFixedWidthModel { [FixedColumn(0, 5)] public int? Age { get; init; } [FixedColumn(5, 10)] public decimal? Amount { get; init; } } /// Nullable value-type column + converter — mirrors . - public readonly partial record struct GenNullableConverterModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenNullableConverterModel : IFixedWidthModel { [FixedColumn(0, 8, Converter = typeof(CentsConverter))] public CentsValue? Amount { get; init; } } - /// Right-aligned integer — mirrors (write formatting options). - public readonly partial record struct GenRightAlignedModel : IFixedWidthModel, IUtf8FixedWidthModel + /// Right-aligned integer — mirrors . + public readonly partial record struct GenRightAlignedModel : IFixedWidthModel { [FixedColumn(0, 5, Alignment = Alignment.Right)] public int Value { get; init; } } /// Zero-padded right-aligned integer — mirrors . - public readonly partial record struct GenZeroPaddedModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenZeroPaddedModel : IFixedWidthModel { [FixedColumn(0, 5, Alignment = Alignment.Right, Padding = '0')] public int Value { get; init; } } /// Double with a fixed format string — mirrors . - public readonly partial record struct GenFormattedModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenFormattedModel : IFixedWidthModel { [FixedColumn(0, 8, Format = "F2")] public double Amount { get; init; } } /// Narrow numeric column; default overflow (throws) — mirrors . - public readonly partial record struct GenNarrowModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenNarrowModel : IFixedWidthModel { [FixedColumn(0, 3)] public int Value { get; init; } } /// '*'-padded integer, read with a matching TrimChar — mirrors . - public readonly partial record struct GenAsteriskTrimIntModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenAsteriskTrimIntModel : IFixedWidthModel { [FixedColumn(0, 5, TrimChar = '*')] public int Value { get; init; } } /// '#'-padded string, read with a matching TrimChar — mirrors . - public readonly partial record struct GenHashTrimStringModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenHashTrimStringModel : IFixedWidthModel { [FixedColumn(0, 8, TrimChar = '#')] public string Code { get; init; } } - /// Non-ASCII TrimChar (U+00A0) — mirrors ; UTF-8 must throw. - public readonly partial record struct GenNonAsciiTrimModel : IFixedWidthModel, IUtf8FixedWidthModel - { - [FixedColumn(0, 5, TrimChar = ' ')] public int Value { get; init; } - } - -#if NET9_0_OR_GREATER /// Ref struct model — proves generated parsing works under `allows ref struct`. Mirrors . - public ref partial struct GenRefPersonModel : IFixedWidthModel, IUtf8FixedWidthModel + public ref partial struct GenRefPersonModel : IFixedWidthModel { [FixedColumn(0, 10)] public string Name { get; set; } [FixedColumn(10, 5)] public int Age { get; set; } [FixedColumn(15, 10)] public double Salary { get; set; } } -#endif - public readonly partial record struct GenDateTimeExactModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenDateTimeExactModel : IFixedWidthModel { [FixedColumn(0, 8, Format = "yyyyMMdd")] public DateTime Date { get; init; } [FixedColumn(8, 8, Format = "yyyyMMdd")] public DateOnly DateOnlyVal { get; init; } } - /// TimeOnly + DateTimeOffset exact columns — mirrors ; drives - /// the generated code's TryTimeOnlyExact / TryDateTimeOffsetExact runtime helpers (char and UTF-8). - public readonly partial record struct GenTimeExactModel : IFixedWidthModel, IUtf8FixedWidthModel + /// TimeOnly + DateTimeOffset exact columns — mirrors . + public readonly partial record struct GenTimeExactModel : IFixedWidthModel { [FixedColumn(0, 6, Format = "HHmmss")] public TimeOnly Time { get; init; } [FixedColumn(6, 20, Format = "yyyyMMddHHmmsszzz")] public DateTimeOffset Timestamp { get; init; } } - public readonly partial record struct GenZeroPaddedLeadingModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenZeroPaddedLeadingModel : IFixedWidthModel { [FixedColumn(0, 5, TrimChar = '0', TrimMode = TrimMode.Leading)] public int Value { get; init; } [FixedColumn(5, 5, TrimChar = '0', TrimMode = TrimMode.Leading)] public double DoubleValue { get; init; } } - public readonly partial record struct GenZeroPaddedBothModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenZeroPaddedBothModel : IFixedWidthModel { [FixedColumn(0, 5, TrimChar = '0', TrimMode = TrimMode.Both)] public int Value { get; init; } } - public readonly partial record struct GenSignPaddedModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenSignPaddedModel : IFixedWidthModel { [FixedColumn(0, 6, Alignment = Alignment.Right, Padding = '0')] public int Value { get; init; } } - public readonly partial record struct GenNullablePaddingModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenNullablePaddingModel : IFixedWidthModel { [FixedColumn(0, 5, TrimChar = '0')] public int? Value { get; init; } [FixedColumn(5, 5, TrimChar = '*')] public double? DoubleValue { get; init; } } + + /// Character-only parsable column — mirrors . + public readonly partial record struct GenShoutModel : IFixedWidthModel + { + [FixedColumn(0, 6)] public Shout Word { get; init; } + } + + /// Latin-1 name and number — mirrors . + public readonly partial record struct GenNameAgeModel : IFixedWidthModel + { + [FixedColumn(0, 6)] public string Name { get; init; } + [FixedColumn(6, 3)] public int Age { get; init; } + } } diff --git a/tests/FixedWidthParser.Tests/GeneratedReaderTests.cs b/tests/FixedWidthParser.Tests/GeneratedReaderTests.cs index ff012bf..7ae9549 100644 --- a/tests/FixedWidthParser.Tests/GeneratedReaderTests.cs +++ b/tests/FixedWidthParser.Tests/GeneratedReaderTests.cs @@ -5,223 +5,136 @@ namespace FixedWidthParser.Tests { + /// + /// The read methods for generated models, sync and async, and their + /// parity with the reflection . + /// public class GeneratedReaderTests { private const string TwoPeople = "John Doe 30 60000.00 \n" + "Jane 28 55000.00 "; - private static MemoryStream Utf8(string text) - { - return new(Encoding.UTF8.GetBytes(text)); - } - - // ----------------------------- Stream overloads – sync ----------------------------- - [Fact] public void Read_Stream_ParsesAll() { - using var stream = Utf8(TwoPeople); - - var people = FixedWidth.Read(stream, formatProvider: Inv).ToList(); + var people = FixedWidth.Read(StreamOf(TwoPeople)).ToList(); Assert.Equal(2, people.Count); Assert.Equal("John Doe", people[0].Name); + Assert.Equal(30, people[0].Age); Assert.Equal("Jane", people[1].Name); Assert.Equal(55000.00, people[1].Salary, 2); } [Fact] - public void Read_Stream_CustomEncoding_IsHonored() + public void Read_StructEnumerator_IteratesManually() { - using var stream = new MemoryStream(Encoding.Latin1.GetBytes("Áb \nCDé\n")); - - var codes = FixedWidth.Read(stream, encoding: Encoding.Latin1) - .Select(m => m.Code) - .ToList(); - - Assert.Equal(["Áb", "CDé"], codes); - } - - [Fact] - public void Read_Stream_LeaveOpenTrue_KeepsStreamOpen() - { - using var stream = Utf8("ABC\nDEF\n"); - - _ = FixedWidth.Read(stream, leaveOpen: true).ToList(); + using var enumerator = FixedWidth.Read(StreamOf(TwoPeople)).GetEnumerator(); + var names = new List(); + while (enumerator.MoveNext()) + { + names.Add(enumerator.Current.Name); + } - Assert.True(stream.CanRead); + Assert.Equal(["John Doe", "Jane"], names); } - [Fact] - public void Read_Stream_LeaveOpenFalse_DisposesStream() + [Theory] + [InlineData("ABC\nDEF\n")] + [InlineData("ABC\r\nDEF\r\n")] + [InlineData("ABC\nDEF")] + public void Read_HandlesLineEndingsAndTrailingNewline(string text) { - var stream = Utf8("ABC\nDEF\n"); + var codes = FixedWidth.Read(StreamOf(text)).Select(m => m.Code).ToList(); - _ = FixedWidth.Read(stream).ToList(); // default leaveOpen: false - - Assert.False(stream.CanRead); + Assert.Equal(["ABC", "DEF"], codes); } [Fact] - public void Read_Stream_NullStream_Throws() + public void Read_SkipsEmptyLinesAndBom() { - Assert.Throws(() => FixedWidth.Read((Stream)null!)); - } + byte[] bytes = [0xEF, 0xBB, 0xBF, .. "ABC\n\n\nDEF\n"u8]; - // ----------------------------- Stream overloads – async ----------------------------- - - [Fact] - public async Task ReadAsync_Stream_ParsesAll() - { - await using var stream = Utf8(TwoPeople); + var codes = FixedWidth.Read(new MemoryStream(bytes)).Select(m => m.Code).ToList(); - var people = await FixedWidth.ReadAsync(stream, formatProvider: Inv).ToListAsync().ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); + Assert.Equal(["ABC", "DEF"], codes); } [Fact] - public async Task ReadAsync_Stream_CustomEncoding_IsHonored() + public void Read_Latin1_DecodesStringsAndKeepsBytePositions() { - await using var stream = new MemoryStream(Encoding.Latin1.GetBytes("Áb \nCDé\n")); + // 'Á' and 'é' are one byte each in Latin-1, so the 3-byte column holds three characters. + var options = new FixedWidthReaderOptions { Encoding = Encoding.Latin1 }; - var codes = await FixedWidth.ReadAsync(stream, encoding: Encoding.Latin1) + var codes = FixedWidth.Read(new MemoryStream(Encoding.Latin1.GetBytes("Áb \nCDé\n")), options: options) .Select(m => m.Code) - .ToListAsync().ConfigureAwait(true); + .ToList(); Assert.Equal(["Áb", "CDé"], codes); } [Fact] - public async Task ReadAsync_Stream_LeaveOpenTrue_KeepsStreamOpen() - { - await using var stream = Utf8("ABC\nDEF\n"); - - _ = await FixedWidth.ReadAsync(stream, leaveOpen: true).ToListAsync().ConfigureAwait(true); - - Assert.True(stream.CanRead); - } - - [Fact] - public async Task ReadAsync_Stream_LeaveOpenFalse_DisposesStream() - { - var stream = Utf8("ABC\nDEF\n"); - - _ = await FixedWidth.ReadAsync(stream).ToListAsync().ConfigureAwait(true); // default leaveOpen: false - - Assert.False(stream.CanRead); - } - - [Fact] - public void ReadAsync_Stream_NullStream_Throws() - { - Assert.Throws(() => FixedWidth.ReadAsync((Stream)null!)); - } - - // ----------------------------- TextReader overloads ----------------------------- - - [Fact] - public void Read_MultipleLines_MatchesReflectionReader() + public void Read_LineLongerThanBuffer_MatchesReflectionReader() { - var reflection = new FixedWidthReader(Inv); + var options = new FixedWidthReaderOptions { BufferSize = 4 }; - var expected = reflection.Read(new StringReader(TwoPeople)).ToList(); - var actual = FixedWidth.Read( - new StringReader(TwoPeople), - formatProvider: Inv).ToList(); + var expected = new FixedWidthReader(options).Read(StreamOf(TwoPeople)).ToList(); + var actual = FixedWidth.Read(StreamOf(TwoPeople), options: options).ToList(); - Assert.Equal(expected.Count, actual.Count); Assert.Equal(expected[0].Name, actual[0].Name); - Assert.Equal(expected[0].Age, actual[0].Age); - Assert.Equal(expected[0].Salary, actual[0].Salary, 2); Assert.Equal(expected[1].Name, actual[1].Name); - Assert.Equal(expected[1].Age, actual[1].Age); Assert.Equal(expected[1].Salary, actual[1].Salary, 2); } - [Theory] - [InlineData("ABC\nDEF\n")] - [InlineData("ABC\r\nDEF\r\n")] - [InlineData("ABC\nDEF")] - public void Read_HandlesLineEndingsAndTrailingNewline(string text) - { - var codes = FixedWidth.Read(new StringReader(text)) - .Select(m => m.Code) - .ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - [Fact] - public void Read_SkipsEmptyLines() + public void Read_InvalidLine_ThrowsSameErrorAsReflectionReader() { - var codes = FixedWidth.Read(new StringReader("ABC\n\n\nDEF\n")) - .Select(m => m.Code) - .ToList(); + const string text = "John Doe 30 60000.00 \nJane XX 55000.00 "; - Assert.Equal(["ABC", "DEF"], codes); + var expected = Assert.Throws(() => new FixedWidthReader().Read(StreamOf(text)).ToList()); + var actual = Assert.Throws(() => FixedWidth.Read(StreamOf(text)).ToList()); + + Assert.Equal(expected.LineNumber, actual.LineNumber); + Assert.Equal(expected.ColumnName, actual.ColumnName); + Assert.Equal(expected.RawValue, actual.RawValue); } [Fact] - public void Read_LineLongerThanBuffer_MatchesReflectionReader() + public void Read_LeaveOpen_ControlsDisposal() { - var reflection = new FixedWidthReader(Inv, bufferSize: 4); + using var kept = StreamOf("ABC\n"); + var owned = StreamOf("ABC\n"); - var expected = reflection.Read(new StringReader(TwoPeople)).ToList(); - var actual = FixedWidth.Read( - new StringReader(TwoPeople), - formatProvider: Inv, - bufferSize: 4).ToList(); + _ = FixedWidth.Read(kept, leaveOpen: true).ToList(); + _ = FixedWidth.Read(owned).ToList(); - Assert.Equal(expected[0].Name, actual[0].Name); - Assert.Equal(expected[1].Name, actual[1].Name); - Assert.Equal(expected[1].Salary, actual[1].Salary, 2); + Assert.True(kept.CanRead); + Assert.False(owned.CanRead); } [Fact] - public void Read_InvalidLine_ThrowsWithSameLineNumber() + public void Read_NullArguments_Throw() { - var reflection = new FixedWidthReader(Inv); - var text = "John Doe 30 60000.00 \nJane XX 55000.00 "; - - var expected = Assert.Throws( - () => reflection.Read(new StringReader(text)).ToList()); - var actual = Assert.Throws( - () => FixedWidth.Read(new StringReader(text), formatProvider: Inv).ToList()); - - Assert.Contains("Line 2", expected.Message, StringComparison.Ordinal); - Assert.Contains("Line 2", actual.Message, StringComparison.Ordinal); + Assert.Throws(() => FixedWidth.Read(null!)); + Assert.Throws(() => FixedWidth.ReadAsync((Stream)null!)); + Assert.Throws(() => FixedWidth.ReadFile("")); } [Fact] public void Read_WithStringPool_InternsRepeatedValues() { - var pool = new StringPool(); + var options = new FixedWidthReaderOptions { StringPool = new StringPool() }; - var codes = FixedWidth.Read( - new StringReader("ABC\nABC\n"), - stringPool: pool).ToList(); + var codes = FixedWidth.Read(StreamOf("ABC\nABC\n"), options: options).ToList(); - Assert.Equal("ABC", codes[0].Code); Assert.Same(codes[0].Code, codes[1].Code); } [Fact] public void GetEnumerator_IsStruct_ForAllocationFreeForeach() { - Assert.True(typeof(RecordEnumeratorCore, TextReaderSource>).IsValueType); - - int count = 0; - foreach (var _ in FixedWidth.Read(new StringReader("ABC\nDEF\n"))) - { - count++; - } - - Assert.Equal(2, count); + Assert.True(typeof(RecordEnumerator>).IsValueType); } [Fact] @@ -245,54 +158,41 @@ public void ReadFile_IsReEnumerable() } [Fact] - public async Task ReadAsync_MultipleLines_MatchesReflectionReader() + public async Task ReadAsync_MatchesReflectionReader() { - var reflection = new FixedWidthReader(Inv); - var expected = await reflection.ReadAsync(new StringReader(TwoPeople)).ToListAsync().ConfigureAwait(true); - var actual = await FixedWidth.ReadAsync( - new StringReader(TwoPeople), - formatProvider: Inv).ToListAsync().ConfigureAwait(true); + var expected = await new FixedWidthReader().ReadAsync(StreamOf(TwoPeople)).ToListAsync(TestContext.Current.CancellationToken); + var actual = await FixedWidth.ReadAsync(StreamOf(TwoPeople)).ToListAsync(TestContext.Current.CancellationToken); Assert.Equal(expected.Count, actual.Count); - Assert.Equal(expected[0].Name, actual[0].Name); - Assert.Equal(expected[0].Age, actual[0].Age); Assert.Equal(expected[1].Name, actual[1].Name); Assert.Equal(expected[1].Salary, actual[1].Salary, 2); } [Fact] - public async Task ReadAsync_LineLongerThanBuffer_MatchesReflectionReader() + public async Task ReadAsync_LeaveOpen_ControlsDisposal() { - var reflection = new FixedWidthReader(Inv, bufferSize: 4); - var expected = await reflection.ReadAsync(new StringReader(TwoPeople)).ToListAsync().ConfigureAwait(true); - var actual = await FixedWidth.ReadAsync( - new StringReader(TwoPeople), - formatProvider: Inv, - bufferSize: 4).ToListAsync().ConfigureAwait(true); + using var kept = StreamOf("ABC\n"); + var owned = StreamOf("ABC\n"); - Assert.Equal(expected[0].Name, actual[0].Name); - Assert.Equal(expected[1].Salary, actual[1].Salary, 2); + _ = await FixedWidth.ReadAsync(kept, leaveOpen: true).ToListAsync(TestContext.Current.CancellationToken); + _ = await FixedWidth.ReadAsync(owned).ToListAsync(TestContext.Current.CancellationToken); + + Assert.True(kept.CanRead); + Assert.False(owned.CanRead); } [Fact] - public async Task ReadAsync_InvalidLine_ThrowsWithSameLineNumber() + public async Task ReadAsync_CancelledToken_Throws() { - var reflection = new FixedWidthReader(Inv); - var text = "John Doe 30 60000.00 \nJane XX 55000.00 "; + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); - var expected = await Assert.ThrowsAsync(async () => + await Assert.ThrowsAnyAsync(async () => { - using StringReader reader = new(text); - await foreach (var _ in reflection.ReadAsync(reader).ConfigureAwait(false)) { } - }).ConfigureAwait(true); - var actual = await Assert.ThrowsAsync(async () => - { - using StringReader reader = new(text); - await foreach (var _ in FixedWidth.ReadAsync(reader, formatProvider: Inv).ConfigureAwait(false)) { } - }).ConfigureAwait(true); - - Assert.Contains("Line 2", expected.Message, StringComparison.Ordinal); - Assert.Contains("Line 2", actual.Message, StringComparison.Ordinal); + await foreach (var _ in FixedWidth.ReadAsync(StreamOf("ABC\nDEF\n")).WithCancellation(cts.Token)) + { + } + }); } [Fact] @@ -301,9 +201,9 @@ public async Task ReadFileAsync_MatchesSyncReadFile() string path = Path.GetTempFileName(); try { - await File.WriteAllTextAsync(path, "ABC\nDEF\nGHI\n").ConfigureAwait(true); + await File.WriteAllTextAsync(path, "ABC\nDEF\nGHI\n", TestContext.Current.CancellationToken); - var asyncCodes = await FixedWidth.ReadFileAsync(path).Select(m => m.Code).ToListAsync().ConfigureAwait(true); + var asyncCodes = await FixedWidth.ReadFileAsync(path).Select(m => m.Code).ToListAsync(TestContext.Current.CancellationToken); var syncCodes = FixedWidth.ReadFile(path).Select(m => m.Code).ToList(); Assert.Equal(["ABC", "DEF", "GHI"], asyncCodes); diff --git a/tests/FixedWidthParser.Tests/GeneratedUtf8ReaderTests.cs b/tests/FixedWidthParser.Tests/GeneratedUtf8ReaderTests.cs deleted file mode 100644 index 28ae2fe..0000000 --- a/tests/FixedWidthParser.Tests/GeneratedUtf8ReaderTests.cs +++ /dev/null @@ -1,328 +0,0 @@ -using System.Text; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Readers; -using static FixedWidthParser.Tests.TestHelpers; - -namespace FixedWidthParser.Tests -{ - /// - /// Covers the source-generated UTF-8 streaming path via (sync and - /// async): raw-byte reading with no transcode, line-ending handling, BOM skipping, buffer growth, - /// error reporting, leaveOpen behavior, struct-enumerator allocation freedom and cancellation. - /// Mirrors but exercises the generated TryParse strategy - /// instead of the reflection parser. - /// - public class GeneratedUtf8ReaderTests - { - private const string TwoPeople = - "John Doe 30 60000.00 \n" + - "Jane 28 55000.00 "; - - private static MemoryStream Utf8(string text) - { - return new(Encoding.UTF8.GetBytes(text)); - } - - // ----------------------------- Synchronous ----------------------------- - - - [Fact] - public void Read_Stream_ParsesAll() - { - using var stream = Utf8(TwoPeople); - - var people = FixedWidthUtf8.Read(stream, formatProvider: Inv).ToList(); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(30, people[0].Age); - Assert.Equal(60000.00, people[0].Salary, 2); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public void Read_StructEnumerator_IteratesManually() - { - // Exercises the public struct core returned by the generated UTF-8 enumerable directly - // (GetEnumerator / MoveNext / Current / Dispose) rather than through foreach. - using var stream = Utf8(TwoPeople); - var enumerable = FixedWidthUtf8.Read(stream, formatProvider: Inv); - - using var enumerator = enumerable.GetEnumerator(); - var names = new List(); - while (enumerator.MoveNext()) - { - names.Add(enumerator.Current.Name); - } - - Assert.Equal(["John Doe", "Jane"], names); - } - - [Theory] - [InlineData("ABC\nDEF\n")] // LF + trailing newline - [InlineData("ABC\r\nDEF\r\n")] // CRLF - [InlineData("ABC\nDEF")] // no trailing newline on the last line - public void Read_HandlesLineEndingsAndTrailingNewline(string text) - { - using var stream = Utf8(text); - - var codes = FixedWidthUtf8.Read(stream).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public void Read_SkipsEmptyLines() - { - using var stream = Utf8("ABC\n\n\nDEF\n"); - - var codes = FixedWidthUtf8.Read(stream).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public void Read_SkipsLeadingUtf8Bom() - { - byte[] bytes = [0xEF, 0xBB, 0xBF, .. "ABC\nDEF\n"u8]; - using var stream = new MemoryStream(bytes); - - var codes = FixedWidthUtf8.Read(stream).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public void Read_LineLongerThanBuffer_StillParses() - { - // A tiny bufferSize forces buffer compaction and growth (25-byte lines). - using var stream = Utf8(TwoPeople); - - var people = FixedWidthUtf8.Read(stream, formatProvider: Inv, bufferSize: 4).ToList(); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public void Read_InvalidLine_ThrowsWithLineNumber() - { - using var stream = Utf8("John Doe 30 60000.00 \nJane XX 55000.00 "); - - var ex = Assert.Throws(() => FixedWidthUtf8.Read(stream, formatProvider: Inv).ToList()); - - Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); - } - - [Fact] - public void Read_LeaveOpenTrue_KeepsStreamOpen() - { - using var stream = Utf8("ABC\nDEF\n"); - - _ = FixedWidthUtf8.Read(stream, leaveOpen: true).ToList(); - - Assert.True(stream.CanRead); - } - - [Fact] - public void Read_LeaveOpenFalse_DisposesStream() - { - var stream = Utf8("ABC\nDEF\n"); - - _ = FixedWidthUtf8.Read(stream).ToList(); // default leaveOpen: false - - Assert.False(stream.CanRead); - } - - [Fact] - public void Read_NullStream_Throws() - { - Assert.Throws(() => FixedWidthUtf8.Read(null!)); - } - - [Fact] - public void GetEnumerator_IsStruct_ForAllocationFreeForeach() - { - Assert.True(typeof(RecordEnumeratorCore, StreamSource>).IsValueType); - - using var stream = Utf8("ABC\nDEF\n"); - int count = 0; - foreach (var _ in FixedWidthUtf8.Read(stream)) - { - count++; - } - - - Assert.Equal(2, count); - } - - [Fact] - public void ReadFile_IsReEnumerable() - { - string path = Path.GetTempFileName(); - try - { - File.WriteAllText(path, "ABC\nDEF\n"); - - var first = FixedWidthUtf8.ReadFile(path).Select(m => m.Code).ToList(); - var second = FixedWidthUtf8.ReadFile(path).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], first); - Assert.Equal(first, second); - } - finally - { - File.Delete(path); - } - } - - // ----------------------------- Asynchronous ----------------------------- - - [Fact] - public async Task ReadAsync_Stream_ParsesAll() - { - await using var stream = Utf8(TwoPeople); - - var people = await FixedWidthUtf8.ReadAsync(stream, formatProvider: Inv).ToListAsync().ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(30, people[0].Age); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public async Task ReadAsync_LineLongerThanBuffer_StillParses() - { - await using var stream = Utf8(TwoPeople); - - var people = await FixedWidthUtf8.ReadAsync(stream, formatProvider: Inv, bufferSize: 4).ToListAsync().ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public async Task ReadAsync_SkipsLeadingUtf8Bom() - { - byte[] bytes = [0xEF, 0xBB, 0xBF, .. "ABC\nDEF\n"u8]; - await using var stream = new MemoryStream(bytes); - - var codes = await FixedWidthUtf8.ReadAsync(stream).Select(m => m.Code).ToListAsync().ConfigureAwait(true); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public async Task ReadAsync_InvalidLine_ThrowsWithLineNumber() - { - await using var stream = Utf8("John Doe 30 60000.00 \nJane XX 55000.00 "); - - var ex = await Assert.ThrowsAsync(async () => - { - await foreach (var _ in FixedWidthUtf8.ReadAsync(stream, formatProvider: Inv).ConfigureAwait(false)) { } - }).ConfigureAwait(true); - - Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); - } - - [Fact] - public async Task ReadAsync_CancelledToken_Throws() - { - await using var stream = Utf8("ABC\nDEF\n"); - using var cts = new CancellationTokenSource(); - await cts.CancelAsync().ConfigureAwait(true); - - await Assert.ThrowsAnyAsync(async () => - { - await foreach (var _ in FixedWidthUtf8.ReadAsync(stream).WithCancellation(cts.Token).ConfigureAwait(false)) { } - }).ConfigureAwait(true); - } - - [Fact] - public async Task ReadAsync_LeaveOpenTrue_KeepsStreamOpen() - { - await using var stream = Utf8("ABC\nDEF\n"); - - _ = await FixedWidthUtf8.ReadAsync(stream, leaveOpen: true).ToListAsync().ConfigureAwait(true); - - Assert.True(stream.CanRead); - } - - [Fact] - public async Task ReadAsync_LeaveOpenFalse_DisposesStream() - { - var stream = Utf8("ABC\nDEF\n"); - - _ = await FixedWidthUtf8.ReadAsync(stream).ToListAsync().ConfigureAwait(true); - - Assert.False(stream.CanRead); - } - - [Fact] - public void ReadAsync_NullStream_Throws() - { - Assert.Throws(() => FixedWidthUtf8.ReadAsync((Stream)null!)); - } - - [Fact] - public async Task ReadFileAsync_MatchesSyncReadFile() - { - string path = Path.GetTempFileName(); - try - { - await File.WriteAllTextAsync(path, "ABC\nDEF\nGHI\n").ConfigureAwait(true); - - var asyncCodes = await FixedWidthUtf8.ReadFileAsync(path).Select(m => m.Code).ToListAsync().ConfigureAwait(true); - var syncCodes = FixedWidthUtf8.ReadFile(path).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF", "GHI"], asyncCodes); - Assert.Equal(syncCodes, asyncCodes); - } - finally - { - File.Delete(path); - } - } - - // ----------------------------- StringPool ----------------------------- - - [Fact] - public void Read_Stream_WithStringPool_InternsRepeatedValues() - { - var pool = new StringPool(); - using var stream = Utf8("ABC\nABC\n"); - - var codes = FixedWidthUtf8.Read(stream, stringPool: pool).ToList(); - - Assert.Equal("ABC", codes[0].Code); - Assert.Same(codes[0].Code, codes[1].Code); - } - - // ----------------------------- generated/reflection parity ----------------------------- - - [Fact] - public void GeneratedReader_MatchesReflectionByteReader_ForAsciiStream() - { - var reflection = new FixedWidthByteReader(Inv); - - using var reflectionStream = Utf8(TwoPeople); - var fromReflection = reflection.Read(reflectionStream).ToList(); - using var generatedStream = Utf8(TwoPeople); - var fromGenerated = FixedWidthUtf8.Read(generatedStream, formatProvider: Inv).ToList(); - - Assert.Equal(fromReflection.Count, fromGenerated.Count); - for (int i = 0; i < fromReflection.Count; i++) - { - Assert.Equal(fromReflection[i].Name, fromGenerated[i].Name); - Assert.Equal(fromReflection[i].Age, fromGenerated[i].Age); - Assert.Equal(fromReflection[i].Salary, fromGenerated[i].Salary, 2); - } - } - } -} diff --git a/tests/FixedWidthParser.Tests/GeneratedWriterTests.cs b/tests/FixedWidthParser.Tests/GeneratedWriterTests.cs index 1bf877f..4d587a8 100644 --- a/tests/FixedWidthParser.Tests/GeneratedWriterTests.cs +++ b/tests/FixedWidthParser.Tests/GeneratedWriterTests.cs @@ -1,23 +1,20 @@ +using System.Globalization; using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { /// - /// Covers the source-generated TryFormat (via ): - /// parity against the reflection-based , round-tripping - /// through the generated TryParse, formatting options (alignment/padding/format/overflow), - /// nullable columns, custom converters, culture, and the destination-too-small case. + /// The source-generated TryFormat (via ): parity + /// with , round-tripping through the generated TryParse, + /// formatting options, nullable columns, converters, culture, and a destination that is too small. /// public class GeneratedWriterTests { private static string WriteGenerated(in TModel model, IFormatProvider? formatProvider = null) where TModel : IFixedWidthModel { - Span buffer = stackalloc char[256]; - bool ok = FixedWidth.TryFormat(in model, buffer, formatProvider ?? Inv, out int written); - Assert.True(ok); - return new string(buffer[..written]); + return Format(in model, CtxWith(formatProvider)); } [Theory] @@ -27,13 +24,10 @@ private static string WriteGenerated(in TModel model, IFormatProvider? f public void Person_MatchesReflection(string name, int age, double salary) { var writer = new FixedWidthWriter(); - var reflectionModel = new PersonModel { Name = name, Age = age, Salary = salary }; - var generatedModel = new GenPersonModel { Name = name, Age = age, Salary = salary }; - string reflectionLine = WriteOne(writer, reflectionModel); - string generatedLine = WriteGenerated(generatedModel); - - Assert.Equal(reflectionLine, generatedLine); + Assert.Equal( + WriteOne(writer, new PersonModel { Name = name, Age = age, Salary = salary }), + WriteGenerated(new GenPersonModel { Name = name, Age = age, Salary = salary })); } [Fact] @@ -42,9 +36,8 @@ public void Person_RoundTripsThroughGeneratedParse() var original = new GenPersonModel { Name = "Bob", Age = 42, Salary = 999.5 }; string line = WriteGenerated(original); - bool ok = FixedWidth.TryParse(line, Inv, null, out var parsed); - Assert.True(ok); + Assert.True(FixedWidth.TryParse(Bytes(line), Ctx, out var parsed)); Assert.Equal(original.Name, parsed.Name); Assert.Equal(original.Age, parsed.Age); Assert.Equal(original.Salary, parsed.Salary, 2); @@ -53,152 +46,127 @@ public void Person_RoundTripsThroughGeneratedParse() [Fact] public void FieldModel_MatchesReflection() { - var writer = new FixedWidthWriter(); - var reflectionModel = new ProductModel { Code = "ABCDE", Quantity = 1234 }; - var generatedModel = new GenProductModel { Code = "ABCDE", Quantity = 1234 }; - - Assert.Equal(WriteOne(writer, reflectionModel), WriteGenerated(generatedModel)); + Assert.Equal( + WriteOne(new FixedWidthWriter(), new ProductModel { Code = "ABCDE", Quantity = 1234 }), + WriteGenerated(new GenProductModel { Code = "ABCDE", Quantity = 1234 })); } [Fact] public void FloatColumn_MatchesReflection() { - var writer = new FixedWidthWriter(); - var reflectionModel = new MeasurementModel { Value = 3.14f }; - var generatedModel = new GenMeasurementModel { Value = 3.14f }; - - Assert.Equal(WriteOne(writer, reflectionModel), WriteGenerated(generatedModel)); + Assert.Equal( + WriteOne(new FixedWidthWriter(), new MeasurementModel { Value = 3.14f }), + WriteGenerated(new GenMeasurementModel { Value = 3.14f })); } [Fact] public void DecimalColumn_MatchesReflection() { - var writer = new FixedWidthWriter(); - var reflectionModel = new DecimalModel { Amount = 1234.56m }; - var generatedModel = new GenDecimalModel { Amount = 1234.56m }; - - Assert.Equal(WriteOne(writer, reflectionModel), WriteGenerated(generatedModel)); + Assert.Equal( + WriteOne(new FixedWidthWriter(), new DecimalModel { Amount = 1234.56m }), + WriteGenerated(new GenDecimalModel { Amount = 1234.56m })); } [Fact] public void RightAligned_MatchesReflection() { - var writer = new FixedWidthWriter(); - Assert.Equal( - WriteOne(writer, new RightAlignedModel { Value = 30 }), + WriteOne(new FixedWidthWriter(), new RightAlignedModel { Value = 30 }), WriteGenerated(new GenRightAlignedModel { Value = 30 })); } [Fact] public void ZeroPadded_MatchesReflection() { - var writer = new FixedWidthWriter(); - Assert.Equal( - WriteOne(writer, new ZeroPaddedModel { Value = 30 }), + WriteOne(new FixedWidthWriter(), new ZeroPaddedModel { Value = 30 }), WriteGenerated(new GenZeroPaddedModel { Value = 30 })); } [Fact] public void FormatString_MatchesReflection() { - var writer = new FixedWidthWriter(); - Assert.Equal( - WriteOne(writer, new FormattedModel { Amount = 1234.5 }), + WriteOne(new FixedWidthWriter(), new FormattedModel { Amount = 1234.5 }), WriteGenerated(new GenFormattedModel { Amount = 1234.5 })); } [Fact] public void Overflow_Throws_MatchesReflection() { - var writer = new FixedWidthWriter(); + Assert.Throws(() => WriteOne(new FixedWidthWriter(), new NarrowModel { Value = 12345 })); - Assert.Throws(() => WriteOne(writer, new NarrowModel { Value = 12345 })); - - var buffer = new char[16]; + var buffer = new byte[16]; var generatedModel = new GenNarrowModel { Value = 12345 }; - Assert.Throws(() => FixedWidth.TryFormat(in generatedModel, buffer, Inv, out _)); + Assert.Throws(() => FixedWidth.TryFormat(in generatedModel, buffer, Ctx, out _)); } [Fact] - public void DestinationTooSmall_ReturnsFalseWithZeroCharsWritten() + public void DestinationTooSmall_ReturnsFalseWithZeroBytesWritten() { var model = new GenPersonModel { Name = "Bob", Age = 1, Salary = 1 }; - Span tooSmall = stackalloc char[10]; + Span tooSmall = stackalloc byte[10]; - bool ok = FixedWidth.TryFormat(in model, tooSmall, Inv, out int charsWritten); + bool ok = FixedWidth.TryFormat(in model, tooSmall, Ctx, out int bytesWritten); Assert.False(ok); - Assert.Equal(0, charsWritten); + Assert.Equal(0, bytesWritten); } - // ----- Nullable columns ----- - [Fact] public void Nullable_Null_MatchesReflection() { - var writer = new FixedWidthWriter(); - var reflectionModel = new NullableModel { Age = null, Amount = null }; - var generatedModel = new GenNullableModel { Age = null, Amount = null }; - - Assert.Equal(WriteOne(writer, reflectionModel), WriteGenerated(generatedModel)); + Assert.Equal( + WriteOne(new FixedWidthWriter(), new NullableModel { Age = null, Amount = null }), + WriteGenerated(new GenNullableModel { Age = null, Amount = null })); } [Fact] public void Nullable_Value_MatchesReflection() { - var writer = new FixedWidthWriter(); - var reflectionModel = new NullableModel { Age = 42, Amount = 123.45m }; - var generatedModel = new GenNullableModel { Age = 42, Amount = 123.45m }; - - Assert.Equal(WriteOne(writer, reflectionModel), WriteGenerated(generatedModel)); + Assert.Equal( + WriteOne(new FixedWidthWriter(), new NullableModel { Age = 42, Amount = 123.45m }), + WriteGenerated(new GenNullableModel { Age = 42, Amount = 123.45m })); } - // ----- Custom converter ----- - [Fact] public void Converter_MatchesReflection() { - var writer = new FixedWidthWriter(); - var reflectionModel = new CentsConverterModel { Amount = new CentsValue(12345) }; - var generatedModel = new GenCentsConverterModel { Amount = new CentsValue(12345) }; - - Assert.Equal(WriteOne(writer, reflectionModel), WriteGenerated(generatedModel)); + Assert.Equal( + WriteOne(new FixedWidthWriter(), new CentsConverterModel { Amount = new CentsValue(12345) }), + WriteGenerated(new GenCentsConverterModel { Amount = new CentsValue(12345) })); } [Fact] - public void NullableConverter_Null_MatchesReflection() + public void NullableConverter_MatchesReflection() { var writer = new FixedWidthWriter(); - var reflectionModel = new NullableConverterModel { Amount = null }; - var generatedModel = new GenNullableConverterModel { Amount = null }; - Assert.Equal(WriteOne(writer, reflectionModel), WriteGenerated(generatedModel)); + Assert.Equal( + WriteOne(writer, new NullableConverterModel { Amount = null }), + WriteGenerated(new GenNullableConverterModel { Amount = null })); + Assert.Equal( + WriteOne(writer, new NullableConverterModel { Amount = new CentsValue(999) }), + WriteGenerated(new GenNullableConverterModel { Amount = new CentsValue(999) })); } [Fact] - public void NullableConverter_Value_MatchesReflection() + public void DecodedParsable_MatchesReflection() { - var writer = new FixedWidthWriter(); - var reflectionModel = new NullableConverterModel { Amount = new CentsValue(999) }; - var generatedModel = new GenNullableConverterModel { Amount = new CentsValue(999) }; - - Assert.Equal(WriteOne(writer, reflectionModel), WriteGenerated(generatedModel)); + Assert.Equal( + WriteOne(new FixedWidthWriter(), new ShoutModel { Word = new Shout("HEY") }), + WriteGenerated(new GenShoutModel { Word = new Shout("HEY") })); } - // ----- Culture ----- - [Fact] public void Culture_HonorsDecimalSeparator_MatchesReflection() { - var deDe = System.Globalization.CultureInfo.GetCultureInfo("de-DE"); - var writer = new FixedWidthWriter(); - var reflectionModel = new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }; - var generatedModel = new GenPersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }; + var deDe = CultureInfo.GetCultureInfo("de-DE"); + var writer = new FixedWidthWriter(new FixedWidthWriterOptions { FormatProvider = deDe }); - Assert.Equal(WriteOne(writer, reflectionModel, deDe), WriteGenerated(generatedModel, deDe)); + Assert.Equal( + WriteOne(writer, new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }), + WriteGenerated(new GenPersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }, deDe)); } } } diff --git a/tests/FixedWidthParser.Tests/NullableColumnTests.cs b/tests/FixedWidthParser.Tests/NullableColumnTests.cs index 3359d15..969fc68 100644 --- a/tests/FixedWidthParser.Tests/NullableColumnTests.cs +++ b/tests/FixedWidthParser.Tests/NullableColumnTests.cs @@ -1,4 +1,3 @@ -using System.Text; using FixedWidthParser.Parsers; using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; @@ -6,104 +5,55 @@ namespace FixedWidthParser.Tests { /// - /// Covers nullable value-type columns (T?): a blank (all-spaces) column parses to - /// without invoking the underlying parser, and a present value round-trips - /// exactly as the non-nullable T would. Exercises reflection parse/write (char + UTF-8) and - /// the generated parity, plus the combination with a custom converter. + /// Nullable value-type columns (T?): a blank column parses to without + /// invoking the underlying parser, and a present value round-trips exactly as T would. /// public class NullableColumnTests { - // NullableModel is Age[0,5) + Amount[5,15): build lines by padding each field to its column - // width, rather than hand-counting spaces in a literal (an easy off-by-one to get wrong). - private static string Line(string age, string amount) + // NullableModel is Age[0,5) + Amount[5,15). + private static byte[] Line(string age, string amount) { - return age.PadRight(5) + amount.PadRight(10); + return Bytes(age.PadRight(5) + amount.PadRight(10)); } - private static readonly string Blank = Line("", ""); - [Fact] - public void Reflection_Char_BlankColumn_ParsesToNull() + public void BlankColumn_ParsesToNull() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse(Blank, Inv, null, out var model); - - Assert.True(ok); + Assert.True(new FixedWidthParser().TryParse(Line("", ""), Ctx, out var model)); Assert.Null(model.Age); Assert.Null(model.Amount); } [Fact] - public void Reflection_Char_PresentValue_Parses() + public void PresentValue_Parses() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse(Line("30", "123.45"), Inv, null, out var model); - - Assert.True(ok); + Assert.True(new FixedWidthParser().TryParse(Line("30", "123.45"), Ctx, out var model)); Assert.Equal(30, model.Age); Assert.Equal(123.45m, model.Amount); } [Fact] - public void Reflection_Char_InvalidValue_ReturnsFalse() + public void InvalidValue_ReturnsFalse() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse(Line("XX", ""), Inv, null, out _); - - Assert.False(ok); + Assert.False(new FixedWidthParser().TryParse(Line("XX", ""), Ctx, out _)); } [Fact] - public void Reflection_Utf8_BlankColumn_ParsesToNull() + public void Write_Null_WritesBlank() { - var parser = new Utf8FixedWidthParser(); - byte[] bytes = Encoding.UTF8.GetBytes(Blank); - - bool ok = parser.TryParse(bytes, Inv, null, out var model); + string line = WriteOne(new FixedWidthWriter(), new NullableModel { Age = null, Amount = null }); - Assert.True(ok); - Assert.Null(model.Age); - Assert.Null(model.Amount); + Assert.Equal(new string(' ', 15), line); } [Fact] - public void Reflection_Utf8_PresentValue_Parses() + public void RoundTrip_PreservesNullAndValue() { - var parser = new Utf8FixedWidthParser(); - byte[] bytes = Encoding.UTF8.GetBytes(Line("30", "123.45")); - - bool ok = parser.TryParse(bytes, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal(30, model.Age); - Assert.Equal(123.45m, model.Amount); - } - - [Fact] - public void Reflection_Write_Null_WritesBlank() - { - var writer = new FixedWidthWriter(); - var model = new NullableModel { Age = null, Amount = null }; - - string line = WriteOne(writer, model); - - Assert.Equal(Blank, line); - } - - [Fact] - public void Reflection_RoundTrip_PreservesNullAndValue() - { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); var original = new NullableModel { Age = 42, Amount = null }; - string line = WriteOne(writer, original); - bool ok = parser.TryParse(line, Inv, null, out var parsed); + string line = WriteOne(new FixedWidthWriter(), original); - Assert.True(ok); + Assert.True(new FixedWidthParser().TryParse(Bytes(line), Ctx, out var parsed)); Assert.Equal(original.Age, parsed.Age); Assert.Null(parsed.Amount); } @@ -112,95 +62,42 @@ public void Reflection_RoundTrip_PreservesNullAndValue() [InlineData(" ")] [InlineData("30 123.45 ")] [InlineData("XX ")] - public void Generated_Char_MatchesReflection(string line) + public void Generated_MatchesReflection(string line) { - var reflection = new FixedWidthParser(); - - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); - - Assert.Equal(okR, okG); - if (okR) - { - Assert.Equal(r.Age, g.Age); - Assert.Equal(r.Amount, g.Amount); - } - } - - [Theory] - [InlineData(" ")] - [InlineData("30 123.45 ")] - public void Generated_Utf8_MatchesReflection(string line) - { - byte[] bytes = Encoding.UTF8.GetBytes(line); - var reflection = new Utf8FixedWidthParser(); - - bool okR = reflection.TryParse(bytes, Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse(bytes, Inv, null, out var g); + bool okR = new FixedWidthParser().TryParse(Bytes(line), Ctx, out var r); + bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); Assert.Equal(okR, okG); Assert.Equal(r.Age, g.Age); Assert.Equal(r.Amount, g.Amount); } - [Fact] - public void NullableConverter_Blank_ParsesToNull() + [Theory] + [InlineData(" ")] + [InlineData("12345 ")] + public void NullableConverter_Generated_MatchesReflection(string line) { - var parser = new FixedWidthParser(); + bool okR = new FixedWidthParser().TryParse(Bytes(line), Ctx, out var r); + bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); - bool ok = parser.TryParse(" ", Inv, null, out var model); - - Assert.True(ok); - Assert.Null(model.Amount); + Assert.True(okR); + Assert.True(okG); + Assert.Equal(r.Amount, g.Amount); } [Fact] public void NullableConverter_PresentValue_Parses() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("12345 ", Inv, null, out var model); - - Assert.True(ok); + Assert.True(new FixedWidthParser().TryParse("12345 "u8, Ctx, out var model)); Assert.Equal(new CentsValue(12345), model.Amount); } [Fact] public void NullableConverter_Write_Null_WritesBlank() { - var writer = new FixedWidthWriter(); - var model = new NullableConverterModel { Amount = null }; - - string line = WriteOne(writer, model); + string line = WriteOne(new FixedWidthWriter(), new NullableConverterModel { Amount = null }); Assert.Equal(new string(' ', 8), line); } - - [Fact] - public void NullableConverter_Generated_Char_MatchesReflection() - { - var reflection = new FixedWidthParser(); - const string line = "12345 "; - - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); - - Assert.Equal(okR, okG); - Assert.Equal(r.Amount, g.Amount); - } - - [Fact] - public void NullableConverter_Generated_Char_Blank_MatchesReflection() - { - var reflection = new FixedWidthParser(); - const string line = " "; - - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); - - Assert.Equal(okR, okG); - Assert.Null(r.Amount); - Assert.Null(g.Amount); - } } } diff --git a/tests/FixedWidthParser.Tests/ParserConstructionTests.cs b/tests/FixedWidthParser.Tests/ParserConstructionTests.cs deleted file mode 100644 index 1bb460b..0000000 --- a/tests/FixedWidthParser.Tests/ParserConstructionTests.cs +++ /dev/null @@ -1,61 +0,0 @@ -using FixedWidthParser.Parsers; - -namespace FixedWidthParser.Tests -{ - /// - /// Build-time paths of the runtime parsers. Both - /// and build their column plan in a static constructor - /// and capture any error, rethrowing it from the public constructor via - /// . These cover the - /// struct-without-declared-constructor branch of BuildModelFactory (default - /// initialization) and the error capture/rethrow (the layout-validation branches are covered by - /// for the char parser). - /// - public class ParserConstructionTests - { - // ----------------------------- char parser ----------------------------- - - [Fact] - public void Parser_StructWithoutDeclaredConstructor_Parses() - { - var parser = new FixedWidthParser(); - - Assert.True(parser.TryParse("ABC", null, null, out var model)); - Assert.Equal("ABC", model.Code); - } - - [Fact] - public void Parser_BuildError_IsRethrownOnEveryConstruction() - { - // The error is captured once in the static constructor and rethrown on each construction. - Assert.Throws(() => new FixedWidthParser()); - Assert.Throws(() => new FixedWidthParser()); - } - - // ----------------------------- UTF-8 parser ----------------------------- - - [Fact] - public void Utf8Parser_StructWithoutDeclaredConstructor_Parses() - { - var parser = new Utf8FixedWidthParser(); - - Assert.True(parser.TryParse("ABC"u8, null, null, out var model)); - Assert.Equal("ABC", model.Code); - } - - [Fact] - public void Utf8Parser_BuildError_IsRethrownOnEveryConstruction() - { - Assert.Throws(() => new Utf8FixedWidthParser()); - Assert.Throws(() => new Utf8FixedWidthParser()); - } - - [Fact] - public void Utf8Parser_OverlappingColumns_ThrowsOnConstruction() - { - var ex = Assert.Throws(() => new Utf8FixedWidthParser()); - - Assert.Contains("Overlapping", ex.Message, StringComparison.Ordinal); - } - } -} diff --git a/tests/FixedWidthParser.Tests/ParserTests.cs b/tests/FixedWidthParser.Tests/ParserTests.cs index 6d947a3..e780bce 100644 --- a/tests/FixedWidthParser.Tests/ParserTests.cs +++ b/tests/FixedWidthParser.Tests/ParserTests.cs @@ -11,8 +11,8 @@ public void TryParse_WellFormedLine_ParsesAllColumns() { var parser = new FixedWidthParser(); - // Name Age Salary - bool ok = parser.TryParse("John Doe 30 60000.00 ", Inv, null, out var model); + // Name Age Salary + bool ok = parser.TryParse("John Doe 30 60000.00 "u8, Ctx, out var model); Assert.True(ok); Assert.Equal("John Doe", model.Name); @@ -25,7 +25,7 @@ public void TryParse_TrimsTrailingSpacesOnStringColumns() { var parser = new FixedWidthParser(); - parser.TryParse("Al 25 100.00 ", Inv, null, out var model); + parser.TryParse("Al 25 100.00 "u8, Ctx, out var model); Assert.Equal("Al", model.Name); } @@ -35,18 +35,17 @@ public void TryParse_InvalidNumber_ReturnsFalse() { var parser = new FixedWidthParser(); - bool ok = parser.TryParse("John Doe XX 60000.00 ", Inv, null, out _); + bool ok = parser.TryParse("John Doe XX 60000.00 "u8, Ctx, out _); Assert.False(ok); } [Fact] - public void TryParse_ShortLine_NumericColumnMissing_ReturnsFalse() + public void TryParse_ShortLine_ReturnsFalse() { var parser = new FixedWidthParser(); - // The line ends before the Age column (start 10): the numeric processor fails. - bool ok = parser.TryParse("John", Inv, null, out _); + bool ok = parser.TryParse("John"u8, Ctx, out _); Assert.False(ok); } @@ -56,7 +55,7 @@ public void TryParse_ShortLine_StringColumnMissing_ReturnsFalse() { var parser = new FixedWidthParser(); - bool ok = parser.TryParse("42", Inv, null, out var model); + bool ok = parser.TryParse("42"u8, Ctx, out var model); Assert.False(ok); Assert.Equal(default, model); @@ -66,10 +65,10 @@ public void TryParse_ShortLine_StringColumnMissing_ReturnsFalse() public void TryParse_WithStringPool_ReturnsInternedInstances() { var parser = new FixedWidthParser(); - var pool = new StringPool(); + var context = CtxWith(pool: new StringPool()); - parser.TryParse("Alice 25 100.00 ", Inv, pool, out var first); - parser.TryParse("Alice 40 200.00 ", Inv, pool, out var second); + parser.TryParse("Alice 25 100.00 "u8, context, out var first); + parser.TryParse("Alice 40 200.00 "u8, context, out var second); Assert.Equal("Alice", first.Name); Assert.Same(first.Name, second.Name); @@ -80,8 +79,8 @@ public void TryParse_WithoutPool_ReturnsDistinctInstances() { var parser = new FixedWidthParser(); - parser.TryParse("Alice 25 100.00 ", Inv, null, out var first); - parser.TryParse("Alice 25 100.00 ", Inv, null, out var second); + parser.TryParse("Alice 25 100.00 "u8, Ctx, out var first); + parser.TryParse("Alice 25 100.00 "u8, Ctx, out var second); Assert.Equal(first.Name, second.Name); Assert.NotSame(first.Name, second.Name); @@ -92,7 +91,7 @@ public void TryParse_FieldBasedModel_ParsesFields() { var parser = new FixedWidthParser(); - bool ok = parser.TryParse("ABC 12 ", Inv, null, out var product); + bool ok = parser.TryParse("ABC 12 "u8, Ctx, out var product); Assert.True(ok); Assert.Equal("ABC", product.Code); @@ -104,10 +103,39 @@ public void TryParse_FloatColumn_Parses() { var parser = new FixedWidthParser(); - bool ok = parser.TryParse("3.14 ", Inv, null, out var measurement); + bool ok = parser.TryParse("3.14 "u8, Ctx, out var measurement); Assert.True(ok); Assert.Equal(3.14f, measurement.Value, 2); } + + [Fact] + public void TryParse_StructWithoutDeclaredConstructor_Parses() + { + var parser = new FixedWidthParser(); + + Assert.True(parser.TryParse("ABC"u8, Ctx, out var model)); + Assert.Equal("ABC", model.Code); + } + + [Fact] + public void Constructor_BuildError_IsRethrownOnEveryConstruction() + { + // The error is captured once in the static constructor and rethrown on each construction. + Assert.Throws(() => new FixedWidthParser()); + Assert.Throws(() => new FixedWidthParser()); + } + + [Fact] + public void TryParse_RefStructModel_Parses() + { + var parser = new FixedWidthParser(); + + bool ok = parser.TryParse("John Doe 30 60000.00 "u8, Ctx, out var model); + + Assert.True(ok); + Assert.Equal("John Doe", model.Name); + Assert.Equal(30, model.Age); + } } } diff --git a/tests/FixedWidthParser.Tests/PipeReaderTests.cs b/tests/FixedWidthParser.Tests/PipeReaderTests.cs index d15389f..696811b 100644 --- a/tests/FixedWidthParser.Tests/PipeReaderTests.cs +++ b/tests/FixedWidthParser.Tests/PipeReaderTests.cs @@ -6,8 +6,8 @@ namespace FixedWidthParser.Tests { /// - /// Covers the overloads of - /// (reflection) and (source-generated): line-ending + /// Covers the overloads of + /// (reflection) and (source-generated): line-ending /// handling, BOM skipping, empty-line skipping, multi-segment lines (the pooled scratch-copy path), /// cancellation, reader-completion behavior, and parity with the stream reader. /// @@ -45,7 +45,7 @@ private static async Task> CollectAsync(IAsyncEnumerable source) [Fact] public async Task ReadAsync_Pipe_ParsesAll() { - var reader = new FixedWidthByteReader(Inv); + var reader = new FixedWidthReader(); var people = await CollectAsync(reader.ReadAsync(Pipe(TwoPeople))).ConfigureAwait(true); @@ -61,7 +61,7 @@ public async Task ReadAsync_Pipe_ParsesAll() public async Task ReadAsync_Pipe_Generated_ParsesAll() { var people = await CollectAsync( - FixedWidthParser.FixedWidthUtf8.ReadAsync(Pipe(TwoPeople), formatProvider: Inv)) + FixedWidth.ReadAsync(Pipe(TwoPeople))) .ConfigureAwait(true); Assert.Equal(2, people.Count); @@ -75,7 +75,7 @@ public async Task ReadAsync_Pipe_Generated_ParsesAll() [InlineData("ABC\nDEF")] // no trailing newline on the last line public async Task ReadAsync_Pipe_HandlesLineEndingsAndTrailingNewline(string text) { - var reader = new FixedWidthByteReader(); + var reader = new FixedWidthReader(); var codes = (await CollectAsync(reader.ReadAsync(Pipe(text))).ConfigureAwait(true)).Select(m => m.Code).ToList(); @@ -85,7 +85,7 @@ public async Task ReadAsync_Pipe_HandlesLineEndingsAndTrailingNewline(string tex [Fact] public async Task ReadAsync_Pipe_SkipsEmptyLines() { - var reader = new FixedWidthByteReader(); + var reader = new FixedWidthReader(); var codes = (await CollectAsync(reader.ReadAsync(Pipe("ABC\n\n\nDEF\n"))).ConfigureAwait(true)).Select(m => m.Code).ToList(); @@ -95,7 +95,7 @@ public async Task ReadAsync_Pipe_SkipsEmptyLines() [Fact] public async Task ReadAsync_Pipe_SkipsLeadingBom() { - var reader = new FixedWidthByteReader(); + var reader = new FixedWidthReader(); var codes = (await CollectAsync(reader.ReadAsync(Pipe("ABC\nDEF\n"))).ConfigureAwait(true)).Select(m => m.Code).ToList(); @@ -107,7 +107,7 @@ public async Task ReadAsync_Pipe_MultiSegmentLine_ParsesViaScratch() { // A tiny pipe buffer (8 bytes) forces each 25-byte record to span several segments, so the // line is non-contiguous and must be copied into the pooled scratch buffer before parsing. - var reader = new FixedWidthByteReader(Inv); + var reader = new FixedWidthReader(); var people = await CollectAsync(reader.ReadAsync(Pipe(TwoPeople, bufferSize: 8))).ConfigureAwait(true); @@ -122,7 +122,7 @@ public async Task ReadAsync_Pipe_MultiSegmentLine_ParsesViaScratch() public async Task ReadAsync_Pipe_MatchesStreamReader() { // Same content, two sources: the pipe path must produce exactly what the stream path does. - var byteReader = new FixedWidthByteReader(Inv); + var byteReader = new FixedWidthReader(); #pragma warning disable IDISP004 // the reader owns and disposes the stream when iteration completes var fromStream = await CollectAsync(byteReader.ReadAsync(StreamOf(TwoPeople))).ConfigureAwait(true); @@ -141,7 +141,7 @@ public async Task ReadAsync_Pipe_MatchesStreamReader() [Fact] public async Task ReadAsync_Pipe_HonorsCancellation() { - var reader = new FixedWidthByteReader(); + var reader = new FixedWidthReader(); using var cts = new CancellationTokenSource(); await cts.CancelAsync().ConfigureAwait(true); @@ -157,7 +157,7 @@ await Assert.ThrowsAnyAsync(async () => [Fact] public async Task ReadAsync_Pipe_CompletesReaderByDefault() { - var reader = new FixedWidthByteReader(); + var reader = new FixedWidthReader(); var pipe = Pipe("ABC\nDEF\n"); _ = await CollectAsync(reader.ReadAsync(pipe)).ConfigureAwait(true); @@ -169,7 +169,7 @@ public async Task ReadAsync_Pipe_CompletesReaderByDefault() [Fact] public async Task ReadAsync_Pipe_LeaveOpen_DoesNotCompleteReader() { - var reader = new FixedWidthByteReader(); + var reader = new FixedWidthReader(); var pipe = Pipe("ABC\nDEF\n"); _ = await CollectAsync(reader.ReadAsync(pipe, leaveOpen: true)).ConfigureAwait(true); @@ -184,7 +184,7 @@ public async Task ReadAsync_Pipe_LeaveOpen_DoesNotCompleteReader() [Fact] public void ReadAsync_Pipe_NullReader_Throws() { - var reader = new FixedWidthByteReader(); + var reader = new FixedWidthReader(); Assert.Throws(() => reader.ReadAsync((PipeReader)null!)); } @@ -194,7 +194,7 @@ public void ReadAsync_Pipe_NullReader_Throws() Justification = "The test deliberately uses the enumerator after disposal to verify it throws.")] public async Task ReadAsync_Pipe_MoveNextAfterDispose_Throws() { - var reader = new FixedWidthByteReader(); + var reader = new FixedWidthReader(); var enumerator = reader.ReadAsync(Pipe("ABC\nDEF\n")).GetAsyncEnumerator(); await enumerator.DisposeAsync().ConfigureAwait(true); @@ -207,7 +207,7 @@ public async Task ReadAsync_Pipe_AssemblesAcrossManySmallWrites() // Feeding one byte at a time forces the BOM check and every line to span multiple reads, // exercising the "need more data" paths (both before and within a line). var pipe = new Pipe(); - var reader = new FixedWidthByteReader(); + var reader = new FixedWidthReader(); var produce = Task.Run(async () => { @@ -228,10 +228,10 @@ public async Task ReadAsync_Pipe_AssemblesAcrossManySmallWrites() [Fact] public Task ReadAsync_Pipe_UnparseableLine_ThrowsFormat() { - var reader = new FixedWidthByteReader(); + var reader = new FixedWidthReader(); // "A" is shorter than CodeModel's 3-wide column, so the trailing line fails to parse. - return Assert.ThrowsAsync(async () => + return Assert.ThrowsAsync(async () => await CollectAsync(reader.ReadAsync(Pipe("A"))).ConfigureAwait(false)); } @@ -239,7 +239,7 @@ public Task ReadAsync_Pipe_UnparseableLine_ThrowsFormat() public async Task ReadAsync_Pipe_CancelPendingRead_ThrowsOperationCanceledException() { var pipe = new Pipe(); - var reader = new FixedWidthByteReader(); + var reader = new FixedWidthReader(); var readTask = CollectAsync(reader.ReadAsync(pipe.Reader)); @@ -252,12 +252,12 @@ public async Task ReadAsync_Pipe_CancelPendingRead_ThrowsOperationCanceledExcept [Fact] public async Task ReadAsync_Pipe_UnparseableLine_WithLeaveOpen_LeavesReaderUsable() { - var reader = new FixedWidthByteReader(); + var reader = new FixedWidthReader(); var pipe = Pipe("A\nDEF\n"); var enumerator = reader.ReadAsync(pipe, leaveOpen: true).GetAsyncEnumerator(); - await Assert.ThrowsAsync(async () => await enumerator.MoveNextAsync().ConfigureAwait(false)).ConfigureAwait(true); + await Assert.ThrowsAsync(async () => await enumerator.MoveNextAsync().ConfigureAwait(false)).ConfigureAwait(true); await enumerator.DisposeAsync().ConfigureAwait(true); diff --git a/tests/FixedWidthParser.Tests/PropertyRoundTripTests.cs b/tests/FixedWidthParser.Tests/PropertyRoundTripTests.cs index 2e69651..06ab7d3 100644 --- a/tests/FixedWidthParser.Tests/PropertyRoundTripTests.cs +++ b/tests/FixedWidthParser.Tests/PropertyRoundTripTests.cs @@ -1,4 +1,3 @@ -using System.Text; using CsCheck; using FixedWidthParser.Attributes; using FixedWidthParser.Parsers; @@ -37,7 +36,7 @@ public void RoundTrip_TextAndNumber_IsLossless() GenTextNumber.Sample(original => { string line = WriteOne(writer, original); - bool ok = parser.TryParse(line, Inv, null, out var parsed); + bool ok = parser.TryParse(Bytes(line), Ctx, out var parsed); Assert.True(ok); Assert.Equal(original.Text, parsed.Text); @@ -55,7 +54,7 @@ public void RoundTrip_Decimal_IsExact() { var original = new MoneyModel { Amount = amount }; string line = WriteOne(writer, original); - bool ok = parser.TryParse(line, Inv, null, out var parsed); + bool ok = parser.TryParse(Bytes(line), Ctx, out var parsed); Assert.True(ok); Assert.Equal(original.Amount, parsed.Amount); @@ -73,35 +72,12 @@ public void RoundTrip_Double_F2_PreservesToTwoDecimals() { var original = new DoubleF2Model { Value = value }; string line = WriteOne(writer, original); - bool ok = parser.TryParse(line, Inv, null, out var parsed); + bool ok = parser.TryParse(Bytes(line), Ctx, out var parsed); Assert.True(ok); Assert.Equal(original.Value, parsed.Value, 2); }); } - - [Fact] - public void CharAndByteParsers_AgreeOnSameLine() - { - // The char writer emits ASCII (letters + digits), so byte offsets equal char offsets and the - // UTF-8 byte parser must produce exactly what the char parser does for the same line. - var writer = new FixedWidthWriter(); - var charParser = new FixedWidthParser(); - var byteParser = new Utf8FixedWidthParser(); - - GenTextNumber.Sample(original => - { - string line = WriteOne(writer, original); - - bool charOk = charParser.TryParse(line, Inv, null, out var fromChar); - bool byteOk = byteParser.TryParse(Encoding.UTF8.GetBytes(line), Inv, null, out var fromByte); - - Assert.True(charOk); - Assert.True(byteOk); - Assert.Equal(fromChar.Text, fromByte.Text); - Assert.Equal(fromChar.Number, fromByte.Number); - }); - } } // ----- Models whose layout makes the generated domain round-trip losslessly ----- diff --git a/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs b/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs new file mode 100644 index 0000000..9293665 --- /dev/null +++ b/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs @@ -0,0 +1,197 @@ +using System.IO.Pipelines; +using System.Text; +using FixedWidthParser.Readers; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// + /// : the column a names, + /// , + /// and validation. + /// + public class ReaderOptionsTests + { + private const string BadAge = + "John Doe 30 60000.00 \n" + + "Jane XX 55000.00 \n" + + "Max 41 72000.00 \n"; + + [Fact] + public void ParseError_NamesColumnAndRawValue_OnBothPaths() + { + var reflected = Assert.Throws(() => new FixedWidthReader().Read(StreamOf(BadAge)).ToList()); + var generated = Assert.Throws(() => FixedWidth.Read(StreamOf(BadAge)).ToList()); + + foreach (var ex in new[] { reflected, generated }) + { + Assert.Equal(2, ex.LineNumber); + Assert.Equal("Age", ex.ColumnName); + Assert.Equal(10, ex.ColumnStart); + Assert.Equal(5, ex.ColumnLength); + Assert.Equal("XX ", ex.RawValue); + Assert.Contains("column 'Age'", ex.Message, StringComparison.Ordinal); + } + } + + [Fact] + public void ParseError_ShortLine_SaysHowLongTheLayoutIs() + { + var reflected = Assert.Throws(() => new FixedWidthReader().Read(StreamOf("John\n")).ToList()); + var generated = Assert.Throws(() => FixedWidth.Read(StreamOf("John\n")).ToList()); + + foreach (var ex in new[] { reflected, generated }) + { + Assert.Null(ex.ColumnName); + Assert.Equal(-1, ex.ColumnStart); + Assert.Contains("is 4 bytes long and the layout needs 25", ex.Message, StringComparison.Ordinal); + } + } + + [Fact] + public void ParseError_IsAFormatException() + { + Assert.ThrowsAny(() => new FixedWidthReader().Read(StreamOf(BadAge)).ToList()); + } + + [Fact] + public void OnParseError_ReturningTrue_SkipsTheLine() + { + var errors = new List(); + var options = new FixedWidthReaderOptions { OnParseError = ex => { errors.Add(ex); return true; } }; + + var reflected = new FixedWidthReader(options).Read(StreamOf(BadAge)).Select(p => p.Name).ToList(); + var generated = FixedWidth.Read(StreamOf(BadAge), options: options).Select(p => p.Name).ToList(); + + Assert.Equal(["John Doe", "Max"], reflected); + Assert.Equal(reflected, generated); + Assert.Equal(2, errors.Count); + Assert.All(errors, ex => Assert.Equal(2, ex.LineNumber)); + } + + [Fact] + public void OnParseError_ReturningFalse_Throws() + { + var options = new FixedWidthReaderOptions { OnParseError = _ => false }; + + Assert.Throws(() => new FixedWidthReader(options).Read(StreamOf(BadAge)).ToList()); + } + + [Fact] + public void OnParseError_SkippingTheLastLine_EndsCleanly() + { + var options = new FixedWidthReaderOptions { OnParseError = _ => true }; + + var codes = new FixedWidthReader(options).Read(StreamOf("ABC\nA")).Select(m => m.Code).ToList(); + + Assert.Equal(["ABC"], codes); + } + + [Fact] + public async Task OnParseError_SkipsOnAsyncAndPipeReads() + { + var options = new FixedWidthReaderOptions { OnParseError = _ => true }; + var reader = new FixedWidthReader(options); + + var fromStream = await reader.ReadAsync(StreamOf(BadAge)).Select(p => p.Name).ToListAsync(TestContext.Current.CancellationToken); + var fromPipe = await reader.ReadAsync(PipeReader.Create(StreamOf(BadAge))).Select(p => p.Name).ToListAsync(TestContext.Current.CancellationToken); + + Assert.Equal(["John Doe", "Max"], fromStream); + Assert.Equal(fromStream, fromPipe); + } + + [Fact] + public void MaxLineLength_UnterminatedLine_Throws() + { + // No line break at all: the buffer would otherwise keep doubling. + var options = new FixedWidthReaderOptions { BufferSize = 16, MaxLineLength = 100 }; + var source = new MemoryStream(new byte[10_000]); + + var ex = Assert.Throws(() => new FixedWidthReader(options).Read(source).ToList()); + + Assert.Contains("MaxLineLength", ex.Message, StringComparison.Ordinal); + Assert.Contains("Line 1", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void MaxLineLength_TerminatedLongLine_Throws() + { + var options = new FixedWidthReaderOptions { MaxLineLength = 5 }; + + var ex = Assert.Throws(() => new FixedWidthReader(options).Read(StreamOf("ABC\nABCDEFGH\n")).ToList()); + + Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void MaxLineLength_LineAtTheLimit_IsRead() + { + var options = new FixedWidthReaderOptions { BufferSize = 2, MaxLineLength = 3 }; + + var codes = new FixedWidthReader(options).Read(StreamOf("ABC\r\nDEF")).Select(m => m.Code).ToList(); + + Assert.Equal(["ABC", "DEF"], codes); + } + + [Fact] + public async Task MaxLineLength_UnterminatedLine_ThrowsOnAsyncRead() + { + var options = new FixedWidthReaderOptions { BufferSize = 16, MaxLineLength = 100 }; + + await Assert.ThrowsAsync(async () => + { + await foreach (var _ in new FixedWidthReader(options).ReadAsync(new MemoryStream(new byte[10_000]))) + { + } + }); + } + + [Fact] + public async Task MaxLineLength_UnterminatedLine_ThrowsOnPipeRead() + { + var options = new FixedWidthReaderOptions { MaxLineLength = 100 }; + var pipe = PipeReader.Create(new MemoryStream(new byte[10_000]), new StreamPipeReaderOptions(bufferSize: 64)); + + await Assert.ThrowsAsync(async () => + { + await foreach (var _ in FixedWidth.ReadAsync(pipe, options: options)) + { + } + }); + } + + [Theory] + [InlineData(0, 10)] + [InlineData(10, 0)] + public void Options_OutOfRange_Throw(int bufferSize, int maxLineLength) + { + var options = new FixedWidthReaderOptions { BufferSize = bufferSize, MaxLineLength = maxLineLength }; + + Assert.Throws(() => new FixedWidthReader(options)); + Assert.Throws(() => FixedWidth.Read(StreamOf("ABC"), options: options)); + } + + [Fact] + public void Latin1Encoding_DoesNotSkipWhatLooksLikeAUtf8Bom() + { + // In Latin-1, EF BB BF are the characters "", not a byte order mark. + var options = new FixedWidthReaderOptions { Encoding = Encoding.Latin1 }; + byte[] bytes = [0xEF, 0xBB, 0xBF, .. "\nABC\n"u8]; + + var codes = new FixedWidthReader(options).Read(new MemoryStream(bytes)).Select(m => m.Code).ToList(); + + Assert.Equal(["", "ABC"], codes); + } + + [Fact] + public void Utf8Text_WithMultiByteCharacters_KeepsBytePositions() + { + // "Zoë" is 4 bytes in UTF-8; the layout counts bytes, so the name column must be 6 wide here + // and the age still starts at byte 6. + var person = new FixedWidthReader().Read(StreamOf("Zoë 42 \n")).Single(); + + Assert.Equal("Zoë", person.Name); + Assert.Equal(42, person.Age); + } + } +} diff --git a/tests/FixedWidthParser.Tests/ReaderStreamTests.cs b/tests/FixedWidthParser.Tests/ReaderStreamTests.cs deleted file mode 100644 index 0fdb206..0000000 --- a/tests/FixedWidthParser.Tests/ReaderStreamTests.cs +++ /dev/null @@ -1,136 +0,0 @@ -using System.Text; -using FixedWidthParser.Readers; -using static FixedWidthParser.Tests.TestHelpers; - -namespace FixedWidthParser.Tests -{ - /// - /// Covers the Stream overloads of (sync and async), - /// including custom encoding and the leaveOpen behavior. - /// - public class ReaderStreamTests - { - private const string TwoPeople = - "John Doe 30 60000.00 \n" + - "Jane 28 55000.00 "; - - private static MemoryStream Utf8(string text) - { - return new(Encoding.UTF8.GetBytes(text)); - } - - // ----------------------------- Synchronous ----------------------------- - - [Fact] - public void Read_Stream_ParsesAll() - { - var reader = new FixedWidthReader(Inv); - using var stream = Utf8(TwoPeople); - - var people = reader.Read(stream).ToList(); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public void Read_Stream_CustomEncoding_IsHonored() - { - // 'Á' (0xC1) is a single byte in Latin1 but invalid as standalone UTF-8. - var reader = new FixedWidthReader(); - using var stream = new MemoryStream(Encoding.Latin1.GetBytes("Áb \n")); - - var codes = reader.Read(stream, Encoding.Latin1).Select(m => m.Code).ToList(); - - Assert.Equal(["Áb"], codes); - } - - [Fact] - public void Read_Stream_LeaveOpenTrue_KeepsStreamOpen() - { - var reader = new FixedWidthReader(); - using var stream = Utf8("ABC\nDEF\n"); - - _ = reader.Read(stream, leaveOpen: true).ToList(); - - Assert.True(stream.CanRead); // not disposed - } - - [Fact] - public void Read_Stream_LeaveOpenFalse_DisposesStream() - { - var reader = new FixedWidthReader(); - var stream = Utf8("ABC\nDEF\n"); - - _ = reader.Read(stream).ToList(); // default leaveOpen: false - - Assert.False(stream.CanRead); // disposed together with the StreamReader - } - - [Fact] - public void Read_NullStream_Throws() - { - var reader = new FixedWidthReader(); - - Assert.Throws(() => reader.Read((Stream)null!)); - } - - // ----------------------------- Asynchronous ----------------------------- - - [Fact] - public async Task ReadAsync_Stream_ParsesAll() - { - var reader = new FixedWidthReader(Inv); - await using var stream = Utf8(TwoPeople); - - var people = await reader.ReadAsync(stream).ToListAsync().ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal("Jane", people[1].Name); - } - - [Fact] - public async Task ReadAsync_Stream_CustomEncoding_IsHonored() - { - var reader = new FixedWidthReader(); - await using MemoryStream stream = new(Encoding.Latin1.GetBytes("Áb \n")); - - var codes = await reader.ReadAsync(stream, Encoding.Latin1).Select(m => m.Code).ToListAsync().ConfigureAwait(true); - - Assert.Equal(["Áb"], codes); - } - - [Fact] - public async Task ReadAsync_Stream_LeaveOpenTrue_KeepsStreamOpen() - { - var reader = new FixedWidthReader(); - await using var stream = Utf8("ABC\nDEF\n"); - - _ = await reader.ReadAsync(stream, leaveOpen: true).ToListAsync().ConfigureAwait(true); - - Assert.True(stream.CanRead); - } - - [Fact] - public async Task ReadAsync_Stream_LeaveOpenFalse_DisposesStream() - { - var reader = new FixedWidthReader(); - var stream = Utf8("ABC\nDEF\n"); - - _ = await reader.ReadAsync(stream).ToListAsync().ConfigureAwait(true); - - Assert.False(stream.CanRead); - } - - [Fact] - public void ReadAsync_NullStream_Throws() - { - var reader = new FixedWidthReader(); - - Assert.Throws(() => reader.ReadAsync((Stream)null!)); - } - } -} diff --git a/tests/FixedWidthParser.Tests/ReaderTests.cs b/tests/FixedWidthParser.Tests/ReaderTests.cs index 2290c7d..8eca5d4 100644 --- a/tests/FixedWidthParser.Tests/ReaderTests.cs +++ b/tests/FixedWidthParser.Tests/ReaderTests.cs @@ -5,9 +5,13 @@ namespace FixedWidthParser.Tests { + /// + /// over streams and files, sync and async: line endings, BOM, + /// buffer growth, error reporting, leaveOpen and string pooling. + /// public class ReaderTests { - // Lines in the PersonModel layout (Name[0,10) Age[10,5) Salary[15,10)), 25 chars each. + // Lines in the PersonModel layout (Name[0,10) Age[10,5) Salary[15,10)), 25 bytes each. private const string TwoPeople = "John Doe 30 60000.00 \n" + "Jane 28 55000.00 "; @@ -15,27 +19,23 @@ public class ReaderTests [Fact] public void Read_MultipleLines_ParsesAll() { - var reader = new FixedWidthReader(Inv); - - var people = reader.Read(new StringReader(TwoPeople)).ToList(); + var people = new FixedWidthReader().Read(StreamOf(TwoPeople)).ToList(); Assert.Equal(2, people.Count); Assert.Equal("John Doe", people[0].Name); Assert.Equal(30, people[0].Age); Assert.Equal(60000.00, people[0].Salary, 2); Assert.Equal("Jane", people[1].Name); - Assert.Equal(28, people[1].Age); + Assert.Equal(55000.00, people[1].Salary, 2); } [Theory] - [InlineData("ABC\nDEF\n")] // LF + trailing newline - [InlineData("ABC\r\nDEF\r\n")] // CRLF - [InlineData("ABC\nDEF")] // no trailing newline on the last line + [InlineData("ABC\nDEF\n")] + [InlineData("ABC\r\nDEF\r\n")] + [InlineData("ABC\nDEF")] public void Read_HandlesLineEndingsAndTrailingNewline(string text) { - var reader = new FixedWidthReader(); - - var codes = reader.Read(new StringReader(text)).Select(m => m.Code).ToList(); + var codes = new FixedWidthReader().Read(StreamOf(text)).Select(m => m.Code).ToList(); Assert.Equal(["ABC", "DEF"], codes); } @@ -43,9 +43,17 @@ public void Read_HandlesLineEndingsAndTrailingNewline(string text) [Fact] public void Read_SkipsEmptyLines() { - var reader = new FixedWidthReader(); + var codes = new FixedWidthReader().Read(StreamOf("ABC\n\n\nDEF\n")).Select(m => m.Code).ToList(); - var codes = reader.Read(new StringReader("ABC\n\n\nDEF\n")).Select(m => m.Code).ToList(); + Assert.Equal(["ABC", "DEF"], codes); + } + + [Fact] + public void Read_SkipsLeadingUtf8Bom() + { + byte[] bytes = [0xEF, 0xBB, 0xBF, .. "ABC\nDEF\n"u8]; + + var codes = new FixedWidthReader().Read(new MemoryStream(bytes)).Select(m => m.Code).ToList(); Assert.Equal(["ABC", "DEF"], codes); } @@ -53,36 +61,60 @@ public void Read_SkipsEmptyLines() [Fact] public void Read_LineLongerThanBuffer_StillParses() { - // A tiny bufferSize forces buffer compaction and growth (25-char lines). - var reader = new FixedWidthReader(Inv, bufferSize: 4); + // A tiny buffer forces compaction and growth (25-byte lines). + var reader = new FixedWidthReader(new FixedWidthReaderOptions { BufferSize = 4 }); - var people = reader.Read(new StringReader(TwoPeople)).ToList(); + var people = reader.Read(StreamOf(TwoPeople)).ToList(); Assert.Equal(2, people.Count); Assert.Equal("John Doe", people[0].Name); - Assert.Equal("Jane", people[1].Name); Assert.Equal(55000.00, people[1].Salary, 2); } [Fact] public void Read_InvalidLine_ThrowsWithLineNumber() { - var reader = new FixedWidthReader(Inv); - var text = "John Doe 30 60000.00 \nJane XX 55000.00 "; // invalid Age on line 2 + var reader = new FixedWidthReader(); - var ex = Assert.Throws( - () => reader.Read(new StringReader(text)).ToList()); + var ex = Assert.Throws( + () => reader.Read(StreamOf("John Doe 30 60000.00 \nJane XX 55000.00 ")).ToList()); + Assert.Equal(2, ex.LineNumber); Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); } + [Fact] + public void Read_LeaveOpenTrue_KeepsStreamOpen() + { + using var stream = StreamOf("ABC\nDEF\n"); + + _ = new FixedWidthReader().Read(stream, leaveOpen: true).ToList(); + + Assert.True(stream.CanRead); + } + + [Fact] + public void Read_LeaveOpenFalse_DisposesStream() + { + var stream = StreamOf("ABC\nDEF\n"); + + _ = new FixedWidthReader().Read(stream).ToList(); + + Assert.False(stream.CanRead); + } + + [Fact] + public void Read_NullStream_Throws() + { + Assert.Throws(() => new FixedWidthReader().Read(null!)); + } + [Fact] public void Read_WithStringPool_InternsRepeatedValues() { - var pool = new StringPool(); - var reader = new FixedWidthReader(stringPool: pool); + var reader = new FixedWidthReader(new FixedWidthReaderOptions { StringPool = new StringPool() }); - var codes = reader.Read(new StringReader("ABC\nABC\n")).ToList(); + var codes = reader.Read(StreamOf("ABC\nABC\n")).ToList(); Assert.Equal("ABC", codes[0].Code); Assert.Same(codes[0].Code, codes[1].Code); @@ -91,11 +123,10 @@ public void Read_WithStringPool_InternsRepeatedValues() [Fact] public void GetEnumerator_IsStruct_ForAllocationFreeForeach() { - Assert.True(typeof(RecordEnumeratorCore, TextReaderSource>).IsValueType); + Assert.True(typeof(RecordEnumerator>).IsValueType); - var reader = new FixedWidthReader(); int count = 0; - foreach (var _ in reader.Read(new StringReader("ABC\nDEF\n"))) + foreach (var _ in new FixedWidthReader().Read(StreamOf("ABC\nDEF\n"))) { count++; } @@ -103,6 +134,16 @@ public void GetEnumerator_IsStruct_ForAllocationFreeForeach() Assert.Equal(2, count); } + [Fact] + public void TryParse_UsesTheReaderOptions() + { + var reader = new FixedWidthReader(new FixedWidthReaderOptions { FormatProvider = System.Globalization.CultureInfo.GetCultureInfo("de-DE") }); + + Assert.True(reader.TryParse("1234,56 "u8, out var model)); + Assert.Equal(1234.56m, model.Amount); + Assert.False(reader.TryParse("short"u8, out _)); + } + [Fact] public void ReadFile_IsReEnumerable() { @@ -124,16 +165,44 @@ public void ReadFile_IsReEnumerable() } } - // ----------------------- Asynchronous reading ----------------------- + [Fact] + public void Writer_To_File_To_Reader_RoundTrip() + { + string path = Path.GetTempFileName(); + try + { + PersonModel[] models = + [ + new() { Name = "John Doe", Age = 30, Salary = 60000 }, + new() { Name = "Jane", Age = 28, Salary = 55000 }, + ]; + using (var fs = File.Create(path)) + { + new FixedWidthWriter().WriteMany(fs, models.AsSpan()); + } + + var parsed = new FixedWidthReader().ReadFile(path).ToList(); + + Assert.Equal(2, parsed.Count); + Assert.Equal("John Doe", parsed[0].Name); + Assert.Equal(60000, parsed[0].Salary, 2); + Assert.Equal("Jane", parsed[1].Name); + } + finally + { + File.Delete(path); + } + } + + // ----------------------------- Asynchronous ----------------------------- [Fact] - public async Task ReadAsync_MultipleLines_ParsesAll() + public async Task ReadAsync_Stream_ParsesAll() { - var reader = new FixedWidthReader(Inv); - var people = await reader.ReadAsync(new StringReader(TwoPeople)).ToListAsync().ConfigureAwait(true); + var people = await new FixedWidthReader().ReadAsync(StreamOf(TwoPeople)).ToListAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, people.Count); Assert.Equal("John Doe", people[0].Name); - Assert.Equal(30, people[0].Age); Assert.Equal("Jane", people[1].Name); Assert.Equal(55000.00, people[1].Salary, 2); } @@ -141,38 +210,71 @@ public async Task ReadAsync_MultipleLines_ParsesAll() [Fact] public async Task ReadAsync_LineLongerThanBuffer_StillParses() { - var reader = new FixedWidthReader(Inv, bufferSize: 4); - var people = await reader.ReadAsync(new StringReader(TwoPeople)).ToListAsync().ConfigureAwait(true); + var reader = new FixedWidthReader(new FixedWidthReaderOptions { BufferSize = 4 }); + + var people = await reader.ReadAsync(StreamOf(TwoPeople)).ToListAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); Assert.Equal(55000.00, people[1].Salary, 2); } + [Fact] + public async Task ReadAsync_SkipsLeadingUtf8Bom() + { + byte[] bytes = [0xEF, 0xBB, 0xBF, .. "ABC\nDEF\n"u8]; + + var codes = await new FixedWidthReader().ReadAsync(new MemoryStream(bytes)).Select(m => m.Code).ToListAsync(TestContext.Current.CancellationToken); + + Assert.Equal(["ABC", "DEF"], codes); + } + [Fact] public async Task ReadAsync_InvalidLine_ThrowsWithLineNumber() { - var reader = new FixedWidthReader(Inv); - var text = "John Doe 30 60000.00 \nJane XX 55000.00 "; + var reader = new FixedWidthReader(); - var ex = await Assert.ThrowsAsync(async () => + var ex = await Assert.ThrowsAsync(async () => { - await foreach (var _ in reader.ReadAsync(new StringReader(text)).ConfigureAwait(false)) { } - }).ConfigureAwait(true); + await foreach (var _ in reader.ReadAsync(StreamOf("John Doe 30 60000.00 \nJane XX 55000.00 "))) + { + } + }); - Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); + Assert.Equal(2, ex.LineNumber); } [Fact] public async Task ReadAsync_CancelledToken_Throws() { - var reader = new FixedWidthReader(); using var cts = new CancellationTokenSource(); - await cts.CancelAsync().ConfigureAwait(true); + await cts.CancelAsync(); await Assert.ThrowsAnyAsync(async () => { - await foreach (var _ in reader.ReadAsync(new StringReader("ABC\nDEF\n")).WithCancellation(cts.Token).ConfigureAwait(false)) { } - }).ConfigureAwait(true); + await foreach (var _ in new FixedWidthReader().ReadAsync(StreamOf("ABC\nDEF\n")).WithCancellation(cts.Token)) + { + } + }); + } + + [Fact] + public async Task ReadAsync_LeaveOpen_ControlsDisposal() + { + var reader = new FixedWidthReader(); + using var kept = StreamOf("ABC\n"); + var owned = StreamOf("ABC\n"); + + _ = await reader.ReadAsync(kept, leaveOpen: true).ToListAsync(TestContext.Current.CancellationToken); + _ = await reader.ReadAsync(owned).ToListAsync(TestContext.Current.CancellationToken); + + Assert.True(kept.CanRead); + Assert.False(owned.CanRead); + } + + [Fact] + public void ReadAsync_NullStream_Throws() + { + Assert.Throws(() => new FixedWidthReader().ReadAsync((Stream)null!)); } [Fact] @@ -181,9 +283,10 @@ public async Task ReadFileAsync_MatchesSyncReadFile() string path = Path.GetTempFileName(); try { - await File.WriteAllTextAsync(path, "ABC\nDEF\nGHI\n").ConfigureAwait(true); + await File.WriteAllTextAsync(path, "ABC\nDEF\nGHI\n", TestContext.Current.CancellationToken); var reader = new FixedWidthReader(); - var asyncCodes = await reader.ReadFileAsync(path).Select(m => m.Code).ToListAsync().ConfigureAwait(true); + + var asyncCodes = await reader.ReadFileAsync(path).Select(m => m.Code).ToListAsync(TestContext.Current.CancellationToken); var syncCodes = reader.ReadFile(path).Select(m => m.Code).ToList(); Assert.Equal(["ABC", "DEF", "GHI"], asyncCodes); @@ -196,36 +299,13 @@ public async Task ReadFileAsync_MatchesSyncReadFile() } [Fact] - public void Writer_To_File_To_Reader_RoundTrip() + public async Task ReadAsync_WithStringPool_InternsRepeatedValues() { - string path = Path.GetTempFileName(); - try - { - var models = new[] - { - new PersonModel { Name = "John Doe", Age = 30, Salary = 60000 }, - new PersonModel { Name = "Jane", Age = 28, Salary = 55000 }, - }; - var writer = new FixedWidthWriter(); - using (var fs = File.Create(path)) - { - writer.WriteMany(fs, models.AsSpan(), Inv); - } + var reader = new FixedWidthReader(new FixedWidthReaderOptions { StringPool = new StringPool() }); - var reader = new FixedWidthReader(Inv); - var parsed = reader.ReadFile(path).ToList(); + var codes = await reader.ReadAsync(StreamOf("ABC\nABC\n")).ToListAsync(TestContext.Current.CancellationToken); - Assert.Equal(2, parsed.Count); - Assert.Equal("John Doe", parsed[0].Name); - Assert.Equal(30, parsed[0].Age); - Assert.Equal(60000, parsed[0].Salary, 2); - Assert.Equal("Jane", parsed[1].Name); - Assert.Equal(55000, parsed[1].Salary, 2); - } - finally - { - File.Delete(path); - } + Assert.Same(codes[0].Code, codes[1].Code); } } } diff --git a/tests/FixedWidthParser.Tests/RefStructTests.cs b/tests/FixedWidthParser.Tests/RefStructTests.cs deleted file mode 100644 index d444052..0000000 --- a/tests/FixedWidthParser.Tests/RefStructTests.cs +++ /dev/null @@ -1,27 +0,0 @@ -using FixedWidthParser.Parsers; -using static FixedWidthParser.Tests.TestHelpers; - -namespace FixedWidthParser.Tests -{ -#if NET9_0_OR_GREATER - /// - /// Confirms the parser's allows ref struct constraint is real: a ref struct model is - /// built and parsed at runtime (Func/expression trees accept ref struct on .NET 10). - /// - public class RefStructTests - { - [Fact] - public void Parser_SupportsRefStructModel() - { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("John Doe 30 60000.00 ", Inv, null, out var model); - - Assert.True(ok); - Assert.Equal("John Doe", model.Name); - Assert.Equal(30, model.Age); - Assert.Equal(60000.00, model.Salary, 2); - } - } -#endif -} diff --git a/tests/FixedWidthParser.Tests/RoundTripTests.cs b/tests/FixedWidthParser.Tests/RoundTripTests.cs index b20d9fa..27a5155 100644 --- a/tests/FixedWidthParser.Tests/RoundTripTests.cs +++ b/tests/FixedWidthParser.Tests/RoundTripTests.cs @@ -26,7 +26,7 @@ public void RoundTrip_Person_PreservesValues(string name, int age, double salary var original = new PersonModel { Name = name, Age = age, Salary = salary }; string line = WriteOne(writer, original); - bool ok = parser.TryParse(line, Inv, null, out var parsed); + bool ok = parser.TryParse(Bytes(line), Ctx, out var parsed); Assert.True(ok); Assert.Equal(original.Name, parsed.Name); @@ -42,7 +42,7 @@ public void RoundTrip_FieldModel_PreservesValues() var original = new ProductModel { Code = "ABCDE", Quantity = 1234 }; string line = WriteOne(writer, original); - bool ok = parser.TryParse(line, Inv, null, out var parsed); + bool ok = parser.TryParse(Bytes(line), Ctx, out var parsed); Assert.True(ok); Assert.Equal(original.Code, parsed.Code); diff --git a/tests/FixedWidthParser.Tests/SourceGenParityTests.cs b/tests/FixedWidthParser.Tests/SourceGenParityTests.cs index 1718416..5c4d3b9 100644 --- a/tests/FixedWidthParser.Tests/SourceGenParityTests.cs +++ b/tests/FixedWidthParser.Tests/SourceGenParityTests.cs @@ -1,28 +1,27 @@ -using System.Globalization; +using System.Text; using CommunityToolkit.HighPerformance.Buffers; using FixedWidthParser.Parsers; +using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { /// - /// Verifies the source-generated TryParse (via ) - /// produces identical results to the runtime, reflection-based . + /// The source-generated TryParse (via ) produces the + /// same results as the runtime , including what each reports + /// about a rejected line. /// public class SourceGenParityTests { - private static readonly CultureInfo Inv = CultureInfo.InvariantCulture; - [Theory] - [InlineData("John Doe 30 6000000.00")] // clean, full width - [InlineData("Jane 28 55000.00 ")] // trailing spaces in the numeric column - [InlineData("John")] // short line: numeric columns out of bounds - [InlineData("")] // empty line - [InlineData("Ann abc 10.5 ")] // non-numeric age: must be rejected by both + [InlineData("John Doe 30 6000000.00")] + [InlineData("Jane 28 55000.00 ")] + [InlineData("John")] + [InlineData("")] + [InlineData("Ann abc 10.5 ")] public void Person_GeneratedMatchesReflection(string line) { - var reflection = new FixedWidthParser(); - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); + bool okR = new FixedWidthParser().TryParse(Bytes(line), Ctx, out var r); + bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); Assert.Equal(okR, okG); if (okR) @@ -36,8 +35,7 @@ public void Person_GeneratedMatchesReflection(string line) [Fact] public void Person_ParsesExpectedValues() { - bool ok = FixedWidth.TryParse("John Doe 30 6000000.00", Inv, null, out var g); - Assert.True(ok); + Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, Ctx, out var g)); Assert.Equal("John Doe", g.Name); Assert.Equal(30, g.Age); Assert.Equal(6000000.00, g.Salary); @@ -46,19 +44,15 @@ public void Person_ParsesExpectedValues() [Theory] [InlineData("ABCDE0042")] [InlineData("X 7 ")] - [InlineData("AB")] // short: Quantity out of bounds -> rejected by both + [InlineData("AB")] public void Product_GeneratedMatchesReflection(string line) { - var reflection = new FixedWidthParser(); - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); + bool okR = new FixedWidthParser().TryParse(Bytes(line), Ctx, out var r); + bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); Assert.Equal(okR, okG); - if (okR) - { - Assert.Equal(r.Code, g.Code); - Assert.Equal(r.Quantity, g.Quantity); - } + Assert.Equal(r.Code, g.Code); + Assert.Equal(r.Quantity, g.Quantity); } [Theory] @@ -67,37 +61,21 @@ public void Product_GeneratedMatchesReflection(string line) [InlineData("notanum ")] public void Measurement_GeneratedMatchesReflection(string line) { - var reflection = new FixedWidthParser(); - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); + bool okR = new FixedWidthParser().TryParse(Bytes(line), Ctx, out var r); + bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); Assert.Equal(okR, okG); - if (okR) - { - Assert.Equal(r.Value, g.Value); - } + Assert.Equal(r.Value, g.Value); } - [Fact] - public void TrailingString_ShortLineGeneratedMatchesReflection() - { - var reflection = new FixedWidthParser(); - bool okR = reflection.TryParse("42", Inv, null, out _); - bool okG = FixedWidth.TryParse("42", Inv, null, out _); - - Assert.False(okR); - Assert.False(okG); - } - - [Fact] - public void TrailingString_FullLineGeneratedMatchesReflection() + [Theory] + [InlineData("42")] + [InlineData("42 short ")] + public void TrailingString_GeneratedMatchesReflection(string line) { - const string line = "42 short "; - var reflection = new FixedWidthParser(); - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); + bool okR = new FixedWidthParser().TryParse(Bytes(line), Ctx, out var r); + bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); - Assert.True(okR); Assert.Equal(okR, okG); Assert.Equal(r.Id, g.Id); Assert.Equal(r.Note, g.Note); @@ -109,48 +87,62 @@ public void TrailingString_FullLineGeneratedMatchesReflection() [InlineData("bad ")] public void Decimal_GeneratedMatchesReflection(string line) { - var reflection = new FixedWidthParser(); - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); + bool okR = new FixedWidthParser().TryParse(Bytes(line), Ctx, out var r); + bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); Assert.Equal(okR, okG); - if (okR) - { - Assert.Equal(r.Amount, g.Amount); - } + Assert.Equal(r.Amount, g.Amount); } -#if NET9_0_OR_GREATER [Fact] - public void RefStruct_GeneratedMatchesReflection() + public void DecodedParsable_GeneratedMatchesReflection() { - const string line = "John Doe 30 6000000.00"; - var reflection = new FixedWidthParser(); - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); + bool okR = new FixedWidthParser().TryParse("hey "u8, Ctx, out var r); + bool okG = FixedWidth.TryParse("hey "u8, Ctx, out var g); Assert.True(okR); - Assert.Equal(okR, okG); + Assert.True(okG); + Assert.Equal("HEY", r.Word.Text); + Assert.Equal(r.Word, g.Word); + } + + [Fact] + public void Latin1_GeneratedMatchesReflection() + { + // "Antônio" doesn't fit; "Joséé" is 5 characters and 5 bytes in Latin-1, 7 bytes in UTF-8. + var context = CtxWith(encoding: Encoding.Latin1); + byte[] line = Encoding.Latin1.GetBytes("Joséé 42 "); + + Assert.True(new FixedWidthParser().TryParse(line, context, out var r)); + Assert.True(FixedWidth.TryParse(line, context, out var g)); + + Assert.Equal("Joséé", r.Name); + Assert.Equal(42, r.Age); + Assert.Equal(r.Name, g.Name); + Assert.Equal(r.Age, g.Age); + } + + [Fact] + public void RefStruct_GeneratedMatchesReflection() + { + Assert.True(new FixedWidthParser().TryParse("John Doe 30 6000000.00"u8, Ctx, out var r)); + Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, Ctx, out var g)); + Assert.Equal(r.Name, g.Name); Assert.Equal(r.Age, g.Age); Assert.Equal(r.Salary, g.Salary); } -#endif [Fact] public void StringPool_InternsAndMatchesReflection() { - const string line = "John Doe 30 6000000.00"; var pool = new StringPool(); - var reflection = new FixedWidthParser(); + var context = CtxWith(pool: pool); - bool okR = reflection.TryParse(line, Inv, pool, out var r); - bool okG = FixedWidth.TryParse(line, Inv, pool, out var g); + Assert.True(new FixedWidthParser().TryParse("John Doe 30 6000000.00"u8, context, out var r)); + Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, context, out var g)); - Assert.True(okR); - Assert.Equal(okR, okG); - Assert.Equal(r.Name, g.Name); - // Same pooled instance is returned for the same content. + Assert.Same(r.Name, g.Name); Assert.Same(pool.GetOrAdd("John Doe".AsSpan()), g.Name); } } diff --git a/tests/FixedWidthParser.Tests/TestHelpers.cs b/tests/FixedWidthParser.Tests/TestHelpers.cs index 1c82f6f..3cd1c23 100644 --- a/tests/FixedWidthParser.Tests/TestHelpers.cs +++ b/tests/FixedWidthParser.Tests/TestHelpers.cs @@ -1,5 +1,6 @@ using System.Globalization; using System.Text; +using CommunityToolkit.HighPerformance.Buffers; using FixedWidthParser.Writers; namespace FixedWidthParser.Tests @@ -8,32 +9,58 @@ internal static class TestHelpers { public static readonly CultureInfo Inv = CultureInfo.InvariantCulture; + /// Invariant culture, UTF-8, no pool. + public static readonly FixedWidthContext Ctx = default; + + public static FixedWidthContext CtxWith(IFormatProvider? provider = null, StringPool? pool = null, Encoding? encoding = null) + { + return new FixedWidthContext { FormatProvider = provider!, StringPool = pool, Encoding = encoding! }; + } + + public static byte[] Bytes(string text) + { + return Encoding.UTF8.GetBytes(text); + } + + public static MemoryStream StreamOf(string content) + { + return new MemoryStream(Encoding.UTF8.GetBytes(content)); + } + + /// Formats a generated model and returns the line as UTF-8 text. + public static string Format(in T model, in FixedWidthContext context = default) where T : IFixedWidthModel + { + Span destination = stackalloc byte[4096]; + Assert.True(FixedWidth.TryFormat(in model, destination, in context, out int written)); + return Encoding.UTF8.GetString(destination[..written]); + } + /// Writes a model and returns the line without the trailing newline. - public static string WriteOne(FixedWidthWriter writer, in T model, IFormatProvider? formatProvider = null) + public static string WriteOne(FixedWidthWriter writer, in T model) { using var ms = new MemoryStream(); - writer.Write(ms, in model, formatProvider ?? Inv); + writer.Write(ms, in model); return Decode(ms).TrimEnd('\r', '\n'); } public static async Task WriteOneAsync(FixedWidthWriter writer, T model) { await using var ms = new MemoryStream(); - await writer.WriteAsync(ms, model, Inv).ConfigureAwait(false); + await writer.WriteAsync(ms, model, TestContext.Current.CancellationToken).ConfigureAwait(false); return Decode(ms).TrimEnd('\r', '\n'); } public static string WriteMany(FixedWidthWriter writer, ReadOnlySpan models) { using var ms = new MemoryStream(); - writer.WriteMany(ms, models, Inv); + writer.WriteMany(ms, models); return Decode(ms); } public static string WriteMany(FixedWidthWriter writer, IEnumerable models) { using var ms = new MemoryStream(); - writer.WriteMany(ms, models, Inv); + writer.WriteMany(ms, models); return Decode(ms); } diff --git a/tests/FixedWidthParser.Tests/TestModels.cs b/tests/FixedWidthParser.Tests/TestModels.cs index 9737595..523731d 100644 --- a/tests/FixedWidthParser.Tests/TestModels.cs +++ b/tests/FixedWidthParser.Tests/TestModels.cs @@ -8,32 +8,12 @@ namespace FixedWidthParser.Tests /// Custom value type with no ISpanParsable — only parseable via a FixedColumnAttribute.Converter. public readonly record struct CentsValue(long Cents); - /// - /// Stateless / for - /// : stores cents as a plain integer. Implements both directions (parse/format) - /// and both element types (char/byte), so one attribute round-trips through every path. - /// - public sealed class CentsConverter : IFixedWidthConverter, IUtf8FixedWidthConverter + /// Stateless for : stores cents as a plain integer. + public sealed class CentsConverter : IFixedWidthConverter { - public bool TryParse(ReadOnlySpan field, IFormatProvider? formatProvider, out CentsValue value) - { - if (!long.TryParse(field, formatProvider, out long cents)) - { - value = default; - return false; - } - value = new CentsValue(cents); - return true; - } - - public bool TryFormat(CentsValue value, Span destination, IFormatProvider? formatProvider, out int written) - { - return value.Cents.TryFormat(destination, out written, provider: formatProvider); - } - - public bool TryParse(ReadOnlySpan field, IFormatProvider? formatProvider, out CentsValue value) + public bool TryParse(ReadOnlySpan field, in FixedWidthContext context, out CentsValue value) { - if (!long.TryParse(field, formatProvider, out long cents)) + if (!long.TryParse(field, context.FormatProvider, out long cents)) { value = default; return false; @@ -42,27 +22,25 @@ public bool TryParse(ReadOnlySpan field, IFormatProvider? formatProvider, return true; } - public bool TryFormat(CentsValue value, Span destination, IFormatProvider? formatProvider, out int written) + public bool TryFormat(CentsValue value, Span destination, in FixedWidthContext context, out int written) { - return value.Cents.TryFormat(destination, out written, provider: formatProvider); + return value.Cents.TryFormat(destination, out written, provider: context.FormatProvider); } } - /// Converter that only implements the char-side interface — used to prove a converter can - /// support one direction/element type without the other being required. + /// Converter for int, attached to a string column to prove the mismatch is rejected. public sealed class IntOnlyConverter : IFixedWidthConverter { - public bool TryParse(ReadOnlySpan field, IFormatProvider? formatProvider, out int value) + public bool TryParse(ReadOnlySpan field, in FixedWidthContext context, out int value) { - return int.TryParse(field, formatProvider, out value); + return int.TryParse(field, context.FormatProvider, out value); } - public bool TryFormat(int value, Span destination, IFormatProvider? formatProvider, out int written) + public bool TryFormat(int value, Span destination, in FixedWidthContext context, out int written) { - return value.TryFormat(destination, out written, provider: formatProvider); + return value.TryFormat(destination, out written, provider: context.FormatProvider); } } - /// Single custom-converted column, exercised by both the reflection parser/writer and the generator. public readonly record struct CentsConverterModel { @@ -332,28 +310,27 @@ public readonly record struct WideUtf8NumericModel [StructLayout(LayoutKind.Auto)] public readonly record struct BigValue(int Length); - /// Char-side-only converter for ; emits N 'x' chars on format. + /// Converter for ; emits N 'x' bytes on format. public sealed class BigConverter : IFixedWidthConverter { - public bool TryParse(ReadOnlySpan field, IFormatProvider? formatProvider, out BigValue value) + public bool TryParse(ReadOnlySpan field, in FixedWidthContext context, out BigValue value) { - value = new BigValue(field.Trim().Length); + value = new BigValue(field.Trim((byte)' ').Length); return true; } - public bool TryFormat(BigValue value, Span destination, IFormatProvider? formatProvider, out int written) + public bool TryFormat(BigValue value, Span destination, in FixedWidthContext context, out int written) { if (destination.Length < value.Length) { written = 0; return false; } - destination[..value.Length].Fill('x'); + destination[..value.Length].Fill((byte)'x'); written = value.Length; return true; } } - [StructLayout(LayoutKind.Auto)] public readonly record struct ZeroPaddedLeadingModel { @@ -385,4 +362,67 @@ public readonly record struct TruncateInvalidModel { [FixedColumn(0, 5, Overflow = OverflowBehavior.Truncate)] public int Value { get; init; } } -} + + /// Padding set to a non-ASCII character: the runtime writer must refuse it. + public readonly record struct NonAsciiPaddingModel + { + [FixedColumn(0, 5, Alignment = Alignment.Right, Padding = 'é')] public int Value { get; init; } + } + + /// A column type with no parser of any kind and no converter. + public readonly record struct UnsupportedTypeModel + { + [FixedColumn(0, 10)] public Uri? Link { get; init; } + } + + /// Parses only from characters (no ), so the byte path decodes it first. + public readonly record struct Shout(string Text) : ISpanParsable, ISpanFormattable + { + public static Shout Parse(ReadOnlySpan s, IFormatProvider? provider) + { + return new Shout(s.ToString().ToUpperInvariant()); + } + + public static bool TryParse(ReadOnlySpan s, IFormatProvider? provider, out Shout result) + { + result = Parse(s, provider); + return true; + } + + [ExcludeFromCodeCoverage] + public static Shout Parse(string s, IFormatProvider? provider) + { + return Parse(s.AsSpan(), provider); + } + + [ExcludeFromCodeCoverage] + public static bool TryParse(string? s, IFormatProvider? provider, out Shout result) + { + return TryParse(s.AsSpan(), provider, out result); + } + + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? provider) + { + charsWritten = Text.Length; + return Text.AsSpan().TryCopyTo(destination); + } + + [ExcludeFromCodeCoverage] + public string ToString(string? format, IFormatProvider? formatProvider) + { + return Text; + } + } + + public readonly record struct ShoutModel + { + [FixedColumn(0, 6)] public Shout Word { get; init; } + } + + /// A Latin-1 name column followed by a number, to prove positions stay bytes in single-byte encodings. + public readonly record struct NameAgeModel + { + [FixedColumn(0, 6)] public string Name { get; init; } + [FixedColumn(6, 3)] public int Age { get; init; } + } +} \ No newline at end of file diff --git a/tests/FixedWidthParser.Tests/TrimCharTests.cs b/tests/FixedWidthParser.Tests/TrimCharTests.cs index 56835f5..a7ec6ce 100644 --- a/tests/FixedWidthParser.Tests/TrimCharTests.cs +++ b/tests/FixedWidthParser.Tests/TrimCharTests.cs @@ -1,120 +1,37 @@ -using System.Text; using FixedWidthParser.Parsers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { - /// - /// Covers FixedColumnAttribute.TrimChar: the character trimmed from the end of a column on - /// the READ side (parsing) is now configurable, instead of hardcoded ' '. Exercises reflection - /// parse (char + UTF-8), generated parity, and the non-ASCII-TrimChar guard on the UTF-8 byte path - /// (mirrors the existing decimal-separator ASCII guard in CultureHelpers). - /// + /// FixedColumnAttribute.TrimChar: the character trimmed from a column when parsing. public class TrimCharTests { - [Fact] - public void Reflection_Char_AsteriskTrim_ParsesUnderlyingValue() - { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("42***", Inv, null, out var model); - - Assert.True(ok); - Assert.Equal(42, model.Value); - } - - [Fact] - public void Reflection_Char_HashTrim_StringColumn() - { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("AB######", Inv, null, out var model); - - Assert.True(ok); - Assert.Equal("AB", model.Code); - } - - [Fact] - public void Reflection_Utf8_AsteriskTrim_ParsesUnderlyingValue() - { - var parser = new Utf8FixedWidthParser(); - byte[] bytes = Encoding.UTF8.GetBytes("42***"); - - bool ok = parser.TryParse(bytes, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal(42, model.Value); - } - - [Fact] - public void Reflection_Utf8_HashTrim_StringColumn() - { - var parser = new Utf8FixedWidthParser(); - byte[] bytes = Encoding.UTF8.GetBytes("AB######"); - - bool ok = parser.TryParse(bytes, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal("AB", model.Code); - } - - [Fact] - public void Reflection_Utf8_NonAsciiTrimChar_Throws() - { - Assert.Throws(() => new Utf8FixedWidthParser()); - } - [Theory] - [InlineData("42***")] - [InlineData("1****")] - public void Generated_Char_MatchesReflection(string line) + [InlineData("42***", 42)] + [InlineData("1****", 1)] + public void AsteriskTrim_ParsesUnderlyingValue_OnBothPaths(string line, int expected) { - var reflection = new FixedWidthParser(); - - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); + Assert.True(new FixedWidthParser().TryParse(Bytes(line), Ctx, out var reflected)); + Assert.True(FixedWidth.TryParse(Bytes(line), Ctx, out var generated)); - Assert.Equal(okR, okG); - Assert.Equal(r.Value, g.Value); + Assert.Equal(expected, reflected.Value); + Assert.Equal(expected, generated.Value); } [Fact] - public void Generated_Utf8_MatchesReflection() + public void HashTrim_StringColumn_OnBothPaths() { - byte[] bytes = Encoding.UTF8.GetBytes("42***"); - var reflection = new Utf8FixedWidthParser(); + Assert.True(new FixedWidthParser().TryParse("AB######"u8, Ctx, out var reflected)); + Assert.True(FixedWidth.TryParse("AB######"u8, Ctx, out var generated)); - bool okR = reflection.TryParse(bytes, Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse(bytes, Inv, null, out var g); - - Assert.Equal(okR, okG); - Assert.Equal(r.Value, g.Value); + Assert.Equal("AB", reflected.Code); + Assert.Equal("AB", generated.Code); } [Fact] - public void Generated_Char_HashTrim_MatchesReflection() + public void NonAsciiTrimChar_Throws() { - const string line = "AB######"; - var reflection = new FixedWidthParser(); - - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); - - Assert.Equal(okR, okG); - Assert.Equal(r.Code, g.Code); - } - - [Fact] - public void Generated_Utf8_NonAsciiTrimChar_Throws() - { - // The guard runs in the generated model's static field initializer (__trim0), so it - // surfaces wrapped in a TypeInitializationException on first use of the type. - byte[] bytes = Encoding.UTF8.GetBytes(" 42"); - - var ex = Assert.Throws( - () => FixedWidthUtf8.TryParse(bytes, Inv, null, out _)); - - Assert.IsType(ex.InnerException); + Assert.Throws(() => new FixedWidthParser()); } } } diff --git a/tests/FixedWidthParser.Tests/Utf8SourceGenParityTests.cs b/tests/FixedWidthParser.Tests/Utf8SourceGenParityTests.cs deleted file mode 100644 index 0e9f3d3..0000000 --- a/tests/FixedWidthParser.Tests/Utf8SourceGenParityTests.cs +++ /dev/null @@ -1,164 +0,0 @@ -using System.Globalization; -using System.Text; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Parsers; - -namespace FixedWidthParser.Tests -{ - /// - /// Verifies the source-generated UTF-8 TryParse (via ) - /// produces identical results to the runtime, reflection-based . - /// Mirrors over the byte path. All fixtures are ASCII, so byte - /// offsets equal character offsets. - /// - public class Utf8SourceGenParityTests - { - private static readonly CultureInfo Inv = CultureInfo.InvariantCulture; - - // u8 literals can't appear in [InlineData]; encode the ASCII fixture at the call site instead. - private static byte[] U8(string s) - { - return Encoding.UTF8.GetBytes(s); - } - - - [Theory] - [InlineData("John Doe 30 6000000.00")] // clean, full width - [InlineData("Jane 28 55000.00 ")] // trailing spaces in the numeric column - [InlineData("John")] // short line: numeric columns out of bounds - [InlineData("")] // empty line - [InlineData("Ann abc 10.5 ")] // non-numeric age: must be rejected by both - public void Person_GeneratedMatchesReflection(string line) - { - var reflection = new Utf8FixedWidthParser(); - bool okR = reflection.TryParse(U8(line), Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse(U8(line), Inv, null, out var g); - - Assert.Equal(okR, okG); - if (okR) - { - Assert.Equal(r.Name, g.Name); - Assert.Equal(r.Age, g.Age); - Assert.Equal(r.Salary, g.Salary); - } - } - - [Fact] - public void Person_ParsesExpectedValues() - { - bool ok = FixedWidthUtf8.TryParse("John Doe 30 6000000.00"u8, Inv, null, out var g); - Assert.True(ok); - Assert.Equal("John Doe", g.Name); - Assert.Equal(30, g.Age); - Assert.Equal(6000000.00, g.Salary); - } - - [Theory] - [InlineData("ABCDE0042")] - [InlineData("X 7 ")] - [InlineData("AB")] // short: Quantity out of bounds -> rejected by both - public void Product_GeneratedMatchesReflection(string line) - { - var reflection = new Utf8FixedWidthParser(); - bool okR = reflection.TryParse(U8(line), Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse(U8(line), Inv, null, out var g); - - Assert.Equal(okR, okG); - if (okR) - { - Assert.Equal(r.Code, g.Code); - Assert.Equal(r.Quantity, g.Quantity); - } - } - - [Theory] - [InlineData("3.14159 ")] - [InlineData(" -2.5 ")] - [InlineData("notanum ")] - public void Measurement_GeneratedMatchesReflection(string line) - { - var reflection = new Utf8FixedWidthParser(); - bool okR = reflection.TryParse(U8(line), Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse(U8(line), Inv, null, out var g); - - Assert.Equal(okR, okG); - if (okR) - { - Assert.Equal(r.Value, g.Value); - } - } - - [Fact] - public void TrailingString_ShortLineGeneratedMatchesReflection() - { - var reflection = new Utf8FixedWidthParser(); - bool okR = reflection.TryParse("42"u8, Inv, null, out _); - bool okG = FixedWidthUtf8.TryParse("42"u8, Inv, null, out _); - - Assert.False(okR); - Assert.False(okG); - } - - [Fact] - public void TrailingString_FullLineGeneratedMatchesReflection() - { - var reflection = new Utf8FixedWidthParser(); - bool okR = reflection.TryParse("42 short "u8, Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse("42 short "u8, Inv, null, out var g); - - Assert.True(okR); - Assert.Equal(okR, okG); - Assert.Equal(r.Id, g.Id); - Assert.Equal(r.Note, g.Note); - } - - [Theory] - [InlineData("123.45 ")] - [InlineData("-0.01 ")] - [InlineData("bad ")] - public void Decimal_GeneratedMatchesReflection(string line) - { - var reflection = new Utf8FixedWidthParser(); - bool okR = reflection.TryParse(U8(line), Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse(U8(line), Inv, null, out var g); - - Assert.Equal(okR, okG); - if (okR) - { - Assert.Equal(r.Amount, g.Amount); - } - } - -#if NET9_0_OR_GREATER - [Fact] - public void RefStruct_GeneratedMatchesReflection() - { - var reflection = new Utf8FixedWidthParser(); - bool okR = reflection.TryParse("John Doe 30 6000000.00"u8, Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse("John Doe 30 6000000.00"u8, Inv, null, out var g); - - Assert.True(okR); - Assert.Equal(okR, okG); - Assert.Equal(r.Name, g.Name); - Assert.Equal(r.Age, g.Age); - Assert.Equal(r.Salary, g.Salary); - } -#endif - - [Fact] - public void StringPool_InternsAndMatchesReflection() - { - var pool = new StringPool(); - var reflection = new Utf8FixedWidthParser(); - - bool okR = reflection.TryParse("John Doe 30 6000000.00"u8, Inv, pool, out var r); - bool okG = FixedWidthUtf8.TryParse("John Doe 30 6000000.00"u8, Inv, pool, out var g); - - Assert.True(okR); - Assert.Equal(okR, okG); - Assert.Equal(r.Name, g.Name); - // Same pooled instance is returned for the same content (UTF-8 decoded once). - Assert.Same(pool.GetOrAdd("John Doe".AsSpan()), g.Name); - } - } -} diff --git a/tests/FixedWidthParser.Tests/WriterAsyncTests.cs b/tests/FixedWidthParser.Tests/WriterAsyncTests.cs index fa0b24b..9c3cac0 100644 --- a/tests/FixedWidthParser.Tests/WriterAsyncTests.cs +++ b/tests/FixedWidthParser.Tests/WriterAsyncTests.cs @@ -5,10 +5,7 @@ namespace FixedWidthParser.Tests { - /// - /// Covers the asynchronous batch write path (WriteManyAsync) for both the Stream and - /// StreamWriter overloads, checking output parity with the synchronous path. - /// + /// WriteManyAsync: output parity with the synchronous path. public class WriterAsyncTests { private static PersonModel[] SampleModels() @@ -21,7 +18,7 @@ private static PersonModel[] SampleModels() } [Fact] - public async Task WriteManyAsync_Stream_MatchesSyncOutput() + public async Task WriteManyAsync_MatchesSyncOutput() { var writer = new FixedWidthWriter(); var models = SampleModels(); @@ -29,44 +26,20 @@ public async Task WriteManyAsync_Stream_MatchesSyncOutput() string sync = WriteMany(writer, (IEnumerable)models); await using var ms = new MemoryStream(); - await writer.WriteManyAsync(ms, models, Inv).ConfigureAwait(true); - string asyncResult = Encoding.UTF8.GetString(ms.ToArray()); + await writer.WriteManyAsync(ms, models, TestContext.Current.CancellationToken); - Assert.Equal(sync, asyncResult); + Assert.Equal(sync, Encoding.UTF8.GetString(ms.ToArray())); } [Fact] - public async Task WriteManyAsync_StreamWriter_MatchesSyncOutput() + public async Task WriteManyAsync_LeavesStreamOpen_AndRoundTripsThroughReader() { - var writer = new FixedWidthWriter(); - var models = SampleModels(); - - string sync = WriteMany(writer, (IEnumerable)models); - - await using var ms = new MemoryStream(); - await using (var sw = new StreamWriter(ms, leaveOpen: true)) - { - await writer.WriteManyAsync(sw, models, Inv).ConfigureAwait(true); - } - string asyncResult = Encoding.UTF8.GetString(ms.ToArray()); - - Assert.Equal(sync, asyncResult); - } - - [Fact] - public async Task WriteManyAsync_Stream_LeavesStreamOpen_AndRoundTripsThroughReader() - { - var writer = new FixedWidthWriter(); - var reader = new FixedWidthReader(Inv); - var models = SampleModels(); - await using var ms = new MemoryStream(); - await writer.WriteManyAsync(ms, models, Inv).ConfigureAwait(true); + await new FixedWidthWriter().WriteManyAsync(ms, SampleModels(), TestContext.Current.CancellationToken); - // The Stream overload uses leaveOpen: true, so the stream stays usable. Assert.True(ms.CanRead); ms.Position = 0; - var parsed = reader.Read(ms, leaveOpen: true).ToList(); + var parsed = new FixedWidthReader().Read(ms, leaveOpen: true).ToList(); Assert.Equal(3, parsed.Count); Assert.Equal("John Doe", parsed[0].Name); @@ -75,26 +48,24 @@ public async Task WriteManyAsync_Stream_LeavesStreamOpen_AndRoundTripsThroughRea } [Fact] - public async Task WriteManyAsync_Stream_EmptyCollection_WritesNothing() + public async Task WriteManyAsync_ManyRecords_SpansSeveralBatches() { - var writer = new FixedWidthWriter(); + // 5,000 records of 27 bytes exceed the 64 KB batch, so the buffer is flushed mid-way. + var models = Enumerable.Range(0, 5000).Select(i => new PersonModel { Name = "P" + i, Age = i % 100, Salary = i }).ToList(); + var writer = new FixedWidthWriter(new FixedWidthWriterOptions { NewLine = "\r\n" }); await using var ms = new MemoryStream(); - await writer.WriteManyAsync(ms, [], Inv).ConfigureAwait(true); + await writer.WriteManyAsync(ms, models, TestContext.Current.CancellationToken); - Assert.Equal(0, ms.Length); + Assert.Equal(5000 * 27, ms.Length); + Assert.Equal(WriteMany(writer, models), Encoding.UTF8.GetString(ms.ToArray())); } [Fact] - public async Task WriteManyAsync_StreamWriter_EmptyCollection_WritesNothing() + public async Task WriteManyAsync_EmptyCollection_WritesNothing() { - var writer = new FixedWidthWriter(); - await using var ms = new MemoryStream(); - await using (var sw = new StreamWriter(ms, leaveOpen: true)) - { - await writer.WriteManyAsync(sw, [], Inv).ConfigureAwait(true); - } + await new FixedWidthWriter().WriteManyAsync(ms, [], TestContext.Current.CancellationToken); Assert.Equal(0, ms.Length); } diff --git a/tests/FixedWidthParser.Tests/WriterEncodingTests.cs b/tests/FixedWidthParser.Tests/WriterEncodingTests.cs new file mode 100644 index 0000000..9fb16b5 --- /dev/null +++ b/tests/FixedWidthParser.Tests/WriterEncodingTests.cs @@ -0,0 +1,96 @@ +using System.Text; +using FixedWidthParser.Writers; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// The writer's encoding, byte-based truncation and line break. + public class WriterEncodingTests + { + [Fact] + public void Latin1_WritesOneBytePerCharacter() + { + var writer = new FixedWidthWriter(new FixedWidthWriterOptions { Encoding = Encoding.Latin1, NewLine = "\n" }); + using var ms = new MemoryStream(); + + writer.Write(ms, new NameAgeModel { Name = "José", Age = 7 }); + + Assert.Equal(Encoding.Latin1.GetBytes("José 7 \n"), ms.ToArray()); + } + + [Fact] + public void Utf8_MultiByteCharacters_CountAsBytes() + { + // "José" is 5 bytes in UTF-8, so one pad byte is left in the 6-byte column. + string line = WriteOne(new FixedWidthWriter(), new NameAgeModel { Name = "José", Age = 7 }); + + Assert.Equal("José 7 ", line); + } + + [Fact] + public void Utf8_Truncation_NeverSplitsACharacter() + { + // "ABCDEé" needs 7 bytes; the 6-byte column keeps "ABCDE" and pads, instead of half of 'é'. + string line = WriteOne(new FixedWidthWriter(), new NameAgeModel { Name = "ABCDEé", Age = 1 }); + + Assert.Equal("ABCDE 1 ", line); + } + + [Fact] + public void Utf8_RightAlignedTruncation_KeepsWholeTrailingCharacters() + { + // Right-aligned width 4 keeps the rightmost whole characters: "éF" is 3 bytes, "DéF" would be 4. + string line = WriteOne(new FixedWidthWriter(), new RightTruncateModel { Code = "ABCDéF" }); + + Assert.Equal("DéF", line); + } + + [Fact] + public void Generated_MatchesReflection_ForLatin1() + { + var context = CtxWith(encoding: Encoding.Latin1); + var writer = new FixedWidthWriter(new FixedWidthWriterOptions { Encoding = Encoding.Latin1 }); + Span generated = stackalloc byte[9]; + Span reflected = stackalloc byte[9]; + + Assert.True(FixedWidth.TryFormat(new GenNameAgeModel { Name = "Joséé", Age = 42 }, generated, context, out int g)); + Assert.True(writer.TryFormat(new NameAgeModel { Name = "Joséé", Age = 42 }, reflected, out int r)); + + Assert.Equal(9, g); + Assert.Equal(9, r); + Assert.True(generated.SequenceEqual(reflected)); + Assert.Equal("Joséé 42 ", Encoding.Latin1.GetString(generated)); + } + + [Fact] + public void NewLine_IsConfigurable() + { + var writer = new FixedWidthWriter(new FixedWidthWriterOptions { NewLine = "\r\n" }); + + Assert.Equal("ABC\r\nDEF\r\n", WriteMany(writer, new[] { new CodeModel { Code = "ABC" }, new CodeModel { Code = "DEF" } })); + } + + [Fact] + public void TryFormat_DestinationTooSmall_ReturnsFalse() + { + var writer = new FixedWidthWriter(); + + Assert.False(writer.TryFormat(new CodeModel { Code = "ABC" }, new byte[2], out int written)); + Assert.Equal(0, written); + } + + [Fact] + public void WriteMany_ManyRecords_SpansSeveralBatches() + { + var models = Enumerable.Range(0, 20_000).Select(i => new CodeModel { Code = (i % 1000).ToString("D3", Inv) }).ToArray(); + var writer = new FixedWidthWriter(new FixedWidthWriterOptions { NewLine = "\n" }); + + string fromSpan = WriteMany(writer, models.AsSpan()); + string fromEnumerable = WriteMany(writer, (IEnumerable)models); + + Assert.Equal(20_000 * 4, fromSpan.Length); + Assert.Equal(fromSpan, fromEnumerable); + Assert.EndsWith("999\n", fromSpan, StringComparison.Ordinal); + } + } +} From 21872ab3a23364e1fe75df751355e6e005c9ec3b Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Thu, 15 Oct 2026 14:38:06 -0300 Subject: [PATCH 08/85] ci: validate the package against the last release on every pull request Packing runs package validation, so a removed or changed public member fails the pull request instead of the release. --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9128d26..149ec48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,6 +56,12 @@ jobs: - name: Build (Release) run: dotnet build FixedWidthParser.slnx --configuration Release --no-restore -p:DeterministicSourcePaths=false + # Packing runs package validation, which fails on any binary break against the last release. + # No --no-build: the generator's analyzer ProjectReference trips NETSDK1085 otherwise (see release.yml). + - name: Validate package against the last release + if: matrix.os == 'ubuntu-latest' + run: dotnet pack src/FixedWidthParser/FixedWidthParser.csproj --configuration Release --no-restore --output ./package-validation + - name: Test (Release) + collect coverage run: >- dotnet test --solution FixedWidthParser.slnx From 84c11f1f62c1682e160686a17d5c5c5266c4c5a6 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Thu, 15 Oct 2026 14:38:07 -0300 Subject: [PATCH 09/85] docs: document the 2.0 API and split the changelog by release The README describes the byte-only reader and writer, the options, the errors, encodings and the versioning rules. The changelog separates what shipped in 1.1.0 and 1.1.1 from 2.0, with a migration table. CONTRIBUTING explains the public API tracking and package validation. --- CHANGELOG.md | 232 ++++++++++++++++++++++-------------------- CONTRIBUTING.md | 14 +++ README.md | 264 ++++++++++++++++-------------------------------- 3 files changed, 221 insertions(+), 289 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdfb095..2fd3bf8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,125 +5,132 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [2.0.0] + +Not released yet. + +2.0 reads and writes **bytes only**: one parser, one reader, one writer and one generated +`TryParse`/`TryFormat`, where 1.x had a `char` and a UTF-8 version of each. Column positions are +bytes; text columns are decoded with an ASCII-compatible encoding, UTF-8 by default. Settings travel +in options objects and a `FixedWidthContext`, so later versions can add to them without another break. +2.0 targets `net10.0` only. + +### Migrating from 1.x + +| 1.x | 2.0 | +|---|---| +| `FixedWidthReader` over a `TextReader`, `FixedWidthByteReader` | `FixedWidthReader` over a `Stream`, a file or a `PipeReader` | +| `FixedWidthUtf8.Read*`, `FixedWidth.Read*` over a `TextReader` | `FixedWidth.Read*` over a `Stream`, a file or a `PipeReader` | +| `new FixedWidthReader(provider, pool, bufferSize)` | `new FixedWidthReader(new FixedWidthReaderOptions { FormatProvider = …, StringPool = …, BufferSize = … })` | +| `FixedWidth.Read(stream, encoding, leaveOpen, provider, pool, bufferSize)` | `FixedWidth.Read(stream, leaveOpen, options)`; the encoding is `FixedWidthReaderOptions.Encoding` | +| `FixedWidthParser`, `Utf8FixedWidthParser` `.TryParse(line, provider, pool, out m)` | `FixedWidthParser.TryParse(ReadOnlySpan line, in FixedWidthContext context, out m)` | +| `FixedWidth.TryParse(chars, provider, pool, out m)`, `FixedWidthUtf8.TryParse(bytes, …)` | `FixedWidth.TryParse(bytes, in FixedWidthContext context, out m)` | +| `FixedWidth.TryFormat(in m, Span, provider, out n)` | `FixedWidth.TryFormat(in m, Span, in FixedWidthContext context, out n)` | +| `IFixedWidthModel` + `IUtf8FixedWidthModel` | `IFixedWidthModel`, whose generated members take bytes and a context | +| `IFixedWidthConverter` + `IUtf8FixedWidthConverter` with `IFormatProvider?` | `IFixedWidthConverter` over bytes, with `in FixedWidthContext` | +| `FixedWidthWriter.Write*(StreamWriter or Stream, …, provider)` | `new FixedWidthWriter(new FixedWidthWriterOptions { FormatProvider = …, Encoding = …, NewLine = … })`, then `Write*(Stream, …)` | +| `catch (FormatException)` on a bad line | still works; the exception is now `FixedWidthParseException` | + +A UTF-8 file with multi-byte text whose layout counts characters no longer lines up: positions are +bytes. Read files written in a single-byte encoding with that encoding (`Encoding.Latin1`, or +Windows-1252 after registering `CodePagesEncodingProvider`). ### Added -- `System.IO.Pipelines` integration on the UTF-8 byte path: `FixedWidthByteReader.ReadAsync(PipeReader)` - (reflection) and `FixedWidthUtf8.ReadAsync(PipeReader)` (source-generated) stream records straight - off a `PipeReader` via `await foreach`. Lines are sliced from the `ReadOnlySequence` and parsed in - place when contiguous, copied into a pooled scratch buffer only when they span segments. Intended for - sources that are already pipes (Kestrel request bodies, sockets, upstream pipeline stages); for plain - files/streams the existing `Stream` overloads remain the faster default (see `PipeReaderBenchmarks`). -- `FixedColumnAttribute.Converter`: a per-property custom converter (`IFixedWidthConverter` for the - `char` path, `IUtf8FixedWidthConverter` for the UTF-8 path — one type can implement both) that - takes priority over the built-in `ISpanParsable`/`IUtf8SpanParsable` fallback, for both parsing and - writing. Wired into reflection (parse + write, both element types) and the source generator (parse, - both element types), with build-time diagnostics `FWP008`/`FWP009` when the converter doesn't - implement the interface the column's type requires. -- Nullable value-type columns (`int?`, `decimal?`, `DateTime?`, …): a blank (trimmed-empty) column - parses to `null` without invoking the underlying parser/converter, and `null` writes as a blank - (padding-filled) column. Works across reflection and generated, char and UTF-8, and composes with - `FixedColumnAttribute.Converter` (the converter always targets the non-nullable `T`). -- **Source-generated writer**: `IFixedWidthModel` gained a static-abstract `TryFormat(in TSelf, - Span, IFormatProvider?, out int)`, implemented by the generator for every model declaring the - `char` marker — reflection-free, AOT-safe writing at parity with the existing generated `TryParse`. - Mirrors `FixedWidthWriter`'s semantics exactly: per-column alignment/padding/format/overflow - (resolved at compile time into a `ColumnFormatOptions` per column), nullable columns write blank when - `null`, and `FixedColumnAttribute.Converter` columns format through the same converter instance used - for parsing. Returns `false` only when the destination span is shorter than the line length; a column - that doesn't fit throws or truncates per its `Overflow`, same as reflection. New facade - `FixedWidth.TryFormat`. New diagnostic `FWP010` when a column's type is neither `string` nor - `ISpanFormattable` and has no converter. (UTF-8 byte writing — reflection or generated — remains out of - scope: no byte writer of either kind exists yet in this library.) -- `FixedColumnAttribute.TrimChar`: the character trimmed from the end of a column when **parsing** - (previously hardcoded `' '` everywhere — `Padding` was write-only). Threaded through reflection - (char/byte) and the source generator (char/byte), including the nullable "blank column is null" check - and the `double`/`float` fast path. On the UTF-8 byte path a non-ASCII `TrimChar` throws - `NotSupportedException` (reflection: at parser construction; generated: from the column's static - `__trim{i}` field, surfacing as `TypeInitializationException` on first use) rather than silently - trimming the wrong byte — mirrors the existing decimal-separator ASCII guard. Unconfigured columns - (the overwhelming majority) still emit/execute the exact space-trim call they did before; the new - argument is only added when `TrimChar` differs from the default. +- `FixedWidthReaderOptions`: `FormatProvider`, `Encoding`, `StringPool`, `BufferSize`, + `MaxLineLength` and `OnParseError`, taken by every read. `FixedWidthWriterOptions`: `FormatProvider`, + `Encoding` and `NewLine`. +- `FixedWidthContext`, passed by `in` reference to generated code and converters: the format provider, + the encoding and the string pool for one line. New settings arrive as new properties. +- `FixedWidthParseException`, a `FormatException` carrying `LineNumber`, `ColumnName`, `ColumnStart`, + `ColumnLength` and `RawValue`. A short line says how long it is and how long the layout needs it to + be. A line that parses pays nothing for this: a failed line is parsed again to find the column. +- `FixedWidthReaderOptions.OnParseError`: return `true` to skip a line that fails to parse. +- `FixedWidthReaderOptions.MaxLineLength` (default 1 MiB): a longer line throws + `InvalidDataException`. Before, a source without line breaks doubled the read buffer until memory ran + out, which a `PipeReader` over a request body made reachable from the network. +- Encodings other than UTF-8 for text columns, dates parsed with a format and types that parse only + from characters. A UTF-8 byte order mark is skipped only when the encoding is UTF-8. +- Columns of any `ISpanParsable` type without a UTF-8 parser, decoded first (they used to be + refused on the byte path). +- `FixedWidthWriter.TryFormat(in model, Span, out int)`. +- `FixedColumnAttribute.Converter`: a per-property `IFixedWidthConverter` that takes priority over the + built-in parsing and formatting, with diagnostic `FWP008` when it targets the wrong type. +- Nullable value-type columns (`int?`, `decimal?`, `DateTime?`, …): a blank column parses to `null` + without invoking the parser or converter, and `null` writes as a blank column. +- The source-generated writer: `IFixedWidthModel.TryFormat`, with the same alignment, padding, + format, overflow, nullable and converter handling as `FixedWidthWriter`, and diagnostic `FWP010` + when a column's type cannot be written. +- `FixedColumnAttribute.TrimChar` and `TrimMode`: what to strip from a column when parsing, and from + which side. +- `FixedColumnAttribute.Format` parses `DateTime`, `DateOnly`, `TimeOnly` and `DateTimeOffset` with + that exact format. +- Diagnostic `FWP012`: `Padding` or `TrimChar` is not ASCII. The runtime parser and writer throw + `InvalidOperationException` for the same models. +- Public API tracking (`PublicAPI.*.txt`) and package validation against the last release, run by CI on + every pull request. ### Changed -- File-backed readers (`FixedWidthReader`, `FixedWidthByteReader`, and their UTF-8/generated - counterparts) now open the underlying `FileStream` with `FileOptions.SequentialScan` and an internal - buffer of 1 byte instead of the configured `bufferSize`: the enumerator core already does its own - block buffering into a pooled buffer, so a matching `FileStream` buffer was pure double-buffering - (an extra memcpy of most of the file) and dropped the OS's sequential-read prefetch hint on Windows. - Async file paths keep `FileOptions.Asynchronous` alongside the new flag. The default `bufferSize` for - `FixedWidthReader` and `FixedWidthByteReader` also increases from 4096 to 65536 bytes, - which is a more appropriate default block size for a throughput-oriented parser. -- **Breaking:** the internal column formatters `StringColumnFormatter` and - `SpanFormattableColumnFormatter` are now `internal` (they were unintentionally - `public`). They are implementation details resolved by the writer; the public extension point is the - `IColumnFormatter` interface. The reader/enumerator strategy types remain `public` because - they appear in `GetAsyncEnumerator()` return types (a deliberate allocation-free design choice). +- **Breaking:** bytes only; see the table above. The column positions of a model are bytes. +- **Breaking:** targets `net10.0` only. .NET 8 reaches end of support on 10 November 2026. +- **Breaking:** a `null` format provider means the invariant culture for every column type, on parse and + write. 1.x used it for `double`/`float` only and fell back to `CurrentCulture` for the rest. +- A culture whose decimal separator is not one byte (U+066B, for example) now parses, where 1.1 threw + `NotSupportedException` on the byte path. +- File-backed readers open the `FileStream` with `FileOptions.SequentialScan` and no internal buffer of + their own, since the reader already buffers. The default buffer is 65,536 bytes. +- The writer batches records into one pooled buffer and writes bytes straight to the stream, instead of + going through a `StreamWriter`. +- The generator targets `netstandard2.0`, so Visual Studio's .NET Framework compiler host can load it. +- The library and generator build with warnings as errors. ### Removed -- **Breaking:** `ColumnParserRegistry` and `Utf8ColumnParserRegistry` (and the `ColumnValueParser`/ - `Utf8ColumnValueParser` delegates they used) are gone. They were process-wide mutable state with a - documented footgun (registration only took effect if it happened before a model's first parse) and no - write-side counterpart; `FixedColumnAttribute.Converter` replaces them with a per-column, type-checked - alternative that also covers writing. The built-in `double`/`float` fast path (csFastFloat) no longer - goes through a registry lookup — it's now a direct type check in the parser factories, with no - behavior change for consumers. +- **Breaking:** `FixedWidthUtf8`, `FixedWidthByteReader`, `Utf8FixedWidthParser`, + `IUtf8FixedWidthModel`, `IUtf8FixedWidthConverter`, `Utf8FixedWidthRuntime` and the `TextReader` + overloads. Diagnostics `FWP007` and `FWP009`, which only applied to the second path, are retired. +- **Breaking:** `ColumnParserRegistry` and `Utf8ColumnParserRegistry` (and their value-parser delegates): + process-wide mutable state that only took effect before a model's first parse. + `FixedColumnAttribute.Converter` replaces them. +- **Breaking:** types that were public without a public use: `ColumnParser`, `Utf8ColumnParser`, + `RefAction`, `RefGetter`, `IColumnFormatter`, `ColumnLayoutValidator`, `ISource`, `TextReaderSource`, `StreamSource`, + `ILineFormat` and the line-format structs. The enumerator and strategy types that stay public + (they appear in return types, so `foreach` allocates nothing) lost their element-type and source + type parameters. + +### Fixed +- Right-aligned `double`/`float` columns could fail to parse under the invariant fast path: csFastFloat + did not consistently count skipped leading whitespace. +- `double`/`float` columns with a thousands separator under a `'.'` culture (`"1,234.56"`) failed + instead of parsing. +- `double`/`float` columns under a `','` culture could silently return a truncated value: `"1.234,50"` + became `1.0`. The fast path is now trusted only for `'.'`, and only when it consumes the whole field. +- The decimal-separator cache thrashed when two format providers alternated. + +## [1.1.1] + +### Added +- Targets `net8.0` alongside `net10.0`. `ref struct` models need .NET 9 or later, so they are + `net10.0`-only. + +## [1.1.0] + +### Added +- `System.IO.Pipelines` integration on the UTF-8 byte path: `FixedWidthByteReader.ReadAsync(PipeReader)` + and `FixedWidthUtf8.ReadAsync(PipeReader)` stream records off a `PipeReader` via `await foreach`. + +### Changed +- **Breaking:** the column formatters `StringColumnFormatter` and + `SpanFormattableColumnFormatter` became `internal`; they were unintentionally + `public`. ### Fixed -- **Right-aligned (leading-space-padded) `double`/`float` columns could fail to parse under the - invariant `'.'` fast path** — the single most common numeric layout in fixed-width files. csFastFloat's - `characters_consumed` did not consistently include skipped leading whitespace across its `double`/`float` - overloads (verified empirically: `FastDoubleParser` counted it, `FastFloatParser` didn't, and vice versa - on the UTF-8 overloads), so the full-consumption check used to reject silent truncation could itself - reject a perfectly valid right-aligned value depending on the element type. Leading whitespace is now - trimmed before the fast path runs, on all four `char`/`byte` × `double`/`float` combinations. -- **`double`/`float` columns using a thousands separator under the invariant `'.'` decimal separator - failed to parse instead of accepting the value** (e.g. `"1,234.56"`) — csFastFloat's single - `decimal_separator` override can't express thousands separators, so the fast path's full-consumption - check failed even though the value is valid. A failed fast-path attempt now retries via the real - `NumberFormatInfo`-aware parser (which still rejects genuine garbage) instead of returning `false` - outright. -- **Breaking:** a `null` `IFormatProvider` now means `CultureInfo.InvariantCulture` for **every** column - type, on both parse and write. Previously `double`/`float` columns treated `null` as invariant while - every other `ISpanParsable`/`ISpanFormattable` column (`decimal`, `int`, `DateTime`, …) fell through to - the BCL's own default of `CurrentCulture` — so on a non-invariant machine, the same line could parse a - `double` column and a `decimal` column to different results under one `null`-provider call, and writing - with `null` (`CurrentCulture`) could produce output the same `null`-provider parse then rejected. Pass - an explicit `IFormatProvider` (e.g. `CultureInfo.CurrentCulture`) if culture-dependent parsing/formatting - is what you want; `null` is now deterministic regardless of the running thread's ambient culture. -- **`double`/`float` columns under a non-'.' decimal separator could silently return a truncated, - wrong value instead of failing.** csFastFloat's `decimal_separator` override does not fail on - trailing content it doesn't recognize — it just stops at the first unrecognized character and - reports success with whatever it parsed up to that point. Under `de-DE`/`pt-BR`-style cultures - (`.` groups thousands, `,` is the decimal separator), a field like `"1.234,50"` silently became - `1.0` instead of `1234.50` (or failing outright). The fast path is now only trusted for the - invariant `'.'` separator, and even then only after confirming the whole (trimmed) field was - consumed; any other separator falls back to real `NumberFormatInfo`-aware parsing - (`double.TryParse`/`float.TryParse` with the actual `IFormatProvider`, which validates the whole - input and understands thousands separators correctly). Applies to reflection and generated, char - and UTF-8 (the byte path transcodes the — always short — numeric field to a small char buffer - when it needs the non-fast-path parse). -- The decimal-separator cache was a single-entry memo that thrashed (recomputed on every call) when - two `IFormatProvider`s were used alternately in the same process (e.g. parsing files in different - locales). Replaced with a `ConditionalWeakTable` keyed by provider identity — no thrash, and no - unbounded growth if callers pass many distinct providers (entries are reclaimed with their provider). -- UTF-8 byte parser: a culture whose decimal separator is not a single ASCII character (e.g. the - Arabic decimal separator U+066B) previously had its separator silently truncated to the wrong - byte, mis-parsing `double`/`float` columns. Such cultures now throw a clear `NotSupportedException` - on the byte path; use the `char`-based parser for them. ASCII separators (`.`/`,`) are unaffected. -- Writer: a value whose `ISpanFormattable.TryFormat` never succeeded could overflow the formatter's - buffer-growth counter into a negative/huge `ArrayPool` rent. The grow loop is now bounded (1M chars) - and throws a clear `InvalidOperationException` naming the column instead. +- UTF-8 byte parser: a culture whose decimal separator is not a single ASCII character had its separator + silently truncated to the wrong byte; it now throws `NotSupportedException`. +- Writer: a value whose `ISpanFormattable.TryFormat` never succeeded could overflow the buffer-growth + counter into a negative rent; the loop is bounded and throws a clear error naming the column. ### Documentation -- Corrected the UTF-8 byte path docs: `string` columns **are** interned through a supplied - `StringPool` (identical to the `char` path); the previous "no pooling" note was inaccurate. -- Documented that empty lines are skipped (counted but not yielded) while a non-empty line shorter - than the declared layout is treated as malformed and throws. -- Reconciled the README's stated requirements with the actual multi-target: the package builds for - `net8.0` and `net10.0`, not ".NET 10" only. Added an explicit note (in Requirements and in the - `ref struct` Models section) that `ref struct` model support needs .NET 9+, since the `allows ref - struct` generic constraint it depends on doesn't exist on `net8.0`. +- `string` columns on the UTF-8 path are interned through a supplied `StringPool`, like the char path. +- Empty lines are skipped (counted but not yielded); a non-empty line shorter than the layout throws. ## [1.0.0] @@ -145,8 +152,9 @@ Initial release. - `ref struct` model support on the parser. - A bundled Roslyn source generator: models implementing `IFixedWidthModel` get a reflection-free `TryParse` generated at compile time, with diagnostics (FWP001-FWP007) for invalid - layouts and unsupported column types. Shipped inside the package as an analyzer — no extra package - required. + layouts and unsupported column types. -[Unreleased]: https://github.com/GabrielMarquezMatte/FixedWidthParser/compare/v1.0.0...HEAD +[2.0.0]: https://github.com/GabrielMarquezMatte/FixedWidthParser/compare/v1.1.1...HEAD +[1.1.1]: https://github.com/GabrielMarquezMatte/FixedWidthParser/compare/v1.1.0...v1.1.1 +[1.1.0]: https://github.com/GabrielMarquezMatte/FixedWidthParser/compare/v1.0.0...v1.1.0 [1.0.0]: https://github.com/GabrielMarquezMatte/FixedWidthParser/releases/tag/v1.0.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index eca683f..48a857a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,6 +50,20 @@ gets a benchmark comparison posted as a comment automatically. - This is a **performance-sensitive** library. Avoid introducing allocations on parse/write hot paths; if a change could affect performance, include benchmark numbers in the PR. +## Public API and compatibility + +- The shipped library records every public member in `src/FixedWidthParser/PublicAPI.Unshipped.txt` + (analyzer RS0016). A new public member fails the build until it is added there; the code fix in the + IDE writes the line for you. At a release, the unshipped entries move to `PublicAPI.Shipped.txt`. +- `dotnet pack` runs package validation against the last release on NuGet + (`PackageValidationBaselineVersion`), and CI packs on every pull request. A removed or changed + member fails the pack. Breaks belong in a major version: list them in `CompatibilitySuppressions.xml` + (generate it with `dotnet pack -p:ApiCompatGenerateSuppressionFile=true`) and in the changelog, then + delete the file once the baseline moves to that major version. +- A minor version never adds a member to `IFixedWidthModel` or `IFixedWidthConverter`; new + information reaches them as a new property on `FixedWidthContext`. +- `src/` builds with warnings as errors. + ## Pull request flow 1. Branch off `develop` (or `master` for hotfixes). diff --git a/README.md b/README.md index e1b339a..c6811cf 100644 --- a/README.md +++ b/README.md @@ -8,33 +8,32 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) [![Benchmarks](https://img.shields.io/badge/benchmarks-GitHub%20Pages-informational)](https://gabrielmarquezmatte.github.io/FixedWidthParser/dev/bench/) -A high-performance, low-allocation library for **parsing and writing fixed-width (flat) files** in .NET. Columns are declared with attributes, layouts are validated up front, and the hot paths work over spans so fixed-width records can be parsed, streamed and written without the usual per-line churn. +A high-performance, low-allocation library for **parsing and writing fixed-width (flat) files** in .NET. Columns are declared with attributes, layouts are validated up front, and lines are parsed straight from the file's bytes, without a string per line. The package includes both: -- a runtime parser/writer API for regular attribute-mapped models; and -- a bundled Roslyn source generator for reflection-free `TryParse` implementations, including a UTF-8 byte path. +- a runtime parser, reader and writer for regular attribute-mapped models; and +- a bundled Roslyn source generator for reflection-free `TryParse` and `TryFormat` implementations. + +Upgrading from 1.x? See [the 2.0 entry in the changelog](CHANGELOG.md#200). ## Features - **Attribute-driven** column mapping (`[FixedColumn(start, length)]`) on properties and public fields. -- **Runtime parsing** for single lines and lazy batch reading from a `TextReader`, `Stream` or file. -- **Source-generated parsing** for `partial` models implementing `IFixedWidthModel`. -- **UTF-8 byte parsing** via `Utf8FixedWidthParser`, `FixedWidthByteReader` and generated `IUtf8FixedWidthModel` models, avoiding `StreamReader` and UTF-16 transcoding for ASCII-style flat files. -- **Synchronous and asynchronous readers** (`IEnumerable` / `IAsyncEnumerable`) with struct enumerators on the synchronous path. -- **`System.IO.Pipelines` support** on the byte path: read records straight from a `PipeReader` (`FixedWidthByteReader.ReadAsync(PipeReader)` / `FixedWidthUtf8.ReadAsync(PipeReader)`). -- **Writing** for single records and batches, synchronous and asynchronous, with `StreamWriter` reuse and `ReadOnlySpan` overloads for zero-allocation output. +- **Byte positions.** Lines are read and written as bytes, with no transcoding to UTF-16. Text columns are decoded with the file's encoding: UTF-8 by default, or any ASCII-compatible encoding such as Latin-1 or Windows-1252. +- **Lazy reading** from a `Stream`, a file or a `System.IO.Pipelines` `PipeReader`, synchronous (`IEnumerable` with a struct enumerator) and asynchronous (`IAsyncEnumerable`). +- **Source-generated parsing and writing** for `partial` models implementing `IFixedWidthModel`. +- **Errors that name the column**: a `FixedWidthParseException` with the line number, column, position and raw text, and an `OnParseError` callback to skip bad lines. +- **Bounded memory on untrusted input**: `MaxLineLength` stops a source without line breaks from growing the buffer. +- **Writing** of single records and batches, synchronous and asynchronous, batched into one pooled buffer. - **Configurable formatting** per column: alignment, padding character, format string and explicit overflow policy. -- **Culture-aware** numeric parsing/formatting, including `double`/`float` via csFastFloat and generic `ISpanParsable` / `ISpanFormattable` support. -- **Layout validation** at construction or generation time: negative `Start`, non-positive `Length` and overlapping columns fail clearly. -- **`ref struct` model support** on parser/source-generated single-line parsing (.NET 9+ only, see Requirements). +- **Culture-aware** numbers and dates, including `double`/`float` via csFastFloat and any `IUtf8SpanParsable`/`ISpanParsable` type. +- **Layout validation** at construction or generation time: negative `Start`, non-positive `Length`, overlapping columns and non-ASCII padding fail clearly. +- **`ref struct` model support** on single-line parsing. ## Requirements -- `net8.0` or `net10.0` (the package multi-targets both). -- `ref struct` models (e.g. `RefPersonModel`-style spans-only types) require .NET 9 or later — the - `allows ref struct` generic constraint that makes them possible doesn't exist on `net8.0`, so that - one feature is compiled out there; everything else behaves identically on both targets. +- `net10.0`. Dependencies: [CommunityToolkit.HighPerformance](https://www.nuget.org/packages/CommunityToolkit.HighPerformance) (`StringPool`) and [csFastFloat](https://www.nuget.org/packages/csFastFloat) (fast `double`/`float` parsing). @@ -44,13 +43,7 @@ Dependencies: [CommunityToolkit.HighPerformance](https://www.nuget.org/packages/ dotnet add package FixedWidthParser.NET ``` -Or as a ``: - -```xml - -``` - -The package ships the Roslyn source generator bundled as an analyzer. Models that implement `IFixedWidthModel` or `IUtf8FixedWidthModel` get generated parsers automatically; no extra package or setup is required. +The package ships the Roslyn source generator bundled as an analyzer. Models that implement `IFixedWidthModel` get a generated parser and writer automatically; no extra package or setup is required. ## Defining a Model @@ -64,8 +57,6 @@ public readonly record struct Person public Person() { Name = string.Empty; - Age = 0; - Salary = 0.0; } [FixedColumn(0, 10)] public string Name { get; init; } @@ -74,17 +65,15 @@ public readonly record struct Person } ``` -`start` is the 0-based offset and `length` is the column width. +`start` is the 0-based offset and `length` is the column width, both **in bytes**. In UTF-8 a character such as `é` takes two bytes, so a layout that counts characters only matches a UTF-8 file whose text is ASCII; for accented text, read the file in the single-byte encoding it was written in (see [Encodings](#encodings)). -For source generation, make the model `partial` and implement one or both marker interfaces: +For source generation, make the model `partial` and implement the marker interface: ```csharp using FixedWidthParser; using FixedWidthParser.Attributes; -public readonly partial record struct GeneratedPerson : - IFixedWidthModel, - IUtf8FixedWidthModel +public readonly partial record struct GeneratedPerson : IFixedWidthModel { [FixedColumn(0, 10)] public string Name { get; init; } [FixedColumn(10, 5)] public int Age { get; init; } @@ -92,49 +81,12 @@ public readonly partial record struct GeneratedPerson : } ``` -The generator emits distinct `TryParse` overloads for `ReadOnlySpan` and `ReadOnlySpan` when both interfaces are present. - -## Parsing - -### Runtime Single-Line Parsing - -```csharp -using System.Globalization; -using FixedWidthParser.Parsers; - -var parser = new FixedWidthParser(); - -if (parser.TryParse("John Doe 30 60000.00 ", CultureInfo.InvariantCulture, stringPool: null, out var person)) -{ - // person.Name == "John Doe" - // person.Age == 30 - // person.Salary == 60000.0 -} -``` - -### Source-Generated Single-Line Parsing - -```csharp -using System.Globalization; -using FixedWidthParser; - -if (FixedWidth.TryParse( - "John Doe 30 60000.00 ", - CultureInfo.InvariantCulture, - stringPool: null, - out var person)) -{ - // reflection-free generated parser -} -``` - -### Reading Text Files +## Reading ```csharp -using System.Globalization; using FixedWidthParser.Readers; -var reader = new FixedWidthReader(CultureInfo.InvariantCulture); +var reader = new FixedWidthReader(); foreach (var person in reader.ReadFile("people.txt")) { @@ -142,20 +94,22 @@ foreach (var person in reader.ReadFile("people.txt")) } ``` -`Read(TextReader)` and `Read(Stream, encoding, leaveOpen)` are also available. Reading is lazy and reuses a pooled buffer; lines are sliced directly from the buffer, so the reader does not allocate a string per line. A malformed line throws a `FormatException` carrying the line number. +`Read(Stream, leaveOpen)` reads a stream once; `ReadFile` reopens the file on each enumeration. Reading is lazy and reuses a pooled buffer; each line is parsed in place, without a string per line. -Empty lines (including a trailing newline at end of file) are skipped: they are counted toward the line number but not yielded as records. A line that is non-empty but shorter than the declared layout is treated as malformed and throws. +Empty lines (including a trailing newline at end of file) are skipped: they are counted toward the line number but not yielded. Lines end at `\n`, with a preceding `\r` dropped, and a leading UTF-8 byte order mark is skipped. -The source-generated facade has matching overloads: +The source-generated facade has the same methods: ```csharp -foreach (var person in FixedWidth.ReadFile("people.txt", formatProvider: CultureInfo.InvariantCulture)) +using FixedWidthParser; + +foreach (var person in FixedWidth.ReadFile("people.txt")) { // generated TryParse for each line } ``` -### Async Reading +### Async and pipes ```csharp await foreach (var person in reader.ReadFileAsync("people.txt")) @@ -164,82 +118,61 @@ await foreach (var person in reader.ReadFileAsync("people.txt")) } ``` -`ReadAsync(TextReader)` and `ReadAsync(Stream, encoding, leaveOpen)` mirror the synchronous overloads; `ReadFileAsync` uses true async file I/O. Cancellation is honored via `WithCancellation`. +`ReadAsync(Stream, leaveOpen)` mirrors the synchronous overload, and cancellation is honored via `WithCancellation`. When the source is already a [`PipeReader`](https://learn.microsoft.com/dotnet/standard/io/pipelines) — a Kestrel request body, a socket, an upstream pipeline stage — `ReadAsync(PipeReader, leaveOpen)` parses straight off it and lets the pipe own buffering. For plain files and streams the `Stream` overloads are the faster default. -## UTF-8 Byte Parsing +### Options -For ASCII/single-byte fixed-width files, the UTF-8 APIs parse directly from bytes. This avoids `StreamReader`, avoids UTF-8 to UTF-16 transcoding, and keeps offsets measured in bytes. +Every read takes an optional `FixedWidthReaderOptions`: ```csharp -using System.Globalization; -using FixedWidthParser.Readers; - -var reader = new FixedWidthByteReader(CultureInfo.InvariantCulture); - -foreach (var person in reader.ReadFile("people.txt")) +var reader = new FixedWidthReader(new FixedWidthReaderOptions { - // parsed from raw UTF-8 bytes -} + FormatProvider = CultureInfo.GetCultureInfo("pt-BR"), + Encoding = Encoding.Latin1, + StringPool = new StringPool(), + BufferSize = 64 * 1024, + MaxLineLength = 4096, + OnParseError = ex => { log.Warning(ex.Message); return true; }, // skip the line +}); ``` -Generated UTF-8 models use `FixedWidthUtf8`: +| Option | Default | Meaning | +|---|---|---| +| `FormatProvider` | invariant culture | Numbers and dates. | +| `Encoding` | UTF-8 | Text columns. Must be ASCII-compatible. | +| `StringPool` | none | Interns repeated string values. | +| `BufferSize` | 65,536 | Bytes read from the source at a time. | +| `MaxLineLength` | 1,048,576 | Longest line accepted; a longer one throws `InvalidDataException`. | +| `OnParseError` | throw | Return `true` to skip a line that fails to parse. | -```csharp -using System.Globalization; -using FixedWidthParser; +### Errors -if (FixedWidthUtf8.TryParse( - "John Doe 30 60000.00 "u8, - CultureInfo.InvariantCulture, - stringPool: null, - out var person)) -{ - // generated byte parser -} +A line that fails to parse throws `FixedWidthParseException`, a `FormatException` that says where: -await using var stream = File.OpenRead("people.txt"); -await foreach (var person in FixedWidthUtf8.ReadAsync(stream, formatProvider: CultureInfo.InvariantCulture)) -{ - // async raw-byte streaming -} +```text +Line 2 could not be parsed into Person: column 'Age' (start 10, length 5) holds "XX ". ``` -Column offsets on the UTF-8 path are byte offsets. That is ideal for the ASCII-style payloads common in flat files; with multi-byte UTF-8 characters, byte offsets and character offsets are not the same. +`LineNumber`, `ColumnName`, `ColumnStart`, `ColumnLength` and `RawValue` carry the same information. A line shorter than the layout reports how long it is and how long the layout needs it to be. The parser pays nothing for this on lines that parse: a failed line is parsed a second time to find the column. -The byte path supports the same `StringPool` interning as the char path: pass a pool to `FixedWidthByteReader` / the `FixedWidthUtf8` methods (the `stringPool` argument above) and string columns are interned through it; pass `null` to decode a fresh string per value. +### Untrusted input -### Reading from a PipeReader +A reader buffers a line until it finds its line break. `MaxLineLength` bounds that buffer, so a request body or file without line breaks throws `InvalidDataException` instead of growing the buffer until memory runs out. Set it close to your layout's line length when the source is untrusted. -When the source is already a [`System.IO.Pipelines.PipeReader`](https://learn.microsoft.com/dotnet/standard/io/pipelines) — a Kestrel request body, a socket, a named pipe, or an upstream pipeline stage — you can parse straight off it, letting the pipe own buffering and read-ahead. Both the reflection reader and the generated facade expose a `PipeReader` overload of `ReadAsync`: +## Single Lines ```csharp -using System.IO.Pipelines; -using System.Globalization; -using FixedWidthParser; -using FixedWidthParser.Readers; - -// Reflection reader: -var reader = new FixedWidthByteReader(CultureInfo.InvariantCulture); -await foreach (var person in reader.ReadAsync(pipeReader)) -{ - // parsed from the pipe -} +var parser = new FixedWidthParser(); +parser.TryParse("John Doe 30 60000.00 "u8, default, out var person); -// Source-generated facade: -await foreach (var person in FixedWidthUtf8.ReadAsync(pipeReader, formatProvider: CultureInfo.InvariantCulture)) -{ - // ... -} +FixedWidth.TryParse(lineBytes, new FixedWidthContext { FormatProvider = culture }, out var generated); ``` -Lines are sliced from the pipe's `ReadOnlySequence` and parsed in place when contiguous, copying into a pooled buffer only when a line spans segment boundaries. The same line semantics apply (BOM skipped once, `\n`/`\r\n`, empty lines skipped, trailing line without a newline yielded). By default the reader is completed when iteration ends; pass `leaveOpen: true` to leave it open. - -> **When to use it.** Prefer the `PipeReader` overload when you already hold a pipe. For plain files and streams the `Stream`/file overloads remain the faster default — a `PipeReader` adds per-read overhead that only pays off when there is real I/O to overlap (network, slow disk), not for in-memory or local-file sources. +`FixedWidthContext` carries the format provider, the encoding and the string pool for one line; `default` means invariant culture, UTF-8 and no pool. `FixedWidthReader.TryParse(line, out model)` parses one line with the reader's options. ## Writing ```csharp -using System.Globalization; using FixedWidthParser.Writers; var writer = new FixedWidthWriter(); @@ -250,10 +183,12 @@ var people = new[] }; using var stream = File.Create("out.txt"); -writer.WriteMany(stream, people.AsSpan(), CultureInfo.InvariantCulture); +writer.WriteMany(stream, people.AsSpan()); ``` -Overloads cover `Stream`/`StreamWriter` with `IEnumerable`/`ReadOnlySpan`, plus `WriteAsync` and `WriteManyAsync`. Reusing a `StreamWriter`, or passing a span, keeps writing allocation-free per line. +`Write`, `WriteMany` (span or enumerable) and their async versions write each record followed by `NewLine`, batched into one pooled buffer. `FixedWidthWriterOptions` sets the `FormatProvider`, the `Encoding` and the `NewLine` (default `Environment.NewLine`). `TryFormat(in model, Span, out int)` formats one line into your own buffer. + +Generated models format through `FixedWidth.TryFormat(in model, Span, in FixedWidthContext, out int)`. ## Formatting Options @@ -263,80 +198,56 @@ Each column can be tuned through named attribute arguments: [FixedColumn(0, 8, Alignment = Alignment.Right, Padding = '0')] public int Id { get; init; } // "00000042" [FixedColumn(8, 10, Format = "F2")] public double Amount { get; init; } // "1234.50 " [FixedColumn(18, 5, Overflow = OverflowBehavior.Truncate)] public string Code { get; init; } +[FixedColumn(23, 5, TrimChar = '0', TrimMode = TrimMode.Leading)] public int Count { get; init; } ``` - **`Alignment`**: `Left` (default) or `Right`. -- **`Padding`**: fill character (default space; for example `'0'` for zero-padding). -- **`Format`**: format string passed to `ISpanFormattable` (for example `"F2"` or `"N0"`); ignored for `string`. -- **`Overflow`**: `Default`, `Truncate` or `Throw`. `Default` resolves per type: strings truncate, numeric types throw. +- **`Padding`**: fill byte when writing (default space; for example `'0'` for zero-padding). Must be ASCII. +- **`TrimChar`** / **`TrimMode`**: what to strip when parsing, and from which side. Must be ASCII. +- **`Format`**: format string for `ISpanFormattable` when writing, and the exact format for `DateTime`, `DateOnly`, `TimeOnly` and `DateTimeOffset` when parsing. +- **`Overflow`**: `Default`, `Truncate` or `Throw`. `Default` resolves per type: strings truncate, everything else throws. A truncated string keeps whole characters, so a multi-byte character is never split. +- **`Converter`**: a type implementing `IFixedWidthConverter`, used instead of the built-in parsing and formatting. -## Culture Handling +## Column Types -Pass an `IFormatProvider` to `TryParse`, the reader constructor, the source-generated facade methods or the write methods. The generic path (`ISpanParsable`/`ISpanFormattable`) and the `double`/`float` processors honor it. When the provider is `null`, `'.'` is used as the decimal separator. +`string`, `double` and `float`, any `IUtf8SpanParsable` (parsed straight from bytes), any other `ISpanParsable` (decoded first), `T?` of any of them (a blank column is `null`), and anything with a `Converter`. Writing needs `string`, `ISpanFormattable` or a converter. -## StringPool +## Encodings -Pass a `CommunityToolkit.HighPerformance.Buffers.StringPool` to intern repeated string-column values: +Column positions are bytes, so the encoding only decides how text inside a column is read and written. UTF-8 files with ASCII text and single-byte encodings (Latin-1, Windows-1252, …) keep one byte per character. UTF-16 and EBCDIC are not supported. ```csharp -using System.Globalization; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Readers; - -var pool = new StringPool(); -var reader = new FixedWidthReader(CultureInfo.InvariantCulture, stringPool: pool); +var options = new FixedWidthReaderOptions { Encoding = Encoding.Latin1 }; +var records = FixedWidth.ReadFile("legacy.txt", options); ``` -This is a time-vs-memory trade-off: pooling removes repeated string allocations but costs extra CPU for hashing and lookup. Prefer it for GC-sensitive or high-concurrency workloads; skip it for raw throughput. - -Pooling applies to both the `char` and UTF-8 byte paths (`FixedWidthReader`/`FixedWidthByteReader` and the `FixedWidth`/`FixedWidthUtf8` facades), and to `ref struct` models. When no pool is supplied, each string column is decoded into a fresh string. +On .NET, Windows-1252 and other code pages need `Encoding.RegisterProvider(CodePagesEncodingProvider.Instance)` first. ## Validation -Invalid layouts fail fast with an `InvalidOperationException` on the runtime parser/writer paths, or generator diagnostics on generated models. Negative `Start`, `Length < 1`, and overlapping columns are rejected. Adjacent columns are valid. - -## `ref struct` Models - -> Requires `net10.0` (or any .NET 9+ target) — the `allows ref struct` constraint doesn't exist on `net8.0`, so this feature is compiled out there. - -The parser accepts `ref struct` models (`where TModel : new(), allows ref struct`), useful for stack-only row processing: - -```csharp -using FixedWidthParser.Attributes; -using FixedWidthParser.Parsers; - -public ref struct Row -{ - public Row() - { - Name = string.Empty; - Age = 0; - } +Invalid layouts fail fast with an `InvalidOperationException` on the runtime parser/writer, or a generator diagnostic on generated models: negative `Start`, `Length < 1`, overlapping columns, a non-ASCII `Padding` or `TrimChar` (FWP012), an unsupported column type and a converter for the wrong type. Adjacent columns are valid. - [FixedColumn(0, 10)] public string Name { get; set; } - [FixedColumn(10, 5)] public int Age { get; set; } -} +## Versioning -var parser = new FixedWidthParser(); -parser.TryParse(line, CultureInfo.InvariantCulture, null, out var row); -``` +The package follows [Semantic Versioning](https://semver.org/). A minor version may add types, members, options and enum values, but does not remove or change anything; every pull request checks the package against the last release. -Batch readers and the writer use regular generic constraints because `IEnumerable` cannot carry a `ref struct`. +- **Interfaces you implement** — `IFixedWidthModel` (normally through the generator) and `IFixedWidthConverter` — never gain a member in a minor version. New settings arrive as new properties on `FixedWidthContext`, which every call receives. +- **Types that appear in signatures only so enumeration stays allocation-free** — `IRecordLineParser`, the `*LineParser` strategy structs and the `*RecordEnumerable`/`*RecordEnumerator` types — are implemented only by the library. Program against `IEnumerable`/`IAsyncEnumerable`; their shape can change in a minor version. +- **`FixedWidthRuntime`** and **`ColumnFormatOptions`** are public because generated code calls them; they are not meant to be called directly. ## Performance -Measured with BenchmarkDotNet (`MemoryDiagnoser`) on .NET 10. Highlights: +Measured with BenchmarkDotNet (`MemoryDiagnoser`) on .NET 10 with 1.1, whose reader decoded text to UTF-16 first; the 2.0 numbers are yet to be measured. Highlights: - Parsing a line is allocation-light on the runtime path and reflection-free on the generated path. -- Text readers avoid allocating a string per line by slicing a reusable buffer. -- UTF-8 byte readers avoid `StreamReader` and transcoding for ASCII-style flat files. -- Writing with `StreamWriter` reuse or `ReadOnlySpan` is zero-alloc per line. +- Readers parse each line in place, without a string per line. +- Writing a span of records is zero-alloc per line. ### Comparison with other libraries #### Reading -Batched read of N fixed-width records (10-char string + 5-char int + 10-char double) from an in-memory string, compared against [FileHelpers](https://www.filehelpers.net/), [FlatFiles](https://github.com/jehugaleahsa/FlatFiles), and [RecordParser](https://github.com/leandromoh/RecordParser). +Batched read of N fixed-width records (10-char string + 5-char int + 10-char double), compared against [FileHelpers](https://www.filehelpers.net/), [FlatFiles](https://github.com/jehugaleahsa/FlatFiles), and [RecordParser](https://github.com/leandromoh/RecordParser). ``` BenchmarkDotNet v0.15.8, Windows 11 (10.0.26200.8655/25H2/2025Update/HudsonValley2) @@ -408,6 +319,5 @@ tests/Benchmarks/ BenchmarkDotNet benchmarks ```bash dotnet build FixedWidthParser.slnx -c Release -dotnet test tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj -dotnet test tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj +dotnet test --solution FixedWidthParser.slnx -c Release ``` From 4fce9bbddab5e493da1e08b35e1d966a71b7b8fa Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 15 Oct 2026 22:47:02 -0300 Subject: [PATCH 10/85] perf(writer): format numbers straight into the column and write plain doubles as scaled integers IUtf8SpanFormattable columns format directly into the line's slice and are aligned in place, with no char buffer, transcode or copy. The UTF-8 bytes are kept when the encoding is UTF-8 or they are ASCII; otherwise the value goes through the configured encoding as before. A double with no Format, at most four decimals and a magnitude under 1e9 is written as a scaled integer, byte-identical to double.TryFormat, for any culture whose decimal separator is one ASCII character and negative sign is "-". Left-aligned strings under UTF-8 transcode straight into the slice. --- CHANGELOG.md | 6 ++ .../FixedWidthModelGenerator.cs | 20 +++- src/FixedWidthParser/FixedWidthContext.cs | 40 ++++++- src/FixedWidthParser/FixedWidthRuntime.cs | 102 +++++++++++++++++- .../Formatters/ColumnFormatOptions.cs | 59 ++++++---- .../Formatters/ColumnFormatters.cs | 10 ++ src/FixedWidthParser/PublicAPI.Unshipped.txt | 1 + .../Writers/FixedWidthWriter.cs | 5 +- .../FixedWidthModelGeneratorTests.cs | 6 +- .../DoubleFormatTests.cs | 46 ++++++++ .../FixedWidthParser.Tests/GeneratedModels.cs | 6 ++ tests/FixedWidthParser.Tests/TestModels.cs | 6 ++ .../WriterEncodingTests.cs | 20 ++++ 13 files changed, 295 insertions(+), 32 deletions(-) create mode 100644 tests/FixedWidthParser.Tests/DoubleFormatTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fd3bf8..f5c6a67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,12 @@ Windows-1252 after registering `CodePagesEncodingProvider`). their own, since the reader already buffers. The default buffer is 65,536 bytes. - The writer batches records into one pooled buffer and writes bytes straight to the stream, instead of going through a `StreamWriter`. +- The writer formats `IUtf8SpanFormattable` columns (every BCL number and date) straight into the line, + and transcodes left-aligned string columns straight into it when the encoding is UTF-8. A formatted + value that is not ASCII under another encoding still goes through that encoding. +- A `double` column with no `Format`, at most four decimals and a magnitude under 1e9 is written as a + scaled integer, about 5x faster, with the same bytes. A line of a string, an `int` and a `double` + writes about 2x faster. - The generator targets `netstandard2.0`, so Visual Studio's .NET Framework compiler host can load it. - The library and generator build with warnings as errors. diff --git a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs index efb79db..df32b10 100644 --- a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs +++ b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs @@ -250,10 +250,17 @@ private static WriteKind ClassifyWrite(ITypeSymbol type, ITypeSymbol? converterT { return WriteKind.String; } - var formattable = compilation.GetTypeByMetadataName(SpanFormattableMetadataName); - return formattable is not null && type.AllInterfaces.Any(i => SymbolEqualityComparer.Default.Equals(i, formattable)) - ? WriteKind.Formattable - : WriteKind.Unsupported; + if (!Implements(type, compilation, SpanFormattableMetadataName)) + { + return WriteKind.Unsupported; + } + return Implements(type, compilation, "System.IUtf8SpanFormattable") ? WriteKind.Utf8Formattable : WriteKind.Formattable; + } + + private static bool Implements(ITypeSymbol type, Compilation compilation, string interfaceMetadataName) + { + var iface = compilation.GetTypeByMetadataName(interfaceMetadataName); + return iface is not null && type.AllInterfaces.Any(i => SymbolEqualityComparer.Default.Equals(i, iface)); } private static ITypeSymbol UnwrapNullable(ITypeSymbol type) @@ -634,6 +641,9 @@ private static void AppendColumnKindFormat(StringBuilder sb, string stmt, Column case WriteKind.Formattable: sb.Append(stmt).Append(RuntimeFqn).Append(".FormatValue(").Append(valueExpr).Append(", ").Append(slice).Append(", context").AppendLine(tail); break; + case WriteKind.Utf8Formattable: + sb.Append(stmt).Append(RuntimeFqn).Append(".FormatUtf8Value(").Append(valueExpr).Append(", ").Append(slice).Append(", context").AppendLine(tail); + break; case WriteKind.Converter: sb.Append(stmt).Append(RuntimeFqn).Append(".FormatConvert(").Append(valueExpr).Append(", ").Append(slice) .Append(", context, __converter").Append(i).AppendLine(tail); @@ -688,7 +698,7 @@ private static string TrimModeLiteral(int value) private enum ColumnKind { String, Double, Float, Utf8Parsable, DecodedParsable, Exact, Converter, BadConverter, Unsupported } - private enum WriteKind { String, Formattable, Converter, Unsupported } + private enum WriteKind { String, Formattable, Utf8Formattable, Converter, Unsupported } private readonly struct ColumnInfo( string name, int start, int length, ColumnKind kind, string typeFqn, string? converterFqn, bool isNullable, diff --git a/src/FixedWidthParser/FixedWidthContext.cs b/src/FixedWidthParser/FixedWidthContext.cs index 6965103..783ba13 100644 --- a/src/FixedWidthParser/FixedWidthContext.cs +++ b/src/FixedWidthParser/FixedWidthContext.cs @@ -17,13 +17,37 @@ namespace FixedWidthParser public readonly record struct FixedWidthContext { private readonly IFormatProvider? _formatProvider; - private readonly Encoding? _encoding; /// The format provider for numbers and dates. Defaults to . public IFormatProvider FormatProvider { get => _formatProvider ?? CultureInfo.InvariantCulture; - init => _formatProvider = value; + init + { + _formatProvider = value; + DecimalSeparator = PlainDecimalSeparator(value); + } + } + + /// + /// The provider's decimal separator as one ASCII byte when its negative sign is "-", so a plain + /// number can be written without going through the provider; 0 otherwise. + /// + // ponytail: read once, so a NumberFormatInfo mutated after this context was built is not seen. + internal byte DecimalSeparator { get => _formatProvider is null ? (byte)'.' : field; private init; } + + private static byte PlainDecimalSeparator(IFormatProvider? provider) + { + if (provider is null) + { + return (byte)'.'; + } + var info = NumberFormatInfo.GetInstance(provider); + if (!string.Equals(info.NegativeSign, "-", StringComparison.Ordinal) || info.NumberDecimalSeparator is not [var separator] || !char.IsAscii(separator)) + { + return 0; + } + return (byte)separator; } /// @@ -33,10 +57,18 @@ public IFormatProvider FormatProvider /// public Encoding Encoding { - get => _encoding ?? Encoding.UTF8; - init => _encoding = value; + get => field ?? Encoding.UTF8; + init + { + field = value; + IsUtf8 = value is null || value.CodePage == Encoding.UTF8.CodePage; + } } + /// True when is UTF-8, so UTF-8 bytes can be written as they are. + // The backing field stores the negation, so default(FixedWidthContext), which means UTF-8, reads as UTF-8. + internal bool IsUtf8 { get => !field; private init => field = !value; } + /// Interns string columns when set; otherwise each string column is a new string. public StringPool? StringPool { get; init; } diff --git a/src/FixedWidthParser/FixedWidthRuntime.cs b/src/FixedWidthParser/FixedWidthRuntime.cs index 51d07cd..7f79c11 100644 --- a/src/FixedWidthParser/FixedWidthRuntime.cs +++ b/src/FixedWidthParser/FixedWidthRuntime.cs @@ -1,8 +1,10 @@ using System.Buffers; +using System.Buffers.Text; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Runtime.CompilerServices; using System.Text; +using System.Text.Unicode; using FixedWidthParser.Attributes; using FixedWidthParser.Formatters; using FixedWidthParser.Processors; @@ -159,6 +161,19 @@ public static bool TryDateTimeOffsetExact(ReadOnlySpan column, string form /// public static void FormatString(string value, Span slice, in FixedWidthContext context, ColumnFormatOptions options, string columnName) { + // Left-aligned UTF-8 transcodes straight into the slice; FromUtf16 stops at a whole character, + // which is exactly the truncation. Overflow that throws falls through to report it. + // ponytail: lone surrogates become U+FFFD here even for a throwing UTF8Encoding. + if (context.IsUtf8 && options.Alignment == Alignment.Left) + { + var status = Utf8.FromUtf16(value, slice, out _, out int written); + if (status == OperationStatus.Done || options.Overflow == OverflowBehavior.Truncate) + { + slice[written..].Fill((byte)options.Padding); + return; + } + } + ReadOnlySpan text = value; var encoding = context.Encoding; if (options.Overflow == OverflowBehavior.Truncate && encoding.GetByteCount(text) > slice.Length) @@ -208,6 +223,89 @@ public static void FormatValue(TValue value, Span slice, in FixedW $"within {maxSize} characters (format \"{options.Format}\")."); } + /// + /// Formats an column straight into , with + /// no intermediate buffer or transcode. The UTF-8 result is kept when the encoding is UTF-8 or the + /// result is ASCII (numbers and dates, almost always); otherwise, and for a value wider than the + /// column, it falls back to , which re-encodes or reports the overflow. + /// + public static void FormatUtf8Value(TValue value, Span slice, in FixedWidthContext context, ColumnFormatOptions options, string columnName) + where TValue : IUtf8SpanFormattable, ISpanFormattable + { + if (TryFormatInto(value, slice, in context, options, out int written) && (context.IsUtf8 || Ascii.IsValid(slice[..written]))) + { + options.AlignInPlace(slice, written); + return; + } + FormatValue(value, slice, in context, options, columnName); + } + + private static bool TryFormatInto(TValue value, Span destination, in FixedWidthContext context, ColumnFormatOptions options, out int written) + where TValue : IUtf8SpanFormattable + { + if (typeof(TValue) == typeof(double) && options.Format is null) + { + return TryFormatDouble(Unsafe.As(ref value), destination, in context, out written); + } + return value.TryFormat(destination, out written, options.Format, context.FormatProvider); + } + + /// + /// Byte-identical to + /// with no format. A value with at most four fractional digits and a magnitude under 1e9 is written + /// as a scaled integer; the smallest scale whose quotient reproduces the value exactly is the + /// shortest round-trip form. + /// + internal static bool TryFormatDouble(double value, Span destination, in FixedWidthContext context, out int written) + { + byte separator = context.DecimalSeparator; + if (separator != 0 && value != 0 && Math.Abs(value) < 1e9) + { + ReadOnlySpan pow10 = [1, 10, 100, 1000, 10000]; + for (int scale = 0; scale < pow10.Length; scale++) + { + double scaled = Math.Round(value * pow10[scale]); + if (scaled / pow10[scale] == value) + { + return TryFormatScaled((long)scaled, scale, separator, destination, out written); + } + } + } + return value.TryFormat(destination, out written, default, context.FormatProvider); + } + + private static bool TryFormatScaled(long scaled, int scale, byte separator, Span destination, out int written) + { + written = 0; + if (scaled < 0) + { + if (destination.IsEmpty) + { + return false; + } + destination[written++] = (byte)'-'; + } + ulong magnitude = (ulong)Math.Abs(scaled); + ulong divisor = scale switch { 0 => 1, 1 => 10, 2 => 100, 3 => 1000, _ => 10000 }; + if (!Utf8Formatter.TryFormat(magnitude / divisor, destination[written..], out int n)) + { + return false; + } + written += n; + if (scale == 0) + { + return true; + } + if (destination.Length < written + 1 + scale) + { + return false; + } + destination[written++] = separator; + Utf8Formatter.TryFormat(magnitude % divisor, destination[written..], out n, new StandardFormat('D', (byte)scale)); + written += n; + return true; + } + /// /// Formats a column via a FixedColumnAttribute.Converter instance into /// per , with the same stack-then-ArrayPool growth as . @@ -262,7 +360,7 @@ internal static byte ToAsciiByte(char value, string columnName, string setting) [SkipLocalsInit] private static void WriteText(ReadOnlySpan text, Span slice, Encoding encoding, ColumnFormatOptions options, string columnName) { - bool ascii = System.Text.Ascii.IsValid(text); + bool ascii = Ascii.IsValid(text); int byteCount = ascii ? text.Length : encoding.GetByteCount(text); byte[]? rented = byteCount <= StackBufferSize ? null : ArrayPool.Shared.Rent(byteCount); Span bytes = rented is null ? stackalloc byte[StackBufferSize] : rented; @@ -284,7 +382,7 @@ private static void WriteText(ReadOnlySpan text, Span slice, Encodin private static int NarrowAscii(ReadOnlySpan text, Span bytes) { - System.Text.Ascii.FromUtf16(text, bytes, out int written); + Ascii.FromUtf16(text, bytes, out int written); return written; } diff --git a/src/FixedWidthParser/Formatters/ColumnFormatOptions.cs b/src/FixedWidthParser/Formatters/ColumnFormatOptions.cs index 7e0c36b..f7e02b4 100644 --- a/src/FixedWidthParser/Formatters/ColumnFormatOptions.cs +++ b/src/FixedWidthParser/Formatters/ColumnFormatOptions.cs @@ -23,26 +23,22 @@ public void WriteInto(ReadOnlySpan content, Span slice, string colum if (length <= width) { - if (Alignment == Alignment.Right) - { - int pad = width - length; - if (Padding != ' ' && content.Length > 0 && (content[0] == (byte)'-' || content[0] == (byte)'+')) - { - slice[0] = content[0]; - slice[1..(pad + 1)].Fill(padding); - content[1..].CopyTo(slice[(pad + 1)..]); - } - else - { - slice[..pad].Fill(padding); - content.CopyTo(slice[pad..]); - } - } - else + if (Alignment != Alignment.Right) { content.CopyTo(slice); slice[length..].Fill(padding); + return; } + int pad = width - length; + if (Padding != ' ' && content.Length > 0 && (content[0] == (byte)'-' || content[0] == (byte)'+')) + { + slice[0] = content[0]; + slice[1..(pad + 1)].Fill(padding); + content[1..].CopyTo(slice[(pad + 1)..]); + return; + } + slice[..pad].Fill(padding); + content.CopyTo(slice[pad..]); return; } @@ -56,11 +52,38 @@ public void WriteInto(ReadOnlySpan content, Span slice, string colum if (Alignment == Alignment.Right) { content[(length - width)..].CopyTo(slice); + return; + } + content[..width].CopyTo(slice); + } + + /// + /// Aligns and pads bytes already formatted at the start of + /// , exactly as would place them. + /// + internal void AlignInPlace(Span slice, int length) + { + byte padding = (byte)Padding; + int pad = slice.Length - length; + if (Alignment != Alignment.Right) + { + slice[length..].Fill(padding); + return; } - else + if (pad == 0) { - content[..width].CopyTo(slice); + return; + } + byte first = slice[0]; + slice[..length].CopyTo(slice[pad..]); + if (Padding != ' ' && length > 0 && (first == (byte)'-' || first == (byte)'+')) + { + // The sign stays leftmost, ahead of the padding. + slice[..(pad + 1)].Fill(padding); + slice[0] = first; + return; } + slice[..pad].Fill(padding); } } } diff --git a/src/FixedWidthParser/Formatters/ColumnFormatters.cs b/src/FixedWidthParser/Formatters/ColumnFormatters.cs index 2d4b37d..58bb9b1 100644 --- a/src/FixedWidthParser/Formatters/ColumnFormatters.cs +++ b/src/FixedWidthParser/Formatters/ColumnFormatters.cs @@ -30,6 +30,16 @@ public void Format(in TModel model, Span line, in FixedWidthContext contex } } + internal sealed class Utf8SpanFormattableColumnFormatter( + int start, int length, ColumnFormatOptions options, string columnName, RefGetter getter) + : IColumnFormatter where TProperty : IUtf8SpanFormattable, ISpanFormattable + { + public void Format(in TModel model, Span line, in FixedWidthContext context) + { + FixedWidthRuntime.FormatUtf8Value(getter(in model), line.Slice(start, length), in context, options, columnName); + } + } + internal sealed class ConverterColumnFormatter( int start, int length, ColumnFormatOptions options, string columnName, RefGetter getter, TConverter converter) : IColumnFormatter where TConverter : IFixedWidthConverter diff --git a/src/FixedWidthParser/PublicAPI.Unshipped.txt b/src/FixedWidthParser/PublicAPI.Unshipped.txt index fb87ce5..4524631 100644 --- a/src/FixedWidthParser/PublicAPI.Unshipped.txt +++ b/src/FixedWidthParser/PublicAPI.Unshipped.txt @@ -152,6 +152,7 @@ static FixedWidthParser.FixedWidthContext.operator !=(FixedWidthParser.FixedWidt static FixedWidthParser.FixedWidthContext.operator ==(FixedWidthParser.FixedWidthContext left, FixedWidthParser.FixedWidthContext right) -> bool static FixedWidthParser.FixedWidthRuntime.FormatConvert(TValue value, System.Span slice, in FixedWidthParser.FixedWidthContext context, TConverter converter, FixedWidthParser.Formatters.ColumnFormatOptions options, string! columnName) -> void static FixedWidthParser.FixedWidthRuntime.FormatString(string! value, System.Span slice, in FixedWidthParser.FixedWidthContext context, FixedWidthParser.Formatters.ColumnFormatOptions options, string! columnName) -> void +static FixedWidthParser.FixedWidthRuntime.FormatUtf8Value(TValue value, System.Span slice, in FixedWidthParser.FixedWidthContext context, FixedWidthParser.Formatters.ColumnFormatOptions options, string! columnName) -> void static FixedWidthParser.FixedWidthRuntime.FormatValue(TValue value, System.Span slice, in FixedWidthParser.FixedWidthContext context, FixedWidthParser.Formatters.ColumnFormatOptions options, string! columnName) -> void static FixedWidthParser.FixedWidthRuntime.IsBlank(System.ReadOnlySpan column, byte trimChar) -> bool static FixedWidthParser.FixedWidthRuntime.String(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> string! diff --git a/src/FixedWidthParser/Writers/FixedWidthWriter.cs b/src/FixedWidthParser/Writers/FixedWidthWriter.cs index 4514b4a..4cf8eea 100644 --- a/src/FixedWidthParser/Writers/FixedWidthWriter.cs +++ b/src/FixedWidthParser/Writers/FixedWidthWriter.cs @@ -288,8 +288,11 @@ private static IColumnFormatter BuildValueFormatter( throw new InvalidOperationException( $"Column '{member.Name}' has type '{valueType}', which is not string or ISpanFormattable; set a Converter for it."); } + var formatterType = typeof(IUtf8SpanFormattable).IsAssignableFrom(valueType) + ? typeof(Utf8SpanFormattableColumnFormatter<,>) + : typeof(SpanFormattableColumnFormatter<,>); return (IColumnFormatter)Activator.CreateInstance( - typeof(SpanFormattableColumnFormatter<,>).MakeGenericType(typeof(TModel), valueType), + formatterType.MakeGenericType(typeof(TModel), valueType), attribute.Start, attribute.Length, options, member.Name, getter)!; } } diff --git a/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs b/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs index 8965600..f905aff 100644 --- a/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs +++ b/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs @@ -51,7 +51,7 @@ public void ValidModel_GeneratesTryFormat() Assert.Contains("global::System.Span destination, in global::FixedWidthParser.FixedWidthContext context, out int bytesWritten", generated, StringComparison.Ordinal); Assert.Contains("if (destination.Length < 8) { bytesWritten = 0; return false; }", generated, StringComparison.Ordinal); Assert.Contains("FixedWidthRuntime.FormatString(model.Code,", generated, StringComparison.Ordinal); - Assert.Contains("FixedWidthRuntime.FormatValue(model.Quantity,", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.FormatUtf8Value(model.Quantity,", generated, StringComparison.Ordinal); Assert.Contains("ColumnFormatOptions __options1", generated, StringComparison.Ordinal); } @@ -108,7 +108,9 @@ public void CharOnlyParsableColumn_ParsesThroughDecodedText() Assert.Empty(result.GeneratorDiagnostics); Assert.Empty(result.CompilationErrors); - Assert.Contains("FixedWidthRuntime.TryParseDecoded", Assert.Single(result.GeneratedSources), StringComparison.Ordinal); + var generated = Assert.Single(result.GeneratedSources); + Assert.Contains("FixedWidthRuntime.TryParseDecoded", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.FormatValue(model.Value,", generated, StringComparison.Ordinal); } [Fact] diff --git a/tests/FixedWidthParser.Tests/DoubleFormatTests.cs b/tests/FixedWidthParser.Tests/DoubleFormatTests.cs new file mode 100644 index 0000000..6222297 --- /dev/null +++ b/tests/FixedWidthParser.Tests/DoubleFormatTests.cs @@ -0,0 +1,46 @@ +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Text; +using FixedWidthParser.Attributes; +using FixedWidthParser.Formatters; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// The writer's short-cut for plain doubles must match byte for byte. + public class DoubleFormatTests + { + [Theory] + [InlineData("")] + [InlineData("pt-BR")] + [SuppressMessage("Security", "CA5394:Do not use insecure randomness", + Justification = "Needs a reproducible seeded PRNG for deterministic test content, not cryptographic randomness.")] + public void FormattedBytesMatchDoubleTryFormat(string cultureName) + { + var culture = CultureInfo.GetCultureInfo(cultureName); + var context = CtxWith(culture); + var options = new ColumnFormatOptions(Alignment.Right, ' ', null, OverflowBehavior.Throw); + Random rng = new(42); + List values = [0, -0.0, 1e9, 999_999_999.9999, -999_999_999.9999, 0.0001, 0.00001, 0.00015, 0.1 + 0.2, 1.0 / 3, 1e15, 2.5, -2.5, double.Epsilon, double.MaxValue, double.NaN, double.NegativeInfinity]; + foreach (double edge in values.ToArray()) + { + values.Add(Math.BitIncrement(edge)); + values.Add(Math.BitDecrement(edge)); + } + for (int i = 0; i < 100_000; i++) + { + long digits = rng.NextInt64(-2_000_000_000_000, 2_000_000_000_000) / (long)Math.Pow(10, rng.Next(0, 12)); + values.Add(digits / Math.Pow(10, rng.Next(0, 7))); + values.Add(rng.NextDouble() * Math.Pow(10, rng.Next(-8, 12))); + values.Add(BitConverter.Int64BitsToDouble(rng.NextInt64())); + } + byte[] actual = new byte[32]; + foreach (double value in values) + { + FixedWidthRuntime.FormatUtf8Value(value, actual, in context, options, "Value"); + string text = value.ToString(culture); + Assert.Equal(text.PadLeft(32 - Encoding.UTF8.GetByteCount(text) + text.Length), Encoding.UTF8.GetString(actual)); + } + } + } +} diff --git a/tests/FixedWidthParser.Tests/GeneratedModels.cs b/tests/FixedWidthParser.Tests/GeneratedModels.cs index e843452..e12126a 100644 --- a/tests/FixedWidthParser.Tests/GeneratedModels.cs +++ b/tests/FixedWidthParser.Tests/GeneratedModels.cs @@ -156,4 +156,10 @@ public partial struct GenProductModel : IFixedWidthModel [FixedColumn(0, 6)] public string Name { get; init; } [FixedColumn(6, 3)] public int Age { get; init; } } + + /// Currency column — mirrors . + public readonly partial record struct GenPriceModel : IFixedWidthModel + { + [FixedColumn(0, 8, Alignment = Alignment.Right, Format = "C")] public decimal Price { get; init; } + } } diff --git a/tests/FixedWidthParser.Tests/TestModels.cs b/tests/FixedWidthParser.Tests/TestModels.cs index 523731d..27bb584 100644 --- a/tests/FixedWidthParser.Tests/TestModels.cs +++ b/tests/FixedWidthParser.Tests/TestModels.cs @@ -425,4 +425,10 @@ public readonly record struct NameAgeModel [FixedColumn(0, 6)] public string Name { get; init; } [FixedColumn(6, 3)] public int Age { get; init; } } + + /// A currency column, whose symbol may be non-ASCII depending on the culture. + public readonly record struct PriceModel + { + [FixedColumn(0, 8, Alignment = Alignment.Right, Format = "C")] public decimal Price { get; init; } + } } \ No newline at end of file diff --git a/tests/FixedWidthParser.Tests/WriterEncodingTests.cs b/tests/FixedWidthParser.Tests/WriterEncodingTests.cs index 9fb16b5..a703c19 100644 --- a/tests/FixedWidthParser.Tests/WriterEncodingTests.cs +++ b/tests/FixedWidthParser.Tests/WriterEncodingTests.cs @@ -62,6 +62,26 @@ public void Generated_MatchesReflection_ForLatin1() Assert.Equal("Joséé 42 ", Encoding.Latin1.GetString(generated)); } + // Numbers format straight to UTF-8; a non-ASCII result must still be re-encoded for other encodings. + [Theory] + [InlineData("utf-8", " é5.00")] + [InlineData("iso-8859-1", " é5.00")] + public void NonAsciiFormattedValue_UsesTheConfiguredEncoding(string encodingName, string expected) + { + var encoding = Encoding.GetEncoding(encodingName); + var culture = (System.Globalization.CultureInfo)Inv.Clone(); + culture.NumberFormat.CurrencySymbol = "é"; + var writer = new FixedWidthWriter(new FixedWidthWriterOptions { Encoding = encoding, FormatProvider = culture }); + Span generated = stackalloc byte[8]; + Span reflected = stackalloc byte[8]; + + Assert.True(FixedWidth.TryFormat(new GenPriceModel { Price = 5m }, generated, CtxWith(culture, encoding: encoding), out _)); + Assert.True(writer.TryFormat(new PriceModel { Price = 5m }, reflected, out _)); + + Assert.Equal(encoding.GetBytes(expected), generated.ToArray()); + Assert.Equal(encoding.GetBytes(expected), reflected.ToArray()); + } + [Fact] public void NewLine_IsConfigurable() { From 759aafc9165b6b326267e4f492019c49326619d0 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 15 Oct 2026 22:47:02 -0300 Subject: [PATCH 11/85] refactor(generator): read one column in its own method Extract was over MA0051's 90-line limit. The per-member work moves to ExtractColumn, which returns null for a member that is not a mapped column. --- .../FixedWidthModelGenerator.cs | 157 +++++++++--------- 1 file changed, 83 insertions(+), 74 deletions(-) diff --git a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs index df32b10..ad8f30f 100644 --- a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs +++ b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs @@ -103,82 +103,10 @@ public void Initialize(IncrementalGeneratorInitializationContext context) var columns = ImmutableArray.CreateBuilder(); foreach (var member in symbol.GetMembers()) { - if (member.IsStatic) + if (ExtractColumn(member, columnAttribute, compilation) is { } column) { - continue; - } - - ITypeSymbol? memberType = member switch - { - IPropertySymbol { SetMethod: not null, DeclaredAccessibility: Accessibility.Public } p => p.Type, - IFieldSymbol { IsReadOnly: false, IsConst: false, DeclaredAccessibility: Accessibility.Public } f => f.Type, - _ => null - }; - if (memberType is null) - { - continue; + columns.Add(column); } - - var attribute = member.GetAttributes().FirstOrDefault(a => - SymbolEqualityComparer.Default.Equals(a.AttributeClass, columnAttribute)); - if (attribute is null || attribute.ConstructorArguments.Length < 2) - { - continue; - } - - int start = attribute.ConstructorArguments[0].Value is int s ? s : 0; - int length = attribute.ConstructorArguments[1].Value is int l ? l : 0; - - ITypeSymbol? converterType = null; - int alignment = 0; // Alignment.Left - char padding = ' '; - string? format = null; - int overflow = 0; // OverflowBehavior.Default - char trimChar = ' '; - int trimMode = 0; // TrimMode.Trailing - foreach (var namedArg in attribute.NamedArguments) - { - switch (namedArg.Key) - { - case "Converter" when namedArg.Value.Value is ITypeSymbol converterSymbol: - converterType = converterSymbol; - break; - case "Alignment" when namedArg.Value.Value is int a: - alignment = a; - break; - case "Padding" when namedArg.Value.Value is char p: - padding = p; - break; - case "Format": - format = namedArg.Value.Value as string; - break; - case "Overflow" when namedArg.Value.Value is int o: - overflow = o; - break; - case "TrimChar" when namedArg.Value.Value is char t: - trimChar = t; - break; - case "TrimMode" when namedArg.Value.Value is int tm: - trimMode = tm; - break; - } - } - - var underlying = UnwrapNullable(memberType); - bool isNullable = !SymbolEqualityComparer.Default.Equals(underlying, memberType); - string? converterFqn = converterType?.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); - var kind = Classify(underlying, converterType, format, compilation); - var writeKind = ClassifyWrite(underlying, converterType, compilation); - // Mirrors FixedWidthWriter: an explicit Overflow wins; otherwise string truncates and everything else throws. - int resolvedOverflow = overflow; - if (resolvedOverflow == 0) - { - resolvedOverflow = writeKind == WriteKind.String ? 1 : 2; - } - - columns.Add(new ColumnInfo( - member.Name, start, length, kind, underlying.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), converterFqn, - isNullable, writeKind, alignment, padding, format, resolvedOverflow, trimChar, trimMode)); } string? ns = symbol.ContainingNamespace.IsGlobalNamespace @@ -196,6 +124,87 @@ public void Initialize(IncrementalGeneratorInitializationContext context) declaration.Identifier.GetLocation()); } + // A settable public instance property or field carrying [FixedColumn]; null for any other member. + private static ColumnInfo? ExtractColumn(ISymbol member, INamedTypeSymbol columnAttribute, Compilation compilation) + { + if (member.IsStatic) + { + return null; + } + + ITypeSymbol? memberType = member switch + { + IPropertySymbol { SetMethod: not null, DeclaredAccessibility: Accessibility.Public } p => p.Type, + IFieldSymbol { IsReadOnly: false, IsConst: false, DeclaredAccessibility: Accessibility.Public } f => f.Type, + _ => null + }; + if (memberType is null) + { + return null; + } + + var attribute = member.GetAttributes().FirstOrDefault(a => + SymbolEqualityComparer.Default.Equals(a.AttributeClass, columnAttribute)); + if (attribute is null || attribute.ConstructorArguments.Length < 2) + { + return null; + } + + int start = attribute.ConstructorArguments[0].Value is int s ? s : 0; + int length = attribute.ConstructorArguments[1].Value is int l ? l : 0; + + ITypeSymbol? converterType = null; + int alignment = 0; // Alignment.Left + char padding = ' '; + string? format = null; + int overflow = 0; // OverflowBehavior.Default + char trimChar = ' '; + int trimMode = 0; // TrimMode.Trailing + foreach (var namedArg in attribute.NamedArguments) + { + switch (namedArg.Key) + { + case "Converter" when namedArg.Value.Value is ITypeSymbol converterSymbol: + converterType = converterSymbol; + break; + case "Alignment" when namedArg.Value.Value is int a: + alignment = a; + break; + case "Padding" when namedArg.Value.Value is char p: + padding = p; + break; + case "Format": + format = namedArg.Value.Value as string; + break; + case "Overflow" when namedArg.Value.Value is int o: + overflow = o; + break; + case "TrimChar" when namedArg.Value.Value is char t: + trimChar = t; + break; + case "TrimMode" when namedArg.Value.Value is int tm: + trimMode = tm; + break; + } + } + + var underlying = UnwrapNullable(memberType); + bool isNullable = !SymbolEqualityComparer.Default.Equals(underlying, memberType); + string? converterFqn = converterType?.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); + var kind = Classify(underlying, converterType, format, compilation); + var writeKind = ClassifyWrite(underlying, converterType, compilation); + // Mirrors FixedWidthWriter: an explicit Overflow wins; otherwise string truncates and everything else throws. + int resolvedOverflow = overflow; + if (resolvedOverflow == 0) + { + resolvedOverflow = writeKind == WriteKind.String ? 1 : 2; + } + + return new ColumnInfo( + member.Name, start, length, kind, underlying.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), converterFqn, + isNullable, writeKind, alignment, padding, format, resolvedOverflow, trimChar, trimMode); + } + // Implements IFixedWidthModel? private static bool ImplementsMarker(INamedTypeSymbol symbol, INamedTypeSymbol marker) { From 868d5280159f6c24b48c35243ff16906e07fbeb2 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 15 Oct 2026 22:47:03 -0300 Subject: [PATCH 12/85] build: align .editorconfig with the style guide and pin LF line endings Expression-bodied methods, constructors, operators and local functions and if without braces warn in the IDE; properties and their accessors are exempt (IDE0027 off). Private consts are PascalCase. MA0051 is enabled at 90 lines / 50 statements instead of suppressed. .gitattributes pins eol=lf so a blob's line endings never depend on who committed it. --- .editorconfig | 83 +++++++++++++++++++++++++++---------------- .gitattributes | 4 +++ Directory.Build.props | 2 +- 3 files changed, 57 insertions(+), 32 deletions(-) create mode 100644 .gitattributes diff --git a/.editorconfig b/.editorconfig index 87bc35d..0026f40 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,52 +1,57 @@ -# EditorConfig — https://editorconfig.org -# Top-most file; stops the search here. root = true -########################################## -# All files -########################################## [*] charset = utf-8 +# lf, not crlf: every tracked file is committed LF. Demanding crlf here would only hold on Windows, +# where core.autocrlf converts on checkout and hides the mismatch. See .gitattributes, which pins +# eol=lf so no contributor's autocrlf setting can change what lands in a blob. end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true indent_style = space indent_size = 4 -[*.{yml,yaml,json,md}] +[*.{yml,yaml,json,md,csproj,props,targets,slnx}] indent_size = 2 +# Two trailing spaces are a hard line break in Markdown. [*.md] -trim_trailing_whitespace = false # trailing spaces are significant (hard line breaks) +trim_trailing_whitespace = false -[*.{csproj,props,targets,slnx}] -indent_size = 2 - -########################################## -# C# files -########################################## [*.cs] -indent_size = 4 max_line_length = 120 -# --- Language conventions --------------------------------------------------- +# The code is block-scoped throughout; switching styles file by file would make every diff a reindent. csharp_style_namespace_declarations = block_scoped:suggestion csharp_using_directive_placement = outside_namespace:suggestion dotnet_sort_system_directives_first = true dotnet_separate_import_directive_groups = false -# var usage — prefer explicit types except when obvious from the right-hand side. csharp_style_var_for_built_in_types = false:suggestion csharp_style_var_when_type_is_apparent = true:suggestion csharp_style_var_elsewhere = false:suggestion -# Expression-bodied members. -csharp_style_expression_bodied_methods = false:suggestion +# STYLEGUIDE.md: "Every method ... must use a block body. Expression bodies (=>) are forbidden on +# methods, no matter how short." Properties are exempted here (project preference overrides the +# guide for the common one-line forwarding getter). +csharp_style_expression_bodied_methods = false:warning +csharp_style_expression_bodied_constructors = false:warning +csharp_style_expression_bodied_operators = false:warning +csharp_style_expression_bodied_local_functions = false:warning csharp_style_expression_bodied_properties = true:suggestion csharp_style_expression_bodied_accessors = false:suggestion +# Accessors (get/set/init) are part of the property exemption: either body form is fine, so IDE0027 +# reports nothing. +dotnet_diagnostic.IDE0027.severity = none + +# STYLEGUIDE.md: "Always use braces on `if`, even single-line bodies. The linter enforces this." +csharp_prefer_braces = true:warning + +# STYLEGUIDE.md ternary rule ("nested ternary" / "branches contain logic" forbidden) has no direct +# analyzer id; IDE0045/IDE0046 (convert if to/from ternary) would fight that rule, so both are off. +dotnet_diagnostic.IDE0045.severity = none +dotnet_diagnostic.IDE0046.severity = none -# Modern C# preferences. -csharp_prefer_braces = true:suggestion csharp_style_prefer_pattern_matching = true:suggestion csharp_style_prefer_switch_expression = true:suggestion csharp_style_throw_expression = true:suggestion @@ -54,19 +59,35 @@ csharp_prefer_simple_using_statement = true:suggestion csharp_style_prefer_range_operator = true:suggestion csharp_style_prefer_index_operator = true:suggestion -# --- Naming: private fields use _camelCase -------------------------------- -dotnet_naming_rule.private_fields_underscore.severity = suggestion -dotnet_naming_rule.private_fields_underscore.symbols = private_fields -dotnet_naming_rule.private_fields_underscore.style = underscore_prefix +dotnet_style_qualification_for_field = false:suggestion +dotnet_style_qualification_for_property = false:suggestion +dotnet_style_qualification_for_method = false:suggestion +dotnet_style_qualification_for_event = false:suggestion + +# Naming: private fields _camelCase, private consts PascalCase, everything else PascalCase/camelCase +# per STYLEGUIDE.md. +dotnet_naming_style.underscore_camel_case.capitalization = camel_case +dotnet_naming_style.underscore_camel_case.required_prefix = _ +dotnet_naming_style.pascal_case.capitalization = pascal_case + +# Consts first — more specific than the general private-field rule below, so it must win. Rule order +# matters: the first matching rule applies. +dotnet_naming_symbols.private_consts.applicable_kinds = field +dotnet_naming_symbols.private_consts.applicable_accessibilities = private +dotnet_naming_symbols.private_consts.required_modifiers = const + +dotnet_naming_rule.private_consts_pascal_case.symbols = private_consts +dotnet_naming_rule.private_consts_pascal_case.style = pascal_case +dotnet_naming_rule.private_consts_pascal_case.severity = warning dotnet_naming_symbols.private_fields.applicable_kinds = field dotnet_naming_symbols.private_fields.applicable_accessibilities = private +dotnet_naming_symbols.private_fields.required_modifiers = -dotnet_naming_style.underscore_prefix.required_prefix = _ -dotnet_naming_style.underscore_prefix.capitalization = camel_case +dotnet_naming_rule.private_fields_underscore_camel_case.symbols = private_fields +dotnet_naming_rule.private_fields_underscore_camel_case.style = underscore_camel_case +dotnet_naming_rule.private_fields_underscore_camel_case.severity = warning -# --- 'this.' qualification -------------------------------------------------- -dotnet_style_qualification_for_field = false:suggestion -dotnet_style_qualification_for_property = false:suggestion -dotnet_style_qualification_for_method = false:suggestion -dotnet_style_qualification_for_event = false:suggestion +dotnet_diagnostic.MA0051.severity = warning +MA0051.maximum_lines_per_method = 90 +MA0051.maximum_statements_per_method = 50 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d8175cc --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# Line endings are pinned here rather than left to each contributor's core.autocrlf, so what lands in +# a blob never depends on who committed it. Every tracked text file is already LF, and +# .editorconfig's end_of_line matches. +* text=auto eol=lf diff --git a/Directory.Build.props b/Directory.Build.props index edf140d..4c6b11c 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -7,7 +7,7 @@ 9999 $(NoWarn);CA1062;CA1008;1591;RS0016;RS0037;RS0026;S1244;SS002;SS008;SS019;MA0016;CA1819;S6964;NU1902;NU1901;IDE0008;IDE0046;IDE0058;CA1515;CA1002 - $(NoWarn);CA1848;CA3003;AD0001;S6960;MA0051;CA1873;MA0049;CA1724;SS017;MA0048 + $(NoWarn);CA1848;CA3003;AD0001;S6960;CA1873;MA0049;CA1724;SS017;MA0048 true HLQ005;HLQ012 From b6f25b2ff558fd6aa64aafe73fb05afd1951078d Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 15 Oct 2026 22:47:03 -0300 Subject: [PATCH 13/85] docs: add a style guide, an architecture map and the writer measurements STYLEGUIDE.md and ARCHITECTURE.md follow ExcelReader's, adapted to this library's twins (runtime and generated, stream/async/pipe readers) and its untrusted-input rules, and add the rule to avoid else. docs/ holds the measurements behind design decisions, starting with the writer. CONTRIBUTING and SECURITY are rewritten in the same shape, and a code of conduct is added. --- ARCHITECTURE.md | 144 +++++++++++++++ CODE_OF_CONDUCT.md | 48 +++++ CONTRIBUTING.md | 157 ++++++++-------- README.md | 4 + SECURITY.md | 39 ++-- STYLEGUIDE.md | 366 +++++++++++++++++++++++++++++++++++++ docs/README.md | 11 ++ docs/writer-performance.md | 69 +++++++ 8 files changed, 741 insertions(+), 97 deletions(-) create mode 100644 ARCHITECTURE.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 STYLEGUIDE.md create mode 100644 docs/README.md create mode 100644 docs/writer-performance.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..8f58256 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,144 @@ +# Architecture + +A map of how the codebase fits together, not a manual. Start here, then follow the file/type names +into the source: the code is expected to carry its own meaning through naming and flow, so read it +directly rather than looking for a commentary alongside it. + +What the code deliberately does *not* carry is the evidence behind a decision — benchmark numbers, +allocation counts, approaches that were tried and rejected. That lives in [`docs/`](docs/README.md), +and [`STYLEGUIDE.md`](STYLEGUIDE.md#comments) says why. + +## Two paths over one set of helpers + +Every model is parsed and written by one of two paths: + +| | Runtime (reflection-built) | Generated | +|---|---|---| +| Model | any class or struct with `[FixedColumn]` members | a `partial` type implementing `IFixedWidthModel` | +| One line | `FixedWidthParser.TryParse`, `FixedWidthWriter.TryFormat` | `FixedWidth.TryParse`/`TryFormat`, calling the generated statics | +| Many lines | `FixedWidthReader`, `FixedWidthWriter` | `FixedWidth.Read*`; writing goes through `TryFormat` | +| Built | once per type, in a static constructor, from compiled expression trees | at compile time by `FixedWidthModelGenerator` | + +Both paths call the same helpers in `FixedWidthRuntime` (`src/FixedWidthParser/FixedWidthRuntime.cs`) +for every column: trim, decode a string, parse a number, format a value, pad and align. That class is +public only because generated code lives in the consumer's assembly. Keeping the semantics in one place +is what makes the two paths agree; `SourceGenParityTests` checks that they do. + +Column positions are bytes throughout. Lines are never transcoded to UTF-16: a column is a +`ReadOnlySpan` slice of the line, and only a `string` column, or a type that parses from +characters alone, is decoded with the configured encoding. + +## Settings: options and the context + +Readers take a `FixedWidthReaderOptions` and the writer a `FixedWidthWriterOptions`. Both reduce to a +`FixedWidthContext` (`src/FixedWidthParser/FixedWidthContext.cs`), a `readonly record struct` passed +by `in` to every parse and format call, generated or not, and to every `IFixedWidthConverter`. + +The context is the extension point. `IFixedWidthModel` and `IFixedWidthConverter` are +implemented by users, so a minor version cannot add a member to them; a new setting arrives as a new +property on the context instead. The context also caches what the hot path would otherwise recompute +per column, such as whether the encoding is UTF-8 and the provider's decimal separator. + +## Building a runtime layout + +`ModelColumns.ForEachColumn` (`src/FixedWidthParser/Attributes/`) walks the `[FixedColumn]` properties, +then fields, and hands each to a visitor; `ColumnLayoutValidator` then rejects a negative start, a +non-positive length and overlapping columns. The parser and the writer share this walk. + +- **Parser.** `ParserBuilder` compiles a model factory and, per column, a setter delegate. + `ColumnParserFactory` wraps each setter with the parse it needs into one `ColumnParser` + closure. Its resolution order — converter, date/time with a `Format`, `string`, `double`/`float`, + `IUtf8SpanParsable`, then `ISpanParsable` over decoded text — is the order the generator's + `Classify` mirrors. +- **Writer.** `FixedWidthWriter` compiles a getter per column and wraps it in an + `IColumnFormatter` (`Formatters/ColumnFormatters.cs`): string, `IUtf8SpanFormattable`, + `ISpanFormattable`, converter, and a nullable wrapper over any of them. `ClassifyWrite` in the + generator mirrors that choice. + +The build runs once per closed generic type in a static constructor. A layout error is captured there +and rethrown from the instance constructor, so the caller sees an `InvalidOperationException` rather +than a `TypeInitializationException` on first use. + +`FixedWidthParser` accepts `ref struct` models (`allows ref struct`); the readers do not, since an +enumerator has to hold its current model in a field. + +## Reading + +``` +source -> line splitter -> IRecordLineParser -> model + (bytes in a (struct strategy: + pooled buffer) runtime or generated) +``` + +Three enumerators share one shape, each specialized by a `struct` parse strategy +(`IRecordLineParser`: `ReflectionLineParser` or `GeneratedLineParser`, in +`Readers/ILineParser.cs`). Because the strategy is a `struct` type argument, the JIT compiles one +enumerator per `(TModel, TParser)` pair and the parse call devirtualizes. + +| Source | Enumerator | Line splitting | +|---|---|---| +| `Stream`, sync | `RecordEnumerator` (a `struct`, so `foreach` allocates nothing) | `LineBufferState` | +| `Stream`, async | `AsyncRecordEnumerator` | `LineBufferState` | +| `PipeReader` | `PipeRecordEnumerator` | its own scan over `ReadOnlySequence` | + +`LineBufferState` (`Readers/LineBufferState.cs`) owns a pooled buffer: it finds the next `\n`, drops a +preceding `\r`, skips a UTF-8 byte order mark once, skips empty lines while counting them, and +compacts or grows the buffer when a line does not fit. Growth stops at `MaxLineLength`, which is what +bounds memory on a source with no line breaks. The pipe enumerator implements the same rules over +pipe segments, parsing a line in place when it sits in one segment and copying it to a pooled scratch +buffer only when it straddles two. The three must stay identical in behavior; see +[STYLEGUIDE.md § Twins](STYLEGUIDE.md#twins). + +The async enumerator keeps the line scan synchronous: `MoveNextAsync` returns a completed +`ValueTask` while the buffer still holds a line, and only `MoveNextSlowAsync` awaits a refill. + +A file path is reopened on each enumeration (`FixedWidthRecordEnumerable` and its async sibling); a +stream is read once and disposed unless the caller passed `leaveOpen`. + +### Errors + +A line that fails to parse costs nothing extra on the lines that succeed. On failure, `ParseErrors` +(`Readers/ParseFailure.cs`) parses the line a second time with a `ParseFailure` sink in the context; +the parser reports the rejecting column, or the short line, through +`FixedWidthContext.ReportFailedColumn`/`ReportShortLine`, which do nothing outside this diagnosis. The +result is a `FixedWidthParseException` naming the line, column, position and raw text, handed to +`OnParseError` to skip the line or thrown. + +## Writing + +`FixedWidthWriter` formats each record into a fixed-length slot: the line, then the encoded +`NewLine`. `WriteMany` and `WriteManyAsync` fill one pooled 64 KB batch with whole records and write it +to the stream when the next record would not fit, so a write allocates nothing per record. Each line is +filled with spaces first, then every column formats into its own slice. + +Per column, `FixedWidthRuntime` picks the cheapest route that keeps the bytes the encoding would +produce: + +- `IUtf8SpanFormattable` values (every BCL number and date) format straight into the column's slice + and are aligned in place. The UTF-8 bytes are kept when the encoding is UTF-8 or they are ASCII; + otherwise the value is formatted again as characters and encoded. +- A `double` with no `Format` is written as a scaled integer when it has at most four decimals, the + same bytes as the BCL's shortest round-trip form. Measurements are in + [writer-performance.md](docs/writer-performance.md). +- A left-aligned `string` under UTF-8 is transcoded straight into the slice; truncation stops at a whole + character. Other strings are encoded and placed by `ColumnFormatOptions.WriteInto`. +- Everything else formats as characters, is encoded, and is placed by `WriteInto`, which applies + alignment, padding (keeping a sign ahead of zero padding) and the overflow policy. + +## The source generator + +`src/FixedWidthParser.Generator/` targets `netstandard2.0` so the .NET Framework compiler host in Visual +Studio can load it. It is an incremental generator: `Extract` turns each candidate type into a +`ModelInfo` of plain, equatable values (`ColumnInfo`, `EquatableArray`), so an edit that does not +change a model's columns does not regenerate it. + +`Emit` validates the model — partial, top-level, a supported type for each column, layout, converter +type, overflow, ASCII padding — and reports diagnostics `FWP001`–`FWP012` instead of generating code +for an invalid model. Otherwise it writes `TryParse` and `TryFormat` as straight-line code: one +`FixedWidthRuntime` call per column, with every attribute setting emitted as a constant. + +## Further reading + +- [`STYLEGUIDE.md`](STYLEGUIDE.md) — code style, and the rules for untrusted input, pooled buffers and twins. +- [`CONTRIBUTING.md`](CONTRIBUTING.md) — build expectations and how to submit a change. +- [`SECURITY.md`](SECURITY.md) — supported versions and how to report a vulnerability. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..aa928b4 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,48 @@ +# Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our community a +harassment-free experience for everyone, regardless of age, body size, visible or invisible +disability, ethnicity, sex characteristics, gender identity and expression, level of experience, +education, socio-economic status, nationality, personal appearance, race, religion, or sexual +identity and orientation. + +## Our Standards + +Examples of behavior that contributes to a positive environment: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing for mistakes, and learning from the experience + +Examples of unacceptable behavior: + +- The use of sexualized language or imagery, and sexual attention or advances of any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information without explicit permission +- Other conduct which could reasonably be considered inappropriate in a professional setting + +## Enforcement Responsibilities + +Maintainers are responsible for clarifying and enforcing these standards, and will take appropriate +and fair corrective action in response to any behavior deemed inappropriate, threatening, offensive, +or harmful. + +## Scope + +This Code of Conduct applies within all project spaces (issues, pull requests, discussions) and when +an individual is officially representing the project in public spaces. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the +maintainers via GitHub's private reporting channel on this repository. All complaints will be +reviewed and investigated promptly and fairly. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), +version 2.1. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 48a857a..529d98f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,78 +1,87 @@ -# Contributing to FixedWidthParser - -Thanks for taking the time to contribute! This document covers how to build, test, -and submit changes. - -## Prerequisites - -- [.NET 10 SDK](https://dotnet.microsoft.com/download) (`net10.0`) -- Git - -## Building - -```bash -dotnet restore FixedWidthParser.slnx -dotnet build FixedWidthParser.slnx --configuration Release -``` - -## Running tests - -```bash -dotnet test FixedWidthParser.slnx --configuration Release -``` - -To collect coverage locally (same as CI): - -```bash -dotnet test FixedWidthParser.slnx --configuration Release --collect:"XPlat Code Coverage" -``` - -The Cobertura report is written under each test project's `TestResults/` folder. - -## Running benchmarks - -```bash -dotnet run --project tests/Benchmarks/Benchmarks.csproj --configuration Release -- --filter '*' -``` - -Filter to a subset with fully-qualified patterns, e.g. -`--filter 'Benchmarks.Perf.ParserBenchmarks.*'`. Results for `master` are published to -[GitHub Pages](https://gabrielmarquezmatte.github.io/FixedWidthParser/dev/bench/); every PR -gets a benchmark comparison posted as a comment automatically. - -## Coding standards - -- The repo enforces a large analyzer suite (SonarAnalyzer, Roslynator, Meziantou, - NetAnalyzers, and more) configured in `Directory.Build.props`, plus formatting rules in - `.editorconfig`. Please make sure the build is warning-clean before opening a PR. -- Match the surrounding style: file-scoped namespaces, `_camelCase` private fields, - span-based hot paths, and XML docs on public members. -- This is a **performance-sensitive** library. Avoid introducing allocations on parse/write - hot paths; if a change could affect performance, include benchmark numbers in the PR. - -## Public API and compatibility - -- The shipped library records every public member in `src/FixedWidthParser/PublicAPI.Unshipped.txt` - (analyzer RS0016). A new public member fails the build until it is added there; the code fix in the - IDE writes the line for you. At a release, the unshipped entries move to `PublicAPI.Shipped.txt`. -- `dotnet pack` runs package validation against the last release on NuGet - (`PackageValidationBaselineVersion`), and CI packs on every pull request. A removed or changed - member fails the pack. Breaks belong in a major version: list them in `CompatibilitySuppressions.xml` - (generate it with `dotnet pack -p:ApiCompatGenerateSuppressionFile=true`) and in the changelog, then - delete the file once the baseline moves to that major version. -- A minor version never adds a member to `IFixedWidthModel` or `IFixedWidthConverter`; new - information reaches them as a new property on `FixedWidthContext`. -- `src/` builds with warnings as errors. - -## Pull request flow - -1. Branch off `develop` (or `master` for hotfixes). -2. Make your change with tests covering it. New behavior without tests will be asked to add them. -3. Ensure `dotnet build` and `dotnet test` pass locally on Release. -4. Open the PR against `master`/`develop`; fill in the PR template. -5. CI (build/test on Linux, Windows, macOS), CodeQL, and the benchmark comparison must be green. +# Contributing + +Thanks for considering a contribution. This project takes small, focused pull requests over large +rewrites — see [ARCHITECTURE.md](ARCHITECTURE.md) for the shape of the codebase before diving in, and +[STYLEGUIDE.md](STYLEGUIDE.md) for the code style, which the analyzers only partly enforce. + +## Build expectations + +- **Warnings are errors in `src/`.** Both library projects set `TreatWarningsAsErrors`, under a + curated `AnalysisMode=All` analyzer set (NetAnalyzers, Meziantou, SonarAnalyzer, Roslynator and + others, in `Directory.Build.props`). A PR that doesn't build clean locally won't build clean in CI + either — run a full build before pushing: + + ```bash + dotnet build FixedWidthParser.slnx --configuration Release + ``` + +- **Public API changes require a `PublicAPI.Unshipped.txt` entry.** `Microsoft.CodeAnalysis.PublicApiAnalyzers` + is active and fails the build on any unrecorded public member. If you add, change, or remove anything + public, update `src/FixedWidthParser/PublicAPI.Unshipped.txt`; the IDE's code fix writes the line for + you. At a release, the unshipped entries move to `PublicAPI.Shipped.txt`. + +- **Minor versions don't break callers.** The policy is in the README's + [Versioning](README.md#versioning) section; in practice: + - `IFixedWidthModel` and `IFixedWidthConverter` are implemented by users, so a minor + version never adds a member to them. New information reaches them as a new property on + `FixedWidthContext`. + - New inputs go on the options objects or a new overload, never as a new optional parameter on an + existing method. + - `dotnet pack` runs package validation against `PackageValidationBaselineVersion` in + `src/FixedWidthParser/FixedWidthParser.csproj`, and CI runs it on every pull request. A break that + belongs in a major version is listed in `CompatibilitySuppressions.xml` (generate it with + `dotnet pack -p:ApiCompatGenerateSuppressionFile=true`) and in the changelog. After each release, + move the baseline to the version just published and delete the suppressions file. + +- **Tests are required for behavior changes.** Run the suite before opening a PR: + + ```bash + dotnet test --solution FixedWidthParser.slnx --configuration Release + ``` + + CI also collects coverage (`-- --coverage --coverage-output-format cobertura`) and uploads it to + Codecov from the Linux leg. + + A change to one path needs the same change, and a parity test, on its twin: the runtime and + generated paths, and the stream, async and pipe readers + ([STYLEGUIDE.md § Twins](STYLEGUIDE.md#twins)). Line-splitting code reads untrusted input; read + [STYLEGUIDE.md § Untrusted Input](STYLEGUIDE.md#untrusted-input) before touching it. + +- **Performance claims come with numbers.** This is a performance-sensitive library: avoid + allocations on the parse and write hot paths, and if a change could affect speed, run the benchmarks + and put the result in the PR. Findings that justify a design go in [`docs/`](docs/README.md), not in + code comments. + + ```bash + dotnet run --project tests/Benchmarks/Benchmarks.csproj --configuration Release -- --filter '*' + ``` + + Filter with fully-qualified patterns, e.g. `--filter 'Benchmarks.Perf.ParserBenchmarks.*'`. Results + for `master` are published to [GitHub Pages](https://gabrielmarquezmatte.github.io/FixedWidthParser/dev/bench/), + and every PR gets a benchmark comparison posted as a comment. + +## Commit messages + +Use [Conventional Commits](https://www.conventionalcommits.org/): `type(scope): summary`, imperative +mood, lowercase summary, no trailing period. Common types in this repo: `feat`, `fix`, `refactor`, +`perf`, `test`, `docs`, `chore`, `ci`, `build`. The scope is usually the area touched (`reader`, +`writer`, `parser`, `generator`). Example: `fix(reader): bound a pipe line by MaxLineLength`. + +## Pull requests + +- Branch off `develop` (or `master` for a hotfix), and open the PR against the same branch. +- One focused change per PR — don't batch unrelated fixes into one commit or one PR. +- If a change is user-visible (new API, behavior change, performance claim), add it to + [CHANGELOG.md](CHANGELOG.md) and mention it in the PR description. +- CI builds and tests on Linux, Windows, and macOS, and runs CodeQL and the benchmark comparison — a + change that only builds on one OS isn't ready to merge. ## Reporting bugs / requesting features Use the [issue templates](https://github.com/GabrielMarquezMatte/FixedWidthParser/issues/new/choose). -For security issues, **do not** open a public issue — see [SECURITY.md](SECURITY.md). +For suspected security vulnerabilities, do **not** open a public issue — see [SECURITY.md](SECURITY.md) +for the private reporting channel. + +## Code of conduct + +This project follows the [Code of Conduct](CODE_OF_CONDUCT.md). diff --git a/README.md b/README.md index c6811cf..bca0641 100644 --- a/README.md +++ b/README.md @@ -313,8 +313,12 @@ src/FixedWidthParser.Generator/ Roslyn source generator tests/FixedWidthParser.Tests/ Runtime, reader, writer and parity tests tests/FixedWidthParser.Generator.Tests/Source generator tests tests/Benchmarks/ BenchmarkDotNet benchmarks +docs/ Measurements behind design decisions ``` +[ARCHITECTURE.md](ARCHITECTURE.md) maps the code; [CONTRIBUTING.md](CONTRIBUTING.md) and +[STYLEGUIDE.md](STYLEGUIDE.md) cover how to change it. + ## Building and Testing ```bash diff --git a/SECURITY.md b/SECURITY.md index 4be3525..9745b3a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,31 +1,24 @@ # Security Policy -## Supported versions +## Supported Versions -Security fixes are applied to the latest released `1.x` version published on NuGet as -[`FixedWidthParser.NET`](https://www.nuget.org/packages/FixedWidthParser.NET). +Only the latest published release of [`FixedWidthParser.NET`](https://www.nuget.org/packages/FixedWidthParser.NET) +receives security fixes. Older releases are not patched retroactively; upgrade to the latest version +before reporting an issue. -| Version | Supported | -| ------- | ------------------ | -| 1.x | :white_check_mark: | -| < 1.0 | :x: | +## Reporting a Vulnerability -## Reporting a vulnerability +Please **do not** open a public GitHub issue for security vulnerabilities. -**Please do not report security vulnerabilities through public GitHub issues.** +Report vulnerabilities privately via +[GitHub Private Vulnerability Reporting](https://github.com/GabrielMarquezMatte/FixedWidthParser/security/advisories/new), +or by email to **gabrielandremarquez.matte@gmail.com**. -Instead, use one of the following private channels: +Include, where possible: +- A minimal reproduction: the model and the input that triggers it +- The reading or writing path involved (stream, async, `PipeReader`, generated or runtime) +- The impact you observed (crash, excessive memory/CPU, incorrect data, etc.) +- The affected version(s) -1. **Preferred:** GitHub's private vulnerability reporting — - [open a report](https://github.com/GabrielMarquezMatte/FixedWidthParser/security/advisories/new). -2. Email the maintainer at **gabrielandremarquez.matte@gmail.com** with the details. - -Please include: - -- a description of the vulnerability and its impact; -- steps to reproduce (a minimal model + input is ideal); -- affected version(s). - -You can expect an initial acknowledgement within **5 business days**. Once the issue is -confirmed, a fix and coordinated disclosure timeline will be agreed with you before any -public announcement. +You should receive an initial response within 5 business days. If the report is +confirmed, a fix will be prepared and released before any public disclosure. diff --git a/STYLEGUIDE.md b/STYLEGUIDE.md new file mode 100644 index 0000000..747ee43 --- /dev/null +++ b/STYLEGUIDE.md @@ -0,0 +1,366 @@ +# FixedWidthParser Style Guide + +Code style rules. For build/PR/test process, see [CONTRIBUTING.md](CONTRIBUTING.md); for the shape of +the codebase, see [ARCHITECTURE.md](ARCHITECTURE.md). + +## Priorities + +1. **Correctness** — the code must be right. No shortcuts that sacrifice safety or accuracy. +2. **Performance** — optimize where it matters, but never at the cost of readability. +3. **Readability** — code is read far more than it is written. + +--- + +## Ternaries + +Use a ternary only when the entire expression fits on one line and both branches are single values — no function calls, no method chains, no nesting. + +```csharp +// OK: two literal values +int resolvedOverflow = writeKind == WriteKind.String ? 1 : 2; + +// OK: one local each side +int byteCount = ascii ? text.Length : encodedLength; + +// NOT OK: nested ternary +string result = a ? b ? "x" : "y" : "z"; + +// NOT OK: branches contain logic +int value = condition ? ComputeA() : ComputeB(); + +// NOT OK: a long condition, even with plain values +byte sign = Padding != ' ' && length > 0 && (first == (byte)'-' || first == (byte)'+') ? first : padding; + +// Use an if that returns instead (see "Avoid else" below): +private static int Value(bool condition) +{ + if (condition) + { + return ComputeA(); + } + return ComputeB(); +} +``` + +When in doubt, use `if`. + +--- + +## Avoid `else` + +Whenever possible, do not write `else`. A branch that ends the method — `return`, `continue`, +`break`, `throw` — makes the `else` redundant, and the code after it reads as the main path. + +```csharp +// NOT OK: two branches at the same depth +if (Alignment != Alignment.Right) +{ + content.CopyTo(slice); + slice[length..].Fill(padding); +} +else +{ + slice[..pad].Fill(padding); + content.CopyTo(slice[pad..]); +} + +// OK: the special case returns, the other case follows unindented +if (Alignment != Alignment.Right) +{ + content.CopyTo(slice); + slice[length..].Fill(padding); + return; +} +slice[..pad].Fill(padding); +content.CopyTo(slice[pad..]); +``` + +When neither branch can end the method, extract the `if` into its own method that returns. Keep +`else` only when both branches must fall through to shared code that follows and extracting them +would split one idea across two methods. + +--- + +## Control Flow — Prefer Early Return + +Avoid wrapping the happy path in a condition. Guard and exit early; keep the main logic at the lowest indentation level. + +```csharp +// NOT OK: body is nested under the guard +internal void Compact() +{ + if (Start > 0) + { + int len = End - Start; + ... + } +} + +// OK: guard exits early +internal void Compact() +{ + if (Start <= 0) + { + return; + } + int len = End - Start; + ... +} +``` + +Always use braces on `if`, even single-line bodies. The linter enforces this. + +--- + +## Scope Nesting — Maximum 3 Levels + +Count every block that introduces indentation: `if`, `else`, `for`, `foreach`, `while`, `using`, `try`, `switch`. A function body is level 0. Nesting beyond 3 is a signal to extract a method. + +``` +level 0: function body +level 1: while (true) +level 2: if (ScanFrom < End) +level 3: if (rel >= 0) ← maximum allowed +level 4: if (...) ← extract this into a private method +``` + +When you hit level 4, stop and ask: what is this inner block doing? Name it and move it. + +--- + +## Method Bodies — Always Block Body + +Every method, constructor, operator and local function must use a block body with braces. Expression +bodies (`=>`) are forbidden on them, no matter how short. + +```csharp +// NOT OK +private static string Foo() => Bar(); + +// OK +private static string Foo() +{ + return Bar(); +} +``` + +Properties and their accessors are exempt: `public TModel Current => _current;` and +`get => field ?? Encoding.UTF8;` are both fine, as is a block body. `.editorconfig` encodes the same +split and turns IDE0027 off. + +This rule exists to keep diffs readable and the line count honest. A one-liner expression body hides complexity behind a single line and makes the 70-line rule easy to game. + +--- + +## Function Length — Maximum 70 Lines + +Count every non-blank, non-comment line inside the function body (opening and closing brace included). If a function exceeds 70 lines, split it. + +70 is the review target. The build enforces a looser backstop through MA0051 (90 lines / 50 statements, see `.editorconfig`), so a method between 71 and 90 lines compiles but still fails review. + +Do not game the limit by collapsing logic onto single lines. The limit exists to keep methods focused, not to reward line-golf. A method that packs three operations on one line to stay under 70 is worse than one that splits them across 71 readable lines. + +Common splits: + +- Read one column's attribute vs. assemble the model from its columns +- Find a line boundary vs. parse the line within it +- The rare growth or error path extracted from the caller + +--- + +## Naming + +- Private fields: `_camelCase` +- Private static methods: `PascalCase` +- Local variables: `camelCase` +- Prefer full words over abbreviations (`index` over `idx`, `source` over `src`) unless the abbreviation is idiomatic in the domain (`buf`, `len`, `pos`) + +### The runtime and generated paths are peers + +A model is parsed and written either by the runtime path (`FixedWidthParser`, `FixedWidthReader`, +`FixedWidthWriter`) or by the generated one (`FixedWidth.*` over `IFixedWidthModel`). A +capability on one gets the same name shape on the other: `FixedWidth.ReadFile` mirrors +`FixedWidthReader.ReadFile`, and the same options object configures both. A caller who learns one +path should be able to guess the other. When you add a member to one, grep the other and match, or add +the missing sibling in the same change. + +--- + +## Comments + +**The default is no comment.** Code is expected to explain itself through naming and execution flow. +Reaching for a comment is the signal to look at the code again first: if a reader cannot follow the +logic, that is a defect in the code, not a gap in its documentation. Rename the variable, extract the +method, flatten the branch. Fix it there, and the comment becomes unnecessary. + +A comment is justified only once that has been tried and the code still cannot carry the meaning — +because the reason lives outside the code and no amount of naming can bring it in: + +- A constraint imposed by a platform, a dependency, or the bytes of a file. +- An invariant a reader could silently break, where breaking it stays compiling and passes review. + +```csharp +// OK: a dependency's behavior the code cannot express +// csFastFloat does not fail on trailing content; it stops and returns the prefix it parsed. + +// NOT OK: describes what the code does +// Loop through each column and parse its value. + +// NOT OK: the code should have said this itself +// i is the index of the '\n' that ends the line +int i = ...; // -> name it lineFeedIndex and delete the comment +``` + +Never reference the task, the ticket, or the caller. + +### Findings do not belong in code + +Measurements, benchmark results, allocation counts, and the reasoning that picked one approach over +another are **not comments**. They go in [`docs/`](docs/README.md). + +This includes the whole class of "why it is this way" content: a throughput figure that justified a +buffer size, an allocation count that motivated pooling, a design that was prototyped and measured +and then rejected. Put it in a document, with the numbers, the machine, and the input. Then the +next person can re-run it and disagree with the evidence — which a comment asserting `// 5x faster` +never lets them do. + +What may stay in the code is the constraint itself, stated without its evidence: + +```csharp +// OK: the ceiling, not the benchmark that found it +// ponytail: at most four decimals; anything finer takes the BCL formatter. + +// NOT OK: a finding, and it rots the moment anything around it changes +// Measured 8 ns vs 46 ns for double.TryFormat, so the scaled path stays. +``` + +### A comment must stay true, or it is worse than no comment + +Comments carry maintenance cost. Two failure modes are specifically banned: + +- **Do not describe a state the code is no longer in.** When you refactor, reread the comments on what + you touched. +- **Do not cite a file that can disappear.** Never point a comment at a plan, a spec, or a section + marker as the authoritative explanation. Those are working artifacts: they get renamed, merged and + deleted, and the comment then sends a reader to nothing, which is worse than silence. A committed + document under [`docs/`](docs/README.md) is different in kind — it is permanent, and pointing at one is + how a finding stays out of the code. The test is whether the target is expected to still exist in a + year, not whether a link is involved. + +--- + +## Error Handling + +Throw at trust boundaries: constructor arguments, public API parameters, malformed external data. Use `ArgumentOutOfRangeException.ThrowIfNegative` and similar .NET 6+ throw helpers. Do not add defensive checks for conditions that are impossible given the internal invariants. + +Match the exception to whose fault it is: + +| Fault | Exception | +|---|---| +| A line that does not fit the layout | `FixedWidthParseException`, naming the line, column and raw text | +| A source that breaks a configured limit (`MaxLineLength`) | `InvalidDataException` | +| A model whose layout is invalid | `InvalidOperationException` from the runtime constructor, or a generator diagnostic | +| A value too wide for its column when writing | `InvalidOperationException` naming the column | +| A bad argument from the caller | `ArgumentException` and its subtypes | + +Never answer a malformed file with `ArgumentException`/`ArgumentOutOfRangeException`: that tells the +caller they made a mistake when the input is what is broken. If file bytes can reach a .NET throw +helper, you are missing a validation step upstream. + +The runtime path builds its layout once per type in a static constructor. A layout error there is +captured and rethrown from the instance constructor, never left to surface as a +`TypeInitializationException`. + +--- + +## Untrusted Input + +Files and request bodies are attacker-controlled. These rules are not optional in reader code. + +- **Bound every buffer a source can grow.** A source without line breaks must hit `MaxLineLength` + before it drives an allocation, on every reader: stream, async stream and `PipeReader`. +- **When you add a guard to one reader, apply it to every sibling reader.** The three readers split + lines independently; a limit enforced on two of them is a hole in the third. +- **Validate before you allocate, not before you use.** Renting a bigger buffer first and discovering the + line is too long afterwards still hands the attacker the allocation. +- **Never let a malformed value silently change semantics.** A column that does not parse fails the + line; it never becomes a default value. +- New or changed line-splitting code needs a test that feeds the malformed shape (no line break, a + line one byte over the limit, a `\r` at a buffer boundary). Assert the exception type and its line + number, not merely that something threw. + +--- + +## Pooled Buffers + +`ArrayPool.Shared` is used throughout the readers and the writer. Misuse here is a correctness bug, +not a performance nit. + +- Every `Rent` needs a matching `Return` on **every** exit path, including thrown exceptions — a + `try`/`finally` around the use. A dropped buffer silently shrinks the pool for the whole process. +- Never `Return` a buffer that anything still references, and never touch a buffer after returning it. +- Pooled arrays come back oversized and dirty. Bound reads by the logical length you tracked, never by + `array.Length`. +- Buffers that live beyond one call are returned in `Dispose`/`DisposeAsync`; guard against double-return, + since `Dispose` may be called twice. + +--- + +## Twins + +Three sets of code paths in this library must stay behaviorally identical, and nothing in the compiler +keeps them honest: + +- **Runtime and generated.** `FixedWidthParser`/`FixedWidthWriter` and the generator's + `TryParse`/`TryFormat` call the same `FixedWidthRuntime` helpers. Put shared behavior there, not in + either side. +- **Stream, async stream and pipe readers.** Line semantics — the byte order mark, `\r\n`, empty + lines, a last line without a break, `MaxLineLength` — must match across all three. +- **Sync and async writers.** `WriteMany` and `WriteManyAsync` batch the same way. + +So: + +- A behavior fix to one twin **must** be applied to the others in the same change. Read all of them + before you edit any. +- Behavior shared by twins belongs in a helper none of them duplicates — the split should cover only + the awaiting or the code generation, not the logic around it. +- Changes to a twinned path need a parity test asserting identical output, including at buffer + boundaries (a line straddling a refill or a pipe segment) — that is exactly where implementations drift. +- A synchronous method does not take a `CancellationToken`. + +--- + +## Tests and Benchmarks + +Harness code is held to the same standard as library code, because a broken harness reports success. + +- **Fail loudly, never silently.** A setup that never ran, an input that parsed to nothing — assert it. + The dangerous case is not the harness that crashes; it is the one that measures zero work and + publishes a number. +- **Benchmarks against another library must compare matched work, or say plainly that they do not.** + If our side writes bytes straight to the stream while the competitor goes through a `StreamWriter`, + say so next to the numbers. An unlabelled comparison that flatters us is a defect. +- **State the machine for every published number**, and never derive a ratio across results measured on + different hardware. Allocation figures are deterministic and comparable across machines; timings are + not. +- Assert real behavior — exact bytes, exact values, exception types and their line and column. A test + that calls a method and asserts no throw documents nothing. + +--- + +## Performance Notes + +These rules apply only when a function is on a measured hot path (splitting lines, parsing or +formatting a column): + +- Work on `ReadOnlySpan`; decode to `string` only for a `string` column. +- Use `ArrayPool.Shared` for buffers that live beyond one stack frame, `stackalloc` for small ones. +- Avoid LINQ in hot paths; use `IndexOf` on spans, which benefits from SIMD. +- Specialize generic code over `struct` strategies (`IRecordLineParser`) so the JIT devirtualizes, + rather than over interfaces or delegates. +- Split cold paths (growth, error diagnosis, throw helpers) into `[MethodImpl(MethodImplOptions.NoInlining)]` + methods so they do not bloat the hot caller's IL and cost it inlining. +- Optimize against a measurement, and record what it showed in [`docs/`](docs/README.md) — never in a comment. + An optimization with no number behind it is a guess that costs readability. + +Outside the hot path (building a layout, the static constructors), ordinary allocations and reflection are fine. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..7443047 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,11 @@ +# Documentation + +The evidence behind the code: measurements, and the designs they picked or rejected. Code comments state +a constraint; the numbers that justified it live here, so they can be re-run and disputed +([STYLEGUIDE.md § Comments](../STYLEGUIDE.md#findings-do-not-belong-in-code)). + +- [writer-performance.md](writer-performance.md) — where a written line spends its time, and the + formatting shortcuts that came out of it. + +Usage documentation is the [README](../README.md); the shape of the code is +[ARCHITECTURE.md](../ARCHITECTURE.md). diff --git a/docs/writer-performance.md b/docs/writer-performance.md new file mode 100644 index 0000000..cc9173b --- /dev/null +++ b/docs/writer-performance.md @@ -0,0 +1,69 @@ +# Writer performance + +## Machine and method + +AMD Ryzen 7 5700X (8C/16T), Windows 10 Pro 19045, `net10.0`, Release. + +Two harnesses: + +- BenchmarkDotNet, `--job short`, over 1,000 rows of the benchmark `SampleModel` (a 10-byte string, a + 5-byte `int`, a 10-byte `double`), `OperationsPerInvoke = 1000`, so every figure is per row. +- A Stopwatch console app for before/after: 2,000 warm-up rounds, then the best of five runs of 4,000 + rounds over the same 1,000 rows. Its absolute numbers run higher than BenchmarkDotNet's; only the + ratio between its two columns is meaningful. + +`Salary` is `1000.0 + i`, so every `double` is an integer value. + +## Where a row's time goes + +BenchmarkDotNet, before the shortcuts below: + +| Operation | ns/row | +|---|--:| +| whole line, runtime writer | 82.7 | +| whole line, generated model | 68.8 | +| `double` column (`FormatUtf8Value`) | 54.8 | +| `double.TryFormat` to UTF-8, alone | 46.1 | +| `double` as a scaled integer, alone | 8.1 | +| `int` column | 8.1 | +| `int.TryFormat` to UTF-8, alone | 1.7 | +| `string` column | 4.5 | + +The `double` column is two thirds of the line. Its cost is the shortest round-trip search, which .NET +runs the same way for `char` and `byte` output, so formatting straight to UTF-8 instead of UTF-16 and +transcoding saved nothing measurable on this model: the `WriterBenchmarks` suite moved within its own +noise. + +The runtime writer's extra ~14 ns over the generated model is one interface call and one compiled +getter delegate per column. Compiling the whole line into one delegate would remove it, but the +generated model already does, so it was not built. + +## The shortcuts + +**Scaled-integer `double`.** A value with at most four decimals and a magnitude under 1e9 is written as +`long` digits with the decimal separator inserted: the smallest scale whose quotient reproduces the +value exactly is the shortest round-trip form. It comes from ExcelReader's XLSX writer, which checks it +against `Utf8Formatter`; here `DoubleFormatTests` checks it against `double.ToString` over ~300,000 +values in the invariant culture and pt-BR. It applies only with no `Format` and a provider whose +decimal separator is one ASCII character and whose negative sign is `-`. + +**Formatting into the column.** `IUtf8SpanFormattable` values format straight into the line's slice +and are aligned there, instead of into a stack buffer that `WriteInto` then copies. The `int` column's +6 ns above `int.TryFormat` was that buffer and copy. + +## Result + +Stopwatch console app, per row: + +| Case | Before | After | +|---|--:|--:| +| runtime writer, `TryFormat` | 110.1 | 43.1 | +| generated model, invariant, `double` with two decimals | 101.3 | 42.6 | +| generated model, pt-BR, `double` with two decimals | 101.4 | 49.3 | +| runtime writer, `WriteMany` to `Stream.Null` | 116.0 | 57.5 | + +## Not done + +- `float` and `decimal` still take the BCL formatter. +- Formatting a batch in parallel: records have a fixed length, so record *i* starts at + `i * RecordLength` and batches split cleanly. Worth it only for very large writes; unmeasured. From a800a147f1318f032e965dec0fc9527e4272a162 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 15 Oct 2026 22:58:20 -0300 Subject: [PATCH 14/85] feat(reader): aggregate a file in parallel FixedWidthReader.AggregateFileAsync and FixedWidth.AggregateFileAsync cut a file into byte ranges, fold each on a worker thread into an IRecordAccumulator, and merge the accumulators in file order. A line belongs to the range its first byte falls in. Fixed-width lines have no quoting, so a worker that starts one byte before its range and skips the line it lands inside finds the exact boundary: every line is parsed exactly once. Line numbers in errors match a sequential read; they are counted only when a line fails. One range is the sequential read, so DegreeOfParallelism = 1 and small files run the same code. --- ARCHITECTURE.md | 20 ++ CHANGELOG.md | 5 + README.md | 32 +++ docs/README.md | 2 + docs/parallel-aggregate.md | 29 +++ src/FixedWidthParser/FixedWidth.cs | 15 ++ .../FixedWidthParallelOptions.cs | 39 ++++ src/FixedWidthParser/FixedWidthParser.csproj | 4 + src/FixedWidthParser/IRecordAccumulator.cs | 31 +++ src/FixedWidthParser/PublicAPI.Unshipped.txt | 9 + .../Readers/FixedWidthReader.cs | 18 ++ .../Readers/LineBufferState.cs | 21 +- .../Readers/ParallelAggregator.cs | 216 ++++++++++++++++++ .../ParallelAggregateTests.cs | 171 ++++++++++++++ 14 files changed, 611 insertions(+), 1 deletion(-) create mode 100644 docs/parallel-aggregate.md create mode 100644 src/FixedWidthParser/FixedWidthParallelOptions.cs create mode 100644 src/FixedWidthParser/IRecordAccumulator.cs create mode 100644 src/FixedWidthParser/Readers/ParallelAggregator.cs create mode 100644 tests/FixedWidthParser.Tests/ParallelAggregateTests.cs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 8f58256..10e15de 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -95,6 +95,26 @@ The async enumerator keeps the line scan synchronous: `MoveNextAsync` returns a A file path is reopened on each enumeration (`FixedWidthRecordEnumerable` and its async sibling); a stream is read once and disposed unless the caller passed `leaveOpen`. +### Parallel aggregation + +`ParallelAggregator` (`Readers/ParallelAggregator.cs`) backs `AggregateFileAsync` on both paths. It +opens the file once as a `SafeFileHandle`, cuts it into byte ranges (four per worker, 1 MB to 64 MB +each), and lets workers pull ranges from a shared counter. Each worker reads its range with +`RandomAccess.Read`, through its own `LineBufferState`, into its own accumulator; the caller merges the +accumulators in range order and throws the first failure in file order. + +A line belongs to the range its first byte falls in, the byte after the previous line break, so a +byte order mark belongs to line 1. A worker starts one byte before its range, skips the line it lands +inside, and stops once the next line would start past its range. Fixed-width lines have no quoting, +so unlike ExcelReader's CSV engine there is no guessed start to confirm or re-read: every line is +parsed exactly once. A single range is the sequential read, which is how `DegreeOfParallelism = 1` and +small files run, so there is no separate sequential twin. + +A worker does not know how many lines precede its range. It counts them only when a line fails — +line breaks from the start of the file up to the failing line, continued from there for later failures +in the same range — so errors carry the sequential line number and the happy path pays nothing. +Measurements are in [parallel-aggregate.md](docs/parallel-aggregate.md). + ### Errors A line that fails to parse costs nothing extra on the lines that succeed. On failure, `ParseErrors` diff --git a/CHANGELOG.md b/CHANGELOG.md index f5c6a67..8cd4678 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,11 @@ bytes. Read files written in a single-byte encoding with that encoding (`Encodin Windows-1252 after registering `CodePagesEncodingProvider`). ### Added +- Parallel aggregation: `FixedWidthReader.AggregateFileAsync` and `FixedWidth.AggregateFileAsync` + read a file in byte-range partitions on several threads and fold each into an + `IRecordAccumulator`, merged in file order. Every line is parsed exactly once, and + errors carry the same line number as a sequential read. `FixedWidthParallelOptions.DegreeOfParallelism` + sets the thread count. - `FixedWidthReaderOptions`: `FormatProvider`, `Encoding`, `StringPool`, `BufferSize`, `MaxLineLength` and `OnParseError`, taken by every read. `FixedWidthWriterOptions`: `FormatProvider`, `Encoding` and `NewLine`. diff --git a/README.md b/README.md index bca0641..608166c 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,38 @@ await foreach (var person in reader.ReadFileAsync("people.txt")) `ReadAsync(Stream, leaveOpen)` mirrors the synchronous overload, and cancellation is honored via `WithCancellation`. When the source is already a [`PipeReader`](https://learn.microsoft.com/dotnet/standard/io/pipelines) — a Kestrel request body, a socket, an upstream pipeline stage — `ReadAsync(PipeReader, leaveOpen)` parses straight off it and lets the pipe own buffering. For plain files and streams the `Stream` overloads are the faster default. +### Parallel aggregation + +To fold a large file into one result — totals, counts, groups — read it on several threads: + +```csharp +public sealed class Payroll : IRecordAccumulator +{ + public double Total { get; private set; } + + public void Add(in Person person) + { + Total += person.Salary; + } + + public void Merge(Payroll following) + { + Total += following.Total; + } +} + +var payroll = await reader.AggregateFileAsync("people.txt"); +// GeneratedPayroll: the same accumulator over GeneratedPerson. +var generated = await FixedWidth.AggregateFileAsync("people.txt"); +``` + +The file is cut into byte ranges, each folded into its own accumulator on a worker thread, and the +accumulators are merged left to right in file order. `Add` must touch only its own instance; `Merge` +runs on one thread. Every line is parsed exactly once, and a failing line reports the same line number +as a sequential read. `FixedWidthParallelOptions.DegreeOfParallelism` caps the threads (`0`, the +default, means one per processor; `1` reads the file in one pass). `OnParseError`, when set, runs on +the worker threads. + ### Options Every read takes an optional `FixedWidthReaderOptions`: diff --git a/docs/README.md b/docs/README.md index 7443047..62e633e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,6 +6,8 @@ a constraint; the numbers that justified it live here, so they can be re-run and - [writer-performance.md](writer-performance.md) — where a written line spends its time, and the formatting shortcuts that came out of it. +- [parallel-aggregate.md](parallel-aggregate.md) — how `AggregateFileAsync` scales with the number of + workers. Usage documentation is the [README](../README.md); the shape of the code is [ARCHITECTURE.md](../ARCHITECTURE.md). diff --git a/docs/parallel-aggregate.md b/docs/parallel-aggregate.md new file mode 100644 index 0000000..8478506 --- /dev/null +++ b/docs/parallel-aggregate.md @@ -0,0 +1,29 @@ +# Parallel aggregation + +## Machine and method + +AMD Ryzen 7 5700X (8C/16T), Windows 10 Pro 19045, `net10.0`, Release. + +A Stopwatch console app over a 104 MB file of 4,000,000 lines, each a 10-byte string, a 5-byte `int` +and a 10-byte `double`. The accumulator sums the `int` and the `double`. Each figure is the best of +five runs after one warm-up run, with the file in the OS cache; every run's total is checked against +the first. The sequential rows are a `foreach` over `ReadFile` into the same accumulator. + +## Result + +| Path | Sequential `foreach` | dop 1 | dop 2 | dop 4 | dop 8 | dop 16 | +|---|--:|--:|--:|--:|--:|--:| +| runtime | 246.1 ms | 260.9 | 136.1 | 83.2 | 54.2 | 49.7 | +| generated | 210.0 ms | 183.9 | 103.9 | 57.5 | 42.0 | 41.2 | + +Scaling is close to linear up to 4 workers and flattens past the 8 physical cores. Each line still +allocates its `string` column, so the GC and memory bandwidth are the likely ceiling; a model without +string columns, or with a `StringPool`, was not measured. + +`dop 1` is one range through the same worker code as the parallel case, and lands within a few +percent of the sequential enumerator on both paths. + +## Not measured + +- A cold file: every worker reads its own range, so on a spinning disk the access pattern is random. +- Files on a network share. diff --git a/src/FixedWidthParser/FixedWidth.cs b/src/FixedWidthParser/FixedWidth.cs index 337bddb..4205359 100644 --- a/src/FixedWidthParser/FixedWidth.cs +++ b/src/FixedWidthParser/FixedWidth.cs @@ -67,5 +67,20 @@ public static FixedWidthAsyncRecordEnumerable + /// Reads a file across several threads and folds every record into a , + /// one instance per partition, merged in file order. + /// + /// Carries the same contract as . + public static Task AggregateFileAsync( + string path, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken ct = default) + where TModel : IFixedWidthModel + where TAccumulator : IRecordAccumulator, new() + { + ArgumentException.ThrowIfNullOrEmpty(path); + return ParallelAggregator.RunAsync, TAccumulator>( + default, path, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), ct); + } } } diff --git a/src/FixedWidthParser/FixedWidthParallelOptions.cs b/src/FixedWidthParser/FixedWidthParallelOptions.cs new file mode 100644 index 0000000..09d59e3 --- /dev/null +++ b/src/FixedWidthParser/FixedWidthParallelOptions.cs @@ -0,0 +1,39 @@ +namespace FixedWidthParser +{ + /// Options for reading a file across several threads with AggregateFileAsync. + public sealed class FixedWidthParallelOptions + { + internal static readonly FixedWidthParallelOptions Default = new(); + + /// + /// The maximum number of reading threads. 0, the default, means + /// ; 1 reads the file as one partition. + /// + public int DegreeOfParallelism { get; init; } + + // Bytes per partition; 0 sizes partitions from the file length. Tests set it to cut every boundary. + internal int ChunkSize { get; init; } + + internal int Workers + { + get + { + if (DegreeOfParallelism == 0) + { + return Environment.ProcessorCount; + } + return DegreeOfParallelism; + } + } + + internal static FixedWidthParallelOptions Validate(FixedWidthParallelOptions? options) + { + options ??= Default; + if (options.DegreeOfParallelism < 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.DegreeOfParallelism, "DegreeOfParallelism must not be negative."); + } + return options; + } + } +} diff --git a/src/FixedWidthParser/FixedWidthParser.csproj b/src/FixedWidthParser/FixedWidthParser.csproj index 8e436d4..870a7cc 100644 --- a/src/FixedWidthParser/FixedWidthParser.csproj +++ b/src/FixedWidthParser/FixedWidthParser.csproj @@ -61,6 +61,10 @@ + + + + diff --git a/src/FixedWidthParser/IRecordAccumulator.cs b/src/FixedWidthParser/IRecordAccumulator.cs new file mode 100644 index 0000000..3f3f876 --- /dev/null +++ b/src/FixedWidthParser/IRecordAccumulator.cs @@ -0,0 +1,31 @@ +namespace FixedWidthParser +{ + /// + /// Folds the records of a file read by AggregateFileAsync into a result. Each partition of the + /// file gets its own instance. + /// + /// The implementing type. + /// The record type. + /// + /// + /// runs concurrently across instances on worker threads, and must mutate only the + /// instance it is called on. Every line is parsed and added exactly once, by the partition its first + /// byte falls in. + /// + /// + /// runs on a single thread, left to right in file order, folding each later + /// partition into the accumulator of everything before it. It may receive an instance that saw no + /// records. A file read as a single partition never calls it. + /// + /// + public interface IRecordAccumulator + where TSelf : IRecordAccumulator + { + /// Folds one record into this instance. + void Add(in TModel model); + + /// Folds the records of the partition that immediately follows this one into this instance. + /// The accumulator of the following partition. It is not used again afterwards. + void Merge(TSelf following); + } +} diff --git a/src/FixedWidthParser/PublicAPI.Unshipped.txt b/src/FixedWidthParser/PublicAPI.Unshipped.txt index 4524631..bc30011 100644 --- a/src/FixedWidthParser/PublicAPI.Unshipped.txt +++ b/src/FixedWidthParser/PublicAPI.Unshipped.txt @@ -172,3 +172,12 @@ static FixedWidthParser.Formatters.ColumnFormatOptions.operator ==(FixedWidthPar ~override FixedWidthParser.FixedWidthContext.ToString() -> string ~override FixedWidthParser.Formatters.ColumnFormatOptions.Equals(object obj) -> bool ~override FixedWidthParser.Formatters.ColumnFormatOptions.ToString() -> string +FixedWidthParser.FixedWidthParallelOptions +FixedWidthParser.FixedWidthParallelOptions.DegreeOfParallelism.get -> int +FixedWidthParser.FixedWidthParallelOptions.DegreeOfParallelism.init -> void +FixedWidthParser.FixedWidthParallelOptions.FixedWidthParallelOptions() -> void +FixedWidthParser.IRecordAccumulator +FixedWidthParser.IRecordAccumulator.Add(in TModel model) -> void +FixedWidthParser.IRecordAccumulator.Merge(TSelf following) -> void +FixedWidthParser.Readers.FixedWidthReader.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! diff --git a/src/FixedWidthParser/Readers/FixedWidthReader.cs b/src/FixedWidthParser/Readers/FixedWidthReader.cs index 366a263..9cecb33 100644 --- a/src/FixedWidthParser/Readers/FixedWidthReader.cs +++ b/src/FixedWidthParser/Readers/FixedWidthReader.cs @@ -67,5 +67,23 @@ public FixedWidthAsyncRecordEnumerable> Rea ArgumentException.ThrowIfNullOrEmpty(path); return new(new ReflectionLineParser(_parser), path, _options); } + + /// + /// Reads a file across several threads and folds every record into a , + /// one instance per partition, merged in file order. + /// + /// + /// The result equals one sequential pass when is + /// associative. Every line is parsed exactly once, and errors report the same line number as a + /// sequential read. runs on the worker threads, + /// possibly concurrently, and the first failure in file order is the one thrown. + /// + public Task AggregateFileAsync(string path, FixedWidthParallelOptions? parallel = null, CancellationToken ct = default) + where TAccumulator : IRecordAccumulator, new() + { + ArgumentException.ThrowIfNullOrEmpty(path); + return ParallelAggregator.RunAsync, TAccumulator>( + new ReflectionLineParser(_parser), path, _options, FixedWidthParallelOptions.Validate(parallel), ct); + } } } diff --git a/src/FixedWidthParser/Readers/LineBufferState.cs b/src/FixedWidthParser/Readers/LineBufferState.cs index f471a09..fa804ab 100644 --- a/src/FixedWidthParser/Readers/LineBufferState.cs +++ b/src/FixedWidthParser/Readers/LineBufferState.cs @@ -23,10 +23,23 @@ internal struct LineBufferState internal int ScanFrom; internal bool Eof; internal int LineNumber; + // Offset, from the first byte read, of the line TryGetLine last returned: the byte after the + // previous line break, so a skipped byte order mark belongs to the first line. + internal long LineStart; + private long _nextLineStart; + private long _compacted; private int _maxLineLength; private bool _preambleChecked; + /// Offset, from the first byte read, of the first byte no line has consumed yet. + internal readonly long Consumed => _compacted + Start; + internal void Rent(FixedWidthReaderOptions options) + { + Rent(options, options.SkipsByteOrderMark); + } + + internal void Rent(FixedWidthReaderOptions options, bool skipByteOrderMark) { Buffer = ArrayPool.Shared.Rent(options.BufferSize); Start = 0; @@ -34,8 +47,11 @@ internal void Rent(FixedWidthReaderOptions options) ScanFrom = 0; Eof = false; LineNumber = 0; + LineStart = 0; + _nextLineStart = 0; + _compacted = 0; _maxLineLength = options.MaxLineLength; - _preambleChecked = !options.SkipsByteOrderMark; + _preambleChecked = !skipByteOrderMark; } internal LineStatus TryGetLine(out ReadOnlySpan line) @@ -110,6 +126,8 @@ private bool TrySliceLine(int rawEnd, int nextStart, out ReadOnlySpan line } line = Buffer.AsSpan(Start, contentEnd - Start); + LineStart = _nextLineStart; + _nextLineStart = _compacted + nextStart; Start = nextStart; ScanFrom = nextStart; LineNumber++; @@ -131,6 +149,7 @@ internal void Compact() End = len; ScanFrom -= Start; + _compacted += Start; Start = 0; } diff --git a/src/FixedWidthParser/Readers/ParallelAggregator.cs b/src/FixedWidthParser/Readers/ParallelAggregator.cs new file mode 100644 index 0000000..d6bdb9a --- /dev/null +++ b/src/FixedWidthParser/Readers/ParallelAggregator.cs @@ -0,0 +1,216 @@ +using System.Buffers; +using System.Runtime.ExceptionServices; +using Microsoft.Win32.SafeHandles; + +namespace FixedWidthParser.Readers +{ + /// + /// Reads a file in byte-range partitions on several threads, folds each partition into its own + /// accumulator, and merges them in file order. A line belongs to the partition its first byte falls + /// in: a worker starts one byte before its range, skips the line it lands inside, and stops once the + /// next line would start past its range. Fixed-width lines have no quoting, so that boundary is exact + /// and every line is parsed exactly once. One partition is the sequential read. + /// + internal static class ParallelAggregator + { + private const int ChunksPerWorker = 4; + private const long MinChunkBytes = 1024 * 1024; + private const long MaxChunkBytes = 64 * 1024 * 1024; + private const int LinesPerCancellationCheck = 4096; + + internal static async Task RunAsync( + TParser parser, string path, FixedWidthReaderOptions options, FixedWidthParallelOptions parallel, CancellationToken ct) + where TParser : struct, IRecordLineParser + where TAccumulator : IRecordAccumulator, new() + { + using var handle = File.OpenHandle(path, FileMode.Open, FileAccess.Read, FileShare.Read); + var plan = new ChunkPlan(RandomAccess.GetLength(handle), parallel); + var results = new TAccumulator[plan.Count]; + var failures = new ExceptionDispatchInfo?[plan.Count]; + var workers = new Task[Math.Min(parallel.Workers, plan.Count)]; + foreach (ref var worker in workers.AsSpan()) + { + worker = Task.Run(() => Drain(parser, handle, plan, options, results, failures, ct), ct); + } + await Task.WhenAll(workers).ConfigureAwait(false); + + failures[0]?.Throw(); + var merged = results[0]; + for (int i = 1; i < plan.Count; i++) + { + failures[i]?.Throw(); + merged.Merge(results[i]); + } + return merged; + } + + private static void Drain( + TParser parser, SafeFileHandle handle, ChunkPlan plan, FixedWidthReaderOptions options, + TAccumulator[] results, ExceptionDispatchInfo?[] failures, CancellationToken ct) + where TParser : struct, IRecordLineParser + where TAccumulator : IRecordAccumulator, new() + { + while (plan.TryTakeNext(out int index)) + { + ct.ThrowIfCancellationRequested(); + try + { + results[index] = AggregateChunk(parser, handle, plan.Start(index), plan.End(index), options, ct); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + failures[index] = ExceptionDispatchInfo.Capture(ex); + } + } + } + + private static TAccumulator AggregateChunk( + TParser parser, SafeFileHandle handle, long start, long end, FixedWidthReaderOptions options, CancellationToken ct) + where TParser : struct, IRecordLineParser + where TAccumulator : IRecordAccumulator, new() + { + var accumulator = new TAccumulator(); + var context = options.Context; + var reader = new ChunkReader(handle, start == 0 ? 0 : start - 1, options.MaxLineLength); + var lines = default(LineBufferState); + lines.Rent(options, start == 0 && options.SkipsByteOrderMark); + long delivered = 0; + try + { + while (reader.Origin + lines.Consumed < end) + { + if (reader.NextLine(ref lines, out var line) == LineStatus.End) + { + break; + } + long lineStart = reader.Origin + lines.LineStart; + if (lineStart < start || lineStart >= end) + { + continue; + } + if (++delivered % LinesPerCancellationCheck == 0) + { + ct.ThrowIfCancellationRequested(); + } + if (parser.TryParse(line, in context, out var model)) + { + accumulator.Add(in model); + continue; + } + ParseErrors.HandleFailure(parser, line, reader.LineNumberAt(lineStart), in context, options.OnParseError); + } + } + finally + { + lines.Return(); + } + return accumulator; + } + + private sealed class ChunkPlan + { + private readonly long _length; + private readonly long _chunkSize; + private int _next = -1; + + internal ChunkPlan(long length, FixedWidthParallelOptions parallel) + { + _length = length; + _chunkSize = ChunkSize(length, parallel); + Count = (int)Math.Max(1, (length + _chunkSize - 1) / _chunkSize); + } + + internal int Count { get; } + + internal long Start(int index) + { + return index * _chunkSize; + } + + internal long End(int index) + { + return Math.Min(Start(index) + _chunkSize, _length); + } + + internal bool TryTakeNext(out int index) + { + index = Interlocked.Increment(ref _next); + return index < Count; + } + + private static long ChunkSize(long length, FixedWidthParallelOptions parallel) + { + if (parallel.ChunkSize > 0) + { + return parallel.ChunkSize; + } + if (parallel.Workers == 1) + { + return Math.Max(1, length); + } + return Math.Clamp(length / ((long)parallel.Workers * ChunksPerWorker), MinChunkBytes, MaxChunkBytes); + } + } + + /// Feeds one partition's bytes to a and numbers its lines on demand. + private sealed class ChunkReader(SafeFileHandle handle, long origin, int maxLineLength) + { + private long _readOffset = origin; + private long _countedTo; + private long _breaksCounted; + + internal long Origin { get; } = origin; + + internal LineStatus NextLine(ref LineBufferState lines, out ReadOnlySpan line) + { + try + { + while (true) + { + var status = lines.TryGetLine(out line); + if (status != LineStatus.NeedData) + { + return status; + } + lines.Compact(); + lines.GrowIfFull(); + int read = RandomAccess.Read(handle, lines.Buffer.AsSpan(lines.End), _readOffset); + _readOffset += read; + lines.Advance(read); + } + } + catch (InvalidDataException) + { + // LineBufferState numbers lines from the partition start; the file numbers them from its own. + throw ParseErrors.LineTooLong(LineNumberAt(Origin + lines.Consumed), maxLineLength); + } + } + + /// The 1-based line number of the line starting at , as a sequential read counts it. + // ponytail: counts line breaks from the start of the file on the first failure in a partition, + // then from the last count; only failing lines pay for it. + internal long LineNumberAt(long offset) + { + byte[] buffer = ArrayPool.Shared.Rent(64 * 1024); + try + { + while (_countedTo < offset) + { + int read = RandomAccess.Read(handle, buffer.AsSpan(0, (int)Math.Min(buffer.Length, offset - _countedTo)), _countedTo); + if (read == 0) + { + break; + } + _breaksCounted += buffer.AsSpan(0, read).Count((byte)'\n'); + _countedTo += read; + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + return _breaksCounted + 1; + } + } + } +} diff --git a/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs b/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs new file mode 100644 index 0000000..7c0b0aa --- /dev/null +++ b/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs @@ -0,0 +1,171 @@ +using System.Collections.Concurrent; +using System.Text; +using FixedWidthParser.Readers; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// Parallel aggregation must equal one sequential pass, whatever the partition size. + public sealed class ParallelAggregateTests : IDisposable + { + private readonly string _path = Path.GetTempFileName(); + + public void Dispose() + { + File.Delete(_path); + } + + public sealed class Ages : IRecordAccumulator + { + public List Values { get; } = []; + + public void Add(in NameAgeModel model) + { + Values.Add(model.Age); + } + + public void Merge(Ages following) + { + Values.AddRange(following.Values); + } + } + + public sealed class GenAges : IRecordAccumulator + { + public List Values { get; } = []; + + public void Add(in GenNameAgeModel model) + { + Values.Add(model.Age); + } + + public void Merge(GenAges following) + { + Values.AddRange(following.Values); + } + } + + // A BOM, CRLF and LF breaks, empty lines and no final line break: every boundary a partition can cut. + private static string Content(int lines, int badLine = 0) + { + var text = new StringBuilder(""); + for (int i = 1; i <= lines; i++) + { + string age = i == badLine ? "XX " : (i % 1000).ToString("D3", Inv); + text.Append("Name").Append(i % 10).Append(' ').Append(age); + if (i < lines) + { + text.Append(i % 3 == 0 ? "\r\n" : "\n"); + } + if (i % 7 == 0) + { + text.Append('\n'); + } + } + return text.ToString(); + } + + private List Sequential() + { + return [.. new FixedWidthReader().ReadFile(_path).Select(m => m.Age)]; + } + + [Theory] + [InlineData(1)] + [InlineData(7)] + [InlineData(10)] + [InlineData(64)] + [InlineData(4096)] + [InlineData(0)] + public async Task EveryPartitionSize_EqualsTheSequentialRead(int chunkSize) + { + File.WriteAllText(_path, Content(2000)); + var parallel = new FixedWidthParallelOptions { DegreeOfParallelism = 4, ChunkSize = chunkSize }; + + var runtime = await new FixedWidthReader().AggregateFileAsync(_path, parallel, TestContext.Current.CancellationToken); + var generated = await FixedWidth.AggregateFileAsync(_path, parallel: parallel, ct: TestContext.Current.CancellationToken); + + var expected = Sequential(); + Assert.Equal(2000, expected.Count); + Assert.Equal(expected, runtime.Values); + Assert.Equal(expected, generated.Values); + } + + [Fact] + public async Task EmptyFile_ReturnsAnEmptyAccumulator() + { + File.WriteAllBytes(_path, []); + + var result = await new FixedWidthReader().AggregateFileAsync(_path, ct: TestContext.Current.CancellationToken); + + Assert.Empty(result.Values); + } + + [Theory] + [InlineData(5)] + [InlineData(4096)] + public async Task ParseFailure_ReportsTheSequentialLineNumber(int chunkSize) + { + File.WriteAllText(_path, Content(2000, badLine: 1537)); + var expected = Assert.Throws(() => Sequential()); + var parallel = new FixedWidthParallelOptions { DegreeOfParallelism = 4, ChunkSize = chunkSize }; + + var actual = await Assert.ThrowsAsync( + () => new FixedWidthReader().AggregateFileAsync(_path, parallel, TestContext.Current.CancellationToken)); + + Assert.Equal(expected.LineNumber, actual.LineNumber); + Assert.Equal(expected.Message, actual.Message); + } + + [Fact] + public async Task OnParseError_SkipsEachBadLineOnce_WithItsLineNumber() + { + File.WriteAllText(_path, Content(2000, badLine: 1537) + "\nbad line\n" + Content(3, badLine: 2)[1..]); + var sequentialErrors = new List(); + var sequentialOptions = new FixedWidthReaderOptions { OnParseError = ex => { sequentialErrors.Add(ex.LineNumber); return true; } }; + var expected = new FixedWidthReader(sequentialOptions).ReadFile(_path).Select(m => m.Age).ToList(); + var parallelErrors = new ConcurrentBag(); + var parallelOptions = new FixedWidthReaderOptions { OnParseError = ex => { parallelErrors.Add(ex.LineNumber); return true; } }; + + var result = await new FixedWidthReader(parallelOptions).AggregateFileAsync( + _path, new FixedWidthParallelOptions { DegreeOfParallelism = 4, ChunkSize = 9 }, TestContext.Current.CancellationToken); + + Assert.Equal(3, sequentialErrors.Count); + Assert.Equal(expected, result.Values); + Assert.Equal(sequentialErrors, parallelErrors.Order()); + } + + [Fact] + public async Task LineTooLong_ReportsTheSequentialLineNumber() + { + File.WriteAllText(_path, Content(300) + "\n" + new string('9', 50) + "\n" + Content(3)[1..]); + var options = new FixedWidthReaderOptions { MaxLineLength = 20 }; + var expected = Assert.Throws(() => new FixedWidthReader(options).ReadFile(_path).ToList()); + + var actual = await Assert.ThrowsAsync(() => new FixedWidthReader(options).AggregateFileAsync( + _path, new FixedWidthParallelOptions { DegreeOfParallelism = 4, ChunkSize = 6 }, TestContext.Current.CancellationToken)); + + Assert.Equal(expected.Message, actual.Message); + } + + [Fact] + public async Task CancelledToken_Throws() + { + File.WriteAllText(_path, Content(10)); + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync( + () => new FixedWidthReader().AggregateFileAsync(_path, ct: cts.Token)); + } + + [Fact] + public async Task NegativeDegreeOfParallelism_Throws() + { + File.WriteAllText(_path, Content(10)); + + await Assert.ThrowsAsync(() => new FixedWidthReader().AggregateFileAsync( + _path, new FixedWidthParallelOptions { DegreeOfParallelism = -1 }, TestContext.Current.CancellationToken)); + } + } +} From a32b1d5b3f8f3574317c62135b2d2d013174f3ef Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 15 Oct 2026 23:14:04 -0300 Subject: [PATCH 15/85] feat(reader): aggregate a stream in parallel FixedWidthReader.AggregateAsync(Stream) and FixedWidth.AggregateAsync fold a stream across several threads. A stream cannot be read at an offset, so one producer reads it into pooled 256 KB chunks cut after the last line break and hands them through a bounded queue to the workers; the accumulators are merged in chunk order. The producer counts the line breaks it hands over, so every chunk knows the number of its first line and errors carry the sequential line number. A producer failure lies past every chunk it emitted, so a worker's failure is thrown first. DegreeOfParallelism = 1 reads with the async enumerator. LineBufferState.LineNumber becomes a long: a chunk's first line number can pass int.MaxValue, and so could the sequential reader's count. Co-Authored-By: Claude Opus 5.5 (1M context) --- ARCHITECTURE.md | 17 ++ CHANGELOG.md | 3 +- README.md | 3 + docs/parallel-aggregate.md | 20 ++ src/FixedWidthParser/FixedWidth.cs | 15 ++ src/FixedWidthParser/PublicAPI.Unshipped.txt | 2 + .../Readers/FixedWidthReader.cs | 15 ++ .../Readers/LineBufferState.cs | 24 +- .../Readers/ParallelAggregator.cs | 19 +- .../Readers/ParallelStreamAggregator.cs | 255 ++++++++++++++++++ .../ParallelAggregateTests.cs | 154 ++++++++--- 11 files changed, 478 insertions(+), 49 deletions(-) create mode 100644 src/FixedWidthParser/Readers/ParallelStreamAggregator.cs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 10e15de..b98f2d2 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -113,6 +113,23 @@ small files run, so there is no separate sequential twin. A worker does not know how many lines precede its range. It counts them only when a line fails — line breaks from the start of the file up to the failing line, continued from there for later failures in the same range — so errors carry the sequential line number and the happy path pays nothing. + +A `Stream` cannot be read at an offset, so `ParallelStreamAggregator` follows ExcelReader's XLSX +pipeline instead: + +``` +source -> producer (1 thread) -> bounded queue -> N workers -> ordered merge + 256 KB pooled chunks, chunk, index, parse + Add by chunk index + cut after the last '\n' lines before +``` + +The producer carries the bytes after the last line break into the next chunk and counts the line +breaks it hands over, so each chunk knows the number of its first line and workers report sequential +line numbers directly. The queue holds about two chunks per worker, which bounds memory whatever the +source's size. A line longer than a chunk grows the chunk, up to `MaxLineLength`. A failure in the +producer lies past every chunk it emitted, so a worker's failure is thrown first. With +`DegreeOfParallelism = 1` the stream is read by the ordinary async enumerator. + Measurements are in [parallel-aggregate.md](docs/parallel-aggregate.md). ### Errors diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cd4678..c27ebc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,7 +40,8 @@ Windows-1252 after registering `CodePagesEncodingProvider`). read a file in byte-range partitions on several threads and fold each into an `IRecordAccumulator`, merged in file order. Every line is parsed exactly once, and errors carry the same line number as a sequential read. `FixedWidthParallelOptions.DegreeOfParallelism` - sets the thread count. + sets the thread count. `AggregateAsync` does the same over a `Stream`: one thread + reads chunks, the others parse them. - `FixedWidthReaderOptions`: `FormatProvider`, `Encoding`, `StringPool`, `BufferSize`, `MaxLineLength` and `OnParseError`, taken by every read. `FixedWidthWriterOptions`: `FormatProvider`, `Encoding` and `NewLine`. diff --git a/README.md b/README.md index 608166c..857bad9 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,9 @@ as a sequential read. `FixedWidthParallelOptions.DegreeOfParallelism` caps the t default, means one per processor; `1` reads the file in one pass). `OnParseError`, when set, runs on the worker threads. +`AggregateAsync` does the same over a `Stream`, which is read once: one thread reads it into chunks +and the others parse. + ### Options Every read takes an optional `FixedWidthReaderOptions`: diff --git a/docs/parallel-aggregate.md b/docs/parallel-aggregate.md index 8478506..730b218 100644 --- a/docs/parallel-aggregate.md +++ b/docs/parallel-aggregate.md @@ -23,6 +23,26 @@ string columns, or with a `StringPool`, was not measured. `dop 1` is one range through the same worker code as the parallel case, and lands within a few percent of the sequential enumerator on both paths. +## Stream + +Same file, generated path, the source opened fresh for each run. The stream engine runs `dop` parsing +workers beside the producer, so `dop 8` is eight parsing threads and one reading flow. The pipe rows +read a `PipeReader.Create(stream, ...)` through `AsStream()`, with the segment size shown. + +| Source | dop 1 | dop 2 | dop 4 | dop 8 | dop 16 | +|---|--:|--:|--:|--:|--:| +| file (`AggregateFileAsync`) | 181.5 | 104.5 | 61.8 | 46.8 | 46.4 | +| `Stream` | 192.2 | 85.1 | 44.9 | 35.6 | 31.8 | +| `PipeReader.AsStream()`, 64 KB segments | 194.8 | 78.1 | 46.8 | 38.0 | 39.6 | +| `PipeReader.AsStream()`, default 4 KB segments | 423.7 | 108.1 | 111.9 | 112.5 | 109.2 | + +A pipe with 4 KB segments caps the run at the speed one thread can copy 4 KB reads into chunks, so +the pipe's own buffer size, not the engine, sets the ceiling. + +The stream engine beats the file engine from `dop 2` up on this cached file. The file engine's ranges +are 3.25 MB at `dop 8` against the stream engine's 256 KB chunks, and each file worker refills a 64 KB +buffer with one `RandomAccess.Read` call at a time. Which of the two costs more is unmeasured. + ## Not measured - A cold file: every worker reads its own range, so on a spinning disk the access pattern is random. diff --git a/src/FixedWidthParser/FixedWidth.cs b/src/FixedWidthParser/FixedWidth.cs index 4205359..908b356 100644 --- a/src/FixedWidthParser/FixedWidth.cs +++ b/src/FixedWidthParser/FixedWidth.cs @@ -82,5 +82,20 @@ public static Task AggregateFileAsync( return ParallelAggregator.RunAsync, TAccumulator>( default, path, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), ct); } + + /// + /// Reads a stream once and folds every record into a across + /// several threads. controls whether the stream is disposed afterwards. + /// + /// Carries the same contract as . + public static Task AggregateAsync( + Stream stream, bool leaveOpen = false, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken ct = default) + where TModel : IFixedWidthModel + where TAccumulator : IRecordAccumulator, new() + { + ArgumentNullException.ThrowIfNull(stream); + return ParallelStreamAggregator.RunAsync, TAccumulator>( + default, stream, leaveOpen, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), ct); + } } } diff --git a/src/FixedWidthParser/PublicAPI.Unshipped.txt b/src/FixedWidthParser/PublicAPI.Unshipped.txt index bc30011..54928f6 100644 --- a/src/FixedWidthParser/PublicAPI.Unshipped.txt +++ b/src/FixedWidthParser/PublicAPI.Unshipped.txt @@ -181,3 +181,5 @@ FixedWidthParser.IRecordAccumulator.Add(in TModel model) -> void FixedWidthParser.IRecordAccumulator.Merge(TSelf following) -> void FixedWidthParser.Readers.FixedWidthReader.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! static FixedWidthParser.FixedWidth.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +FixedWidthParser.Readers.FixedWidthReader.AggregateAsync(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.AggregateAsync(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! diff --git a/src/FixedWidthParser/Readers/FixedWidthReader.cs b/src/FixedWidthParser/Readers/FixedWidthReader.cs index 9cecb33..a7ddf9c 100644 --- a/src/FixedWidthParser/Readers/FixedWidthReader.cs +++ b/src/FixedWidthParser/Readers/FixedWidthReader.cs @@ -85,5 +85,20 @@ public Task AggregateFileAsync(string path, FixedWid return ParallelAggregator.RunAsync, TAccumulator>( new ReflectionLineParser(_parser), path, _options, FixedWidthParallelOptions.Validate(parallel), ct); } + + /// + /// Reads a stream once and folds every record into a across + /// several threads: one thread reads the stream into chunks, the others parse them, one instance + /// per chunk, merged in stream order. controls whether the stream is + /// disposed afterwards. + /// + /// Carries the same contract as . + public Task AggregateAsync(Stream stream, bool leaveOpen = false, FixedWidthParallelOptions? parallel = null, CancellationToken ct = default) + where TAccumulator : IRecordAccumulator, new() + { + ArgumentNullException.ThrowIfNull(stream); + return ParallelStreamAggregator.RunAsync, TAccumulator>( + new ReflectionLineParser(_parser), stream, leaveOpen, _options, FixedWidthParallelOptions.Validate(parallel), ct); + } } } diff --git a/src/FixedWidthParser/Readers/LineBufferState.cs b/src/FixedWidthParser/Readers/LineBufferState.cs index fa804ab..8c8f215 100644 --- a/src/FixedWidthParser/Readers/LineBufferState.cs +++ b/src/FixedWidthParser/Readers/LineBufferState.cs @@ -22,7 +22,7 @@ internal struct LineBufferState internal int End; internal int ScanFrom; internal bool Eof; - internal int LineNumber; + internal long LineNumber; // Offset, from the first byte read, of the line TryGetLine last returned: the byte after the // previous line break, so a skipped byte order mark belongs to the first line. internal long LineStart; @@ -41,12 +41,26 @@ internal void Rent(FixedWidthReaderOptions options) internal void Rent(FixedWidthReaderOptions options, bool skipByteOrderMark) { - Buffer = ArrayPool.Shared.Rent(options.BufferSize); + Init(ArrayPool.Shared.Rent(options.BufferSize), 0, eof: false, linesBefore: 0, options, skipByteOrderMark); + } + + /// + /// Takes ownership of a pooled already holding the whole input: its + /// first bytes, numbered from the line after . + /// + internal void Wrap(byte[] buffer, int length, long linesBefore, FixedWidthReaderOptions options, bool skipByteOrderMark) + { + Init(buffer, length, eof: true, linesBefore, options, skipByteOrderMark); + } + + private void Init(byte[] buffer, int end, bool eof, long linesBefore, FixedWidthReaderOptions options, bool skipByteOrderMark) + { + Buffer = buffer; Start = 0; - End = 0; + End = end; ScanFrom = 0; - Eof = false; - LineNumber = 0; + Eof = eof; + LineNumber = linesBefore; LineStart = 0; _nextLineStart = 0; _compacted = 0; diff --git a/src/FixedWidthParser/Readers/ParallelAggregator.cs b/src/FixedWidthParser/Readers/ParallelAggregator.cs index d6bdb9a..d94b16a 100644 --- a/src/FixedWidthParser/Readers/ParallelAggregator.cs +++ b/src/FixedWidthParser/Readers/ParallelAggregator.cs @@ -92,12 +92,10 @@ private static TAccumulator AggregateChunk( { ct.ThrowIfCancellationRequested(); } - if (parser.TryParse(line, in context, out var model)) + if (!TryAdd(parser, line, in context, ref accumulator)) { - accumulator.Add(in model); - continue; + ParseErrors.HandleFailure(parser, line, reader.LineNumberAt(lineStart), in context, options.OnParseError); } - ParseErrors.HandleFailure(parser, line, reader.LineNumberAt(lineStart), in context, options.OnParseError); } } finally @@ -107,6 +105,19 @@ private static TAccumulator AggregateChunk( return accumulator; } + /// Parses one line and folds it in; when it does not parse. + internal static bool TryAdd(TParser parser, ReadOnlySpan line, in FixedWidthContext context, ref TAccumulator accumulator) + where TParser : struct, IRecordLineParser + where TAccumulator : IRecordAccumulator + { + if (!parser.TryParse(line, in context, out var model)) + { + return false; + } + accumulator.Add(in model); + return true; + } + private sealed class ChunkPlan { private readonly long _length; diff --git a/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs b/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs new file mode 100644 index 0000000..7ab1285 --- /dev/null +++ b/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs @@ -0,0 +1,255 @@ +using System.Buffers; +using System.Runtime.ExceptionServices; +using System.Runtime.InteropServices; +using System.Threading.Channels; + +namespace FixedWidthParser.Readers +{ + /// + /// Parallel aggregation over a sequential source. One producer reads the source into pooled chunks, + /// cuts each after its last line break, carries the rest into the next chunk, and counts the line + /// breaks it hands over, so every chunk knows the number of its first line. Workers fold the chunks + /// from a bounded queue into one accumulator each; the caller merges them in chunk order. A queue + /// about twice the worker count bounds memory to roughly that many chunks, whatever the source's size. + /// + internal static class ParallelStreamAggregator + { + private const int DefaultChunkSize = 256 * 1024; + private const int LinesPerCancellationCheck = 4096; + + [StructLayout(LayoutKind.Auto)] + private readonly record struct Chunk(int Index, byte[] Buffer, int Length, long LinesBefore); + + [StructLayout(LayoutKind.Auto)] + private readonly record struct Outcome(int Index, TAccumulator? Accumulator, ExceptionDispatchInfo? Failure); + + /// Folds , disposing it afterwards unless . + internal static async Task RunAsync( + TParser parser, Stream source, bool leaveOpen, FixedWidthReaderOptions options, FixedWidthParallelOptions parallel, CancellationToken ct) + where TParser : struct, IRecordLineParser + where TAccumulator : IRecordAccumulator, new() + { + try + { + if (parallel.Workers == 1) + { + return await SequentialAsync(parser, source, options, ct).ConfigureAwait(false); + } + return await PartitionedAsync(parser, source, options, parallel, ct).ConfigureAwait(false); + } + finally + { + if (!leaveOpen) + { + await source.DisposeAsync().ConfigureAwait(false); + } + } + } + + private static async Task SequentialAsync( + TParser parser, Stream source, FixedWidthReaderOptions options, CancellationToken ct) + where TParser : struct, IRecordLineParser + where TAccumulator : IRecordAccumulator, new() + { + var accumulator = new TAccumulator(); + var records = new FixedWidthAsyncRecordEnumerable(parser, source, ownsStream: false, options); + await foreach (var model in records.WithCancellation(ct).ConfigureAwait(false)) + { + accumulator.Add(in model); + } + return accumulator; + } + + private static async Task PartitionedAsync( + TParser parser, Stream source, FixedWidthReaderOptions options, FixedWidthParallelOptions parallel, CancellationToken ct) + where TParser : struct, IRecordLineParser + where TAccumulator : IRecordAccumulator, new() + { + int workerCount = parallel.Workers; + var queue = Channel.CreateBounded(new BoundedChannelOptions(workerCount * 2) { SingleWriter = true }); + var workers = new Task>>[workerCount]; + foreach (ref var worker in workers.AsSpan()) + { + worker = Task.Run(() => ConsumeAsync(parser, queue.Reader, options, ct), ct); + } + + ExceptionDispatchInfo? producerFailure = null; + try + { + await ProduceAsync(source, queue.Writer, options, parallel.ChunkSize > 0 ? parallel.ChunkSize : DefaultChunkSize, ct).ConfigureAwait(false); + } + // Cancellation too: the workers must finish before the chunks left in the queue are returned. +#pragma warning disable CA1031 // Rethrown by Merge once the workers have finished. + catch (Exception ex) +#pragma warning restore CA1031 + { + producerFailure = ExceptionDispatchInfo.Capture(ex); + } + finally + { + queue.Writer.TryComplete(); + } + + try + { + var outcomes = (await Task.WhenAll(workers).ConfigureAwait(false)).SelectMany(o => o).OrderBy(o => o.Index); + return Merge(outcomes, producerFailure); + } + finally + { + while (queue.Reader.TryRead(out var abandoned)) + { + ArrayPool.Shared.Return(abandoned.Buffer); + } + } + } + + // Every chunk precedes the point where the producer failed, so a worker's failure comes first in the source. + private static TAccumulator Merge(IEnumerable> outcomes, ExceptionDispatchInfo? producerFailure) + where TAccumulator : IRecordAccumulator, new() + { + TAccumulator? merged = default; + bool first = true; + foreach (var outcome in outcomes) + { + outcome.Failure?.Throw(); + if (first) + { + merged = outcome.Accumulator; + first = false; + continue; + } + merged!.Merge(outcome.Accumulator!); + } + producerFailure?.Throw(); + if (first) + { + return new TAccumulator(); + } + return merged!; + } + + private static async Task ProduceAsync(Stream source, ChannelWriter queue, FixedWidthReaderOptions options, int chunkSize, CancellationToken ct) + { + byte[] buffer = ArrayPool.Shared.Rent(chunkSize); + int filled = 0; + int index = 0; + long linesBefore = 0; + try + { + while (true) + { + if (filled == buffer.Length) + { + buffer = GrowUnterminated(buffer, linesBefore, options.MaxLineLength); + } + int read = await source.ReadAsync(buffer.AsMemory(filled), ct).ConfigureAwait(false); + filled += read; + bool eof = read == 0; + if (eof && filled == 0) + { + return; + } + if (!eof && filled < buffer.Length) + { + continue; + } + + int cut = eof ? filled : buffer.AsSpan(0, filled).LastIndexOf((byte)'\n') + 1; + if (cut == 0) + { + continue; + } + int tail = filled - cut; + byte[] next = ArrayPool.Shared.Rent(Math.Max(chunkSize, tail * 2)); + buffer.AsSpan(cut, tail).CopyTo(next); + long lines = buffer.AsSpan(0, cut).Count((byte)'\n'); + try + { + await queue.WriteAsync(new Chunk(index++, buffer, cut, linesBefore), ct).ConfigureAwait(false); + } + catch + { + ArrayPool.Shared.Return(next); + throw; + } + buffer = next; + filled = tail; + linesBefore += lines; + if (eof) + { + return; + } + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + // A full chunk with no line break holds the start of one line; grow it as LineBufferState does. + private static byte[] GrowUnterminated(byte[] buffer, long linesBefore, int maxLineLength) + { + if (buffer.Length > maxLineLength + 1L) + { + throw ParseErrors.LineTooLong(linesBefore + 1, maxLineLength); + } + byte[] bigger = ArrayPool.Shared.Rent((int)Math.Min(buffer.Length * 2L, maxLineLength + 2L)); + buffer.CopyTo(bigger, 0); + ArrayPool.Shared.Return(buffer); + return bigger; + } + + private static async Task>> ConsumeAsync( + TParser parser, ChannelReader queue, FixedWidthReaderOptions options, CancellationToken ct) + where TParser : struct, IRecordLineParser + where TAccumulator : IRecordAccumulator, new() + { + var outcomes = new List>(); + var context = options.Context; + await foreach (var chunk in queue.ReadAllAsync(ct).ConfigureAwait(false)) + { + try + { + outcomes.Add(new(chunk.Index, Fold(parser, chunk, options, in context, ct), null)); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + outcomes.Add(new(chunk.Index, default, ExceptionDispatchInfo.Capture(ex))); + } + } + return outcomes; + } + + private static TAccumulator Fold( + TParser parser, Chunk chunk, FixedWidthReaderOptions options, in FixedWidthContext context, CancellationToken ct) + where TParser : struct, IRecordLineParser + where TAccumulator : IRecordAccumulator, new() + { + var accumulator = new TAccumulator(); + var lines = default(LineBufferState); + lines.Wrap(chunk.Buffer, chunk.Length, chunk.LinesBefore, options, chunk.Index == 0 && options.SkipsByteOrderMark); + long delivered = 0; + try + { + while (lines.TryGetLine(out var line) == LineStatus.Line) + { + if (++delivered % LinesPerCancellationCheck == 0) + { + ct.ThrowIfCancellationRequested(); + } + if (!ParallelAggregator.TryAdd(parser, line, in context, ref accumulator)) + { + ParseErrors.HandleFailure(parser, line, lines.LineNumber, in context, options.OnParseError); + } + } + } + finally + { + lines.Return(); + } + return accumulator; + } + } +} diff --git a/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs b/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs index 7c0b0aa..786f813 100644 --- a/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs +++ b/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs @@ -5,9 +5,15 @@ namespace FixedWidthParser.Tests { - /// Parallel aggregation must equal one sequential pass, whatever the partition size. + /// Parallel aggregation must equal one sequential pass, whatever the source and partition size. public sealed class ParallelAggregateTests : IDisposable { + public enum Source + { + File, + Stream, + } + private readonly string _path = Path.GetTempFileName(); public void Dispose() @@ -45,6 +51,24 @@ public void Merge(GenAges following) } } + public static TheoryData SourcesAndChunkSizes() + { + var data = new TheoryData(); + foreach (var source in Enum.GetValues()) + { + foreach (int chunkSize in new[] { 1, 7, 10, 64, 4096, 0 }) + { + data.Add(source, chunkSize); + } + } + return data; + } + + public static TheoryData Sources() + { + return new TheoryData(Enum.GetValues()); + } + // A BOM, CRLF and LF breaks, empty lines and no final line break: every boundary a partition can cut. private static string Content(int lines, int badLine = 0) { @@ -65,25 +89,41 @@ private static string Content(int lines, int badLine = 0) return text.ToString(); } - private List Sequential() + private List Sequential(FixedWidthReaderOptions? options = null) + { + return [.. new FixedWidthReader(options).ReadFile(_path).Select(m => m.Age)]; + } + + private Task Aggregate(Source source, FixedWidthParallelOptions parallel, FixedWidthReaderOptions? options = null) { - return [.. new FixedWidthReader().ReadFile(_path).Select(m => m.Age)]; + var reader = new FixedWidthReader(options); + var ct = TestContext.Current.CancellationToken; + return source switch + { + Source.File => reader.AggregateFileAsync(_path, parallel, ct), + _ => reader.AggregateAsync(File.OpenRead(_path), leaveOpen: false, parallel, ct), + }; + } + + private Task AggregateGenerated(Source source, FixedWidthParallelOptions parallel) + { + var ct = TestContext.Current.CancellationToken; + return source switch + { + Source.File => FixedWidth.AggregateFileAsync(_path, parallel: parallel, ct: ct), + _ => FixedWidth.AggregateAsync(File.OpenRead(_path), parallel: parallel, ct: ct), + }; } [Theory] - [InlineData(1)] - [InlineData(7)] - [InlineData(10)] - [InlineData(64)] - [InlineData(4096)] - [InlineData(0)] - public async Task EveryPartitionSize_EqualsTheSequentialRead(int chunkSize) + [MemberData(nameof(SourcesAndChunkSizes))] + public async Task EveryPartitionSize_EqualsTheSequentialRead(Source source, int chunkSize) { File.WriteAllText(_path, Content(2000)); var parallel = new FixedWidthParallelOptions { DegreeOfParallelism = 4, ChunkSize = chunkSize }; - var runtime = await new FixedWidthReader().AggregateFileAsync(_path, parallel, TestContext.Current.CancellationToken); - var generated = await FixedWidth.AggregateFileAsync(_path, parallel: parallel, ct: TestContext.Current.CancellationToken); + var runtime = await Aggregate(source, parallel); + var generated = await AggregateGenerated(source, parallel); var expected = Sequential(); Assert.Equal(2000, expected.Count); @@ -91,81 +131,117 @@ public async Task EveryPartitionSize_EqualsTheSequentialRead(int chunkSize) Assert.Equal(expected, generated.Values); } - [Fact] - public async Task EmptyFile_ReturnsAnEmptyAccumulator() + [Theory] + [MemberData(nameof(Sources))] + public async Task OneWorker_EqualsTheSequentialRead(Source source) + { + File.WriteAllText(_path, Content(500)); + + var result = await Aggregate(source, new FixedWidthParallelOptions { DegreeOfParallelism = 1 }); + + Assert.Equal(Sequential(), result.Values); + } + + [Theory] + [MemberData(nameof(Sources))] + public async Task EmptySource_ReturnsAnEmptyAccumulator(Source source) { File.WriteAllBytes(_path, []); - var result = await new FixedWidthReader().AggregateFileAsync(_path, ct: TestContext.Current.CancellationToken); + var result = await Aggregate(source, new FixedWidthParallelOptions { DegreeOfParallelism = 4, ChunkSize = 8 }); Assert.Empty(result.Values); } [Theory] - [InlineData(5)] - [InlineData(4096)] - public async Task ParseFailure_ReportsTheSequentialLineNumber(int chunkSize) + [MemberData(nameof(SourcesAndChunkSizes))] + public async Task ParseFailure_ReportsTheSequentialLineNumber(Source source, int chunkSize) { File.WriteAllText(_path, Content(2000, badLine: 1537)); var expected = Assert.Throws(() => Sequential()); var parallel = new FixedWidthParallelOptions { DegreeOfParallelism = 4, ChunkSize = chunkSize }; - var actual = await Assert.ThrowsAsync( - () => new FixedWidthReader().AggregateFileAsync(_path, parallel, TestContext.Current.CancellationToken)); + var actual = await Assert.ThrowsAsync(() => Aggregate(source, parallel)); Assert.Equal(expected.LineNumber, actual.LineNumber); Assert.Equal(expected.Message, actual.Message); } - [Fact] - public async Task OnParseError_SkipsEachBadLineOnce_WithItsLineNumber() + [Theory] + [MemberData(nameof(Sources))] + public async Task OnParseError_SkipsEachBadLineOnce_WithItsLineNumber(Source source) { File.WriteAllText(_path, Content(2000, badLine: 1537) + "\nbad line\n" + Content(3, badLine: 2)[1..]); var sequentialErrors = new List(); - var sequentialOptions = new FixedWidthReaderOptions { OnParseError = ex => { sequentialErrors.Add(ex.LineNumber); return true; } }; - var expected = new FixedWidthReader(sequentialOptions).ReadFile(_path).Select(m => m.Age).ToList(); + var expected = Sequential(new FixedWidthReaderOptions { OnParseError = ex => { sequentialErrors.Add(ex.LineNumber); return true; } }); var parallelErrors = new ConcurrentBag(); - var parallelOptions = new FixedWidthReaderOptions { OnParseError = ex => { parallelErrors.Add(ex.LineNumber); return true; } }; + var options = new FixedWidthReaderOptions { OnParseError = ex => { parallelErrors.Add(ex.LineNumber); return true; } }; - var result = await new FixedWidthReader(parallelOptions).AggregateFileAsync( - _path, new FixedWidthParallelOptions { DegreeOfParallelism = 4, ChunkSize = 9 }, TestContext.Current.CancellationToken); + var result = await Aggregate(source, new FixedWidthParallelOptions { DegreeOfParallelism = 4, ChunkSize = 9 }, options); Assert.Equal(3, sequentialErrors.Count); Assert.Equal(expected, result.Values); Assert.Equal(sequentialErrors, parallelErrors.Order()); } - [Fact] - public async Task LineTooLong_ReportsTheSequentialLineNumber() + [Theory] + [MemberData(nameof(SourcesAndChunkSizes))] + public async Task LineTooLong_ReportsTheSequentialLineNumber(Source source, int chunkSize) { File.WriteAllText(_path, Content(300) + "\n" + new string('9', 50) + "\n" + Content(3)[1..]); var options = new FixedWidthReaderOptions { MaxLineLength = 20 }; - var expected = Assert.Throws(() => new FixedWidthReader(options).ReadFile(_path).ToList()); + var expected = Assert.Throws(() => Sequential(options)); + var parallel = new FixedWidthParallelOptions { DegreeOfParallelism = 4, ChunkSize = chunkSize }; - var actual = await Assert.ThrowsAsync(() => new FixedWidthReader(options).AggregateFileAsync( - _path, new FixedWidthParallelOptions { DegreeOfParallelism = 4, ChunkSize = 6 }, TestContext.Current.CancellationToken)); + var actual = await Assert.ThrowsAsync(() => Aggregate(source, parallel, options)); Assert.Equal(expected.Message, actual.Message); } - [Fact] - public async Task CancelledToken_Throws() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Stream_IsDisposedUnlessLeftOpen(bool leaveOpen) + { + File.WriteAllText(_path, Content(10)); + await using var stream = File.OpenRead(_path); + + await new FixedWidthReader().AggregateAsync( + stream, leaveOpen, new FixedWidthParallelOptions { DegreeOfParallelism = 2, ChunkSize = 16 }, TestContext.Current.CancellationToken); + + Assert.Equal(leaveOpen, stream.CanRead); + } + + [Theory] + [MemberData(nameof(Sources))] + public async Task CancelledToken_Throws(Source source) { File.WriteAllText(_path, Content(10)); using var cts = new CancellationTokenSource(); await cts.CancelAsync(); + var reader = new FixedWidthReader(); + var parallel = new FixedWidthParallelOptions { DegreeOfParallelism = 2 }; - await Assert.ThrowsAnyAsync( - () => new FixedWidthReader().AggregateFileAsync(_path, ct: cts.Token)); + await Assert.ThrowsAnyAsync(() => source switch + { + Source.File => reader.AggregateFileAsync(_path, parallel, cts.Token), + _ => reader.AggregateAsync(File.OpenRead(_path), leaveOpen: false, parallel, cts.Token), + }); } + // Arguments are validated before the stream is taken over, so the caller still owns it. [Fact] public async Task NegativeDegreeOfParallelism_Throws() { File.WriteAllText(_path, Content(10)); - - await Assert.ThrowsAsync(() => new FixedWidthReader().AggregateFileAsync( - _path, new FixedWidthParallelOptions { DegreeOfParallelism = -1 }, TestContext.Current.CancellationToken)); + var reader = new FixedWidthReader(); + var parallel = new FixedWidthParallelOptions { DegreeOfParallelism = -1 }; + var ct = TestContext.Current.CancellationToken; + await using var stream = File.OpenRead(_path); + + await Assert.ThrowsAsync(() => reader.AggregateFileAsync(_path, parallel, ct)); + await Assert.ThrowsAsync(() => reader.AggregateAsync(stream, leaveOpen: false, parallel, ct)); + Assert.True(stream.CanRead); } } } From d36b9580be972285aa87a0d35e7be957450a3c7b Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 15 Oct 2026 23:14:04 -0300 Subject: [PATCH 16/85] feat(reader)!: drop PipeReader reading in favour of AsStream The dedicated PipeRecordEnumerator read a 104 MB file in 472 ms; the same pipe through PipeReader.AsStream() into the Stream reader took 187-230 ms. It also duplicated every line rule LineBufferState implements, so removing it leaves one line splitter for every reader. Measurements are in docs/pipe-reader.md. BREAKING CHANGE: ReadAsync(PipeReader) is removed from FixedWidthReader and FixedWidth, with FixedWidthPipeRecordEnumerable and PipeRecordEnumerator. Pass pipeReader.AsStream() to the Stream overloads. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/benchmark-pages.yml | 1 - .github/workflows/benchmark-pr.yml | 1 - ARCHITECTURE.md | 27 +- CHANGELOG.md | 9 +- CONTRIBUTING.md | 4 +- README.md | 6 +- SECURITY.md | 2 +- STYLEGUIDE.md | 14 +- docs/README.md | 6 +- docs/pipe-reader.md | 28 ++ src/FixedWidthParser/FixedWidth.cs | 12 - src/FixedWidthParser/PublicAPI.Unshipped.txt | 8 - .../Readers/FixedWidthPipeRecordEnumerable.cs | 37 --- .../Readers/FixedWidthReader.cs | 15 +- .../Readers/PipeRecordEnumerator.cs | 262 ----------------- tests/Benchmarks/Perf/PipeReaderBenchmarks.cs | 63 ---- .../FixedWidthParser.Tests/PipeReaderTests.cs | 271 ------------------ .../ReaderOptionsTests.cs | 19 +- 18 files changed, 67 insertions(+), 718 deletions(-) create mode 100644 docs/pipe-reader.md delete mode 100644 src/FixedWidthParser/Readers/FixedWidthPipeRecordEnumerable.cs delete mode 100644 src/FixedWidthParser/Readers/PipeRecordEnumerator.cs delete mode 100644 tests/Benchmarks/Perf/PipeReaderBenchmarks.cs delete mode 100644 tests/FixedWidthParser.Tests/PipeReaderTests.cs diff --git a/.github/workflows/benchmark-pages.yml b/.github/workflows/benchmark-pages.yml index 0e96c30..9e34c53 100644 --- a/.github/workflows/benchmark-pages.yml +++ b/.github/workflows/benchmark-pages.yml @@ -42,7 +42,6 @@ jobs: - name: byte filter: >- Benchmarks.Perf.ReaderPoolingBenchmarks.* - Benchmarks.Perf.PipeReaderBenchmarks.* - name: writer filter: Benchmarks.Perf.WriterBenchmarks.* - name: comparison diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index 4492318..84b0e50 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -40,7 +40,6 @@ jobs: - name: byte filter: >- Benchmarks.Perf.ReaderPoolingBenchmarks.* - Benchmarks.Perf.PipeReaderBenchmarks.* - name: writer filter: Benchmarks.Perf.WriterBenchmarks.* - name: comparison diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index b98f2d2..cc2c126 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -70,24 +70,23 @@ source -> line splitter -> IRecordLineParser -> model pooled buffer) runtime or generated) ``` -Three enumerators share one shape, each specialized by a `struct` parse strategy +Two enumerators share one shape, each specialized by a `struct` parse strategy (`IRecordLineParser`: `ReflectionLineParser` or `GeneratedLineParser`, in `Readers/ILineParser.cs`). Because the strategy is a `struct` type argument, the JIT compiles one enumerator per `(TModel, TParser)` pair and the parse call devirtualizes. -| Source | Enumerator | Line splitting | -|---|---|---| -| `Stream`, sync | `RecordEnumerator` (a `struct`, so `foreach` allocates nothing) | `LineBufferState` | -| `Stream`, async | `AsyncRecordEnumerator` | `LineBufferState` | -| `PipeReader` | `PipeRecordEnumerator` | its own scan over `ReadOnlySequence` | - -`LineBufferState` (`Readers/LineBufferState.cs`) owns a pooled buffer: it finds the next `\n`, drops a -preceding `\r`, skips a UTF-8 byte order mark once, skips empty lines while counting them, and -compacts or grows the buffer when a line does not fit. Growth stops at `MaxLineLength`, which is what -bounds memory on a source with no line breaks. The pipe enumerator implements the same rules over -pipe segments, parsing a line in place when it sits in one segment and copying it to a pooled scratch -buffer only when it straddles two. The three must stay identical in behavior; see -[STYLEGUIDE.md § Twins](STYLEGUIDE.md#twins). +| Source | Enumerator | +|---|---| +| `Stream` or file, sync | `RecordEnumerator` (a `struct`, so `foreach` allocates nothing) | +| `Stream` or file, async | `AsyncRecordEnumerator` | + +Every reader, the parallel ones included, splits lines with `LineBufferState` +(`Readers/LineBufferState.cs`). It owns a pooled buffer: it finds the next `\n`, drops a preceding +`\r`, skips a UTF-8 byte order mark once, skips empty lines while counting them, and compacts or grows +the buffer when a line does not fit. Growth stops at `MaxLineLength`, which is what bounds memory on a +source with no line breaks. Keeping one splitter is deliberate: a `PipeReader` reader with its own +splitter over `ReadOnlySequence` was removed because it duplicated these rules and read slower +than `PipeReader.AsStream()` ([pipe-reader.md](docs/pipe-reader.md)). The async enumerator keeps the line scan synchronous: `MoveNextAsync` returns a completed `ValueTask` while the buffer still holds a line, and only `MoveNextSlowAsync` awaits a refill. diff --git a/CHANGELOG.md b/CHANGELOG.md index c27ebc1..d20a235 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,8 @@ in options objects and a `FixedWidthContext`, so later versions can add to them | 1.x | 2.0 | |---|---| -| `FixedWidthReader` over a `TextReader`, `FixedWidthByteReader` | `FixedWidthReader` over a `Stream`, a file or a `PipeReader` | -| `FixedWidthUtf8.Read*`, `FixedWidth.Read*` over a `TextReader` | `FixedWidth.Read*` over a `Stream`, a file or a `PipeReader` | +| `FixedWidthReader` over a `TextReader`, `FixedWidthByteReader` | `FixedWidthReader` over a `Stream` or a file | +| `FixedWidthUtf8.Read*`, `FixedWidth.Read*` over a `TextReader` | `FixedWidth.Read*` over a `Stream` or a file | | `new FixedWidthReader(provider, pool, bufferSize)` | `new FixedWidthReader(new FixedWidthReaderOptions { FormatProvider = …, StringPool = …, BufferSize = … })` | | `FixedWidth.Read(stream, encoding, leaveOpen, provider, pool, bufferSize)` | `FixedWidth.Read(stream, leaveOpen, options)`; the encoding is `FixedWidthReaderOptions.Encoding` | | `FixedWidthParser`, `Utf8FixedWidthParser` `.TryParse(line, provider, pool, out m)` | `FixedWidthParser.TryParse(ReadOnlySpan line, in FixedWidthContext context, out m)` | @@ -53,7 +53,7 @@ Windows-1252 after registering `CodePagesEncodingProvider`). - `FixedWidthReaderOptions.OnParseError`: return `true` to skip a line that fails to parse. - `FixedWidthReaderOptions.MaxLineLength` (default 1 MiB): a longer line throws `InvalidDataException`. Before, a source without line breaks doubled the read buffer until memory ran - out, which a `PipeReader` over a request body made reachable from the network. + out, which a request body made reachable from the network. - Encodings other than UTF-8 for text columns, dates parsed with a format and types that parse only from characters. A UTF-8 byte order mark is skipped only when the encoding is UTF-8. - Columns of any `ISpanParsable` type without a UTF-8 parser, decoded first (they used to be @@ -96,6 +96,9 @@ Windows-1252 after registering `CodePagesEncodingProvider`). - The library and generator build with warnings as errors. ### Removed +- **Breaking:** reading from a `PipeReader` (`ReadAsync(PipeReader)` and its enumerator types). Pass + `pipeReader.AsStream()` to the `Stream` overloads instead; it reads twice as fast as the dedicated + path did ([docs/pipe-reader.md](docs/pipe-reader.md)). - **Breaking:** `FixedWidthUtf8`, `FixedWidthByteReader`, `Utf8FixedWidthParser`, `IUtf8FixedWidthModel`, `IUtf8FixedWidthConverter`, `Utf8FixedWidthRuntime` and the `TextReader` overloads. Diagnostics `FWP007` and `FWP009`, which only applied to the second path, are retired. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 529d98f..54ff55c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,7 +43,7 @@ rewrites — see [ARCHITECTURE.md](ARCHITECTURE.md) for the shape of the codebas Codecov from the Linux leg. A change to one path needs the same change, and a parity test, on its twin: the runtime and - generated paths, and the stream, async and pipe readers + generated paths, the sync and async readers, and the file and stream aggregators ([STYLEGUIDE.md § Twins](STYLEGUIDE.md#twins)). Line-splitting code reads untrusted input; read [STYLEGUIDE.md § Untrusted Input](STYLEGUIDE.md#untrusted-input) before touching it. @@ -65,7 +65,7 @@ rewrites — see [ARCHITECTURE.md](ARCHITECTURE.md) for the shape of the codebas Use [Conventional Commits](https://www.conventionalcommits.org/): `type(scope): summary`, imperative mood, lowercase summary, no trailing period. Common types in this repo: `feat`, `fix`, `refactor`, `perf`, `test`, `docs`, `chore`, `ci`, `build`. The scope is usually the area touched (`reader`, -`writer`, `parser`, `generator`). Example: `fix(reader): bound a pipe line by MaxLineLength`. +`writer`, `parser`, `generator`). Example: `fix(reader): bound an unterminated line by MaxLineLength`. ## Pull requests diff --git a/README.md b/README.md index 857bad9..1265e1c 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Upgrading from 1.x? See [the 2.0 entry in the changelog](CHANGELOG.md#200). - **Attribute-driven** column mapping (`[FixedColumn(start, length)]`) on properties and public fields. - **Byte positions.** Lines are read and written as bytes, with no transcoding to UTF-16. Text columns are decoded with the file's encoding: UTF-8 by default, or any ASCII-compatible encoding such as Latin-1 or Windows-1252. -- **Lazy reading** from a `Stream`, a file or a `System.IO.Pipelines` `PipeReader`, synchronous (`IEnumerable` with a struct enumerator) and asynchronous (`IAsyncEnumerable`). +- **Lazy reading** from a `Stream` or a file, synchronous (`IEnumerable` with a struct enumerator) and asynchronous (`IAsyncEnumerable`). - **Source-generated parsing and writing** for `partial` models implementing `IFixedWidthModel`. - **Errors that name the column**: a `FixedWidthParseException` with the line number, column, position and raw text, and an `OnParseError` callback to skip bad lines. - **Bounded memory on untrusted input**: `MaxLineLength` stops a source without line breaks from growing the buffer. @@ -109,7 +109,7 @@ foreach (var person in FixedWidth.ReadFile("people.txt")) } ``` -### Async and pipes +### Async ```csharp await foreach (var person in reader.ReadFileAsync("people.txt")) @@ -118,7 +118,7 @@ await foreach (var person in reader.ReadFileAsync("people.txt")) } ``` -`ReadAsync(Stream, leaveOpen)` mirrors the synchronous overload, and cancellation is honored via `WithCancellation`. When the source is already a [`PipeReader`](https://learn.microsoft.com/dotnet/standard/io/pipelines) — a Kestrel request body, a socket, an upstream pipeline stage — `ReadAsync(PipeReader, leaveOpen)` parses straight off it and lets the pipe own buffering. For plain files and streams the `Stream` overloads are the faster default. +`ReadAsync(Stream, leaveOpen)` mirrors the synchronous overload, and cancellation is honored via `WithCancellation`. A [`PipeReader`](https://learn.microsoft.com/dotnet/standard/io/pipelines) — a Kestrel request body, a socket — is read through `pipeReader.AsStream()`. ### Parallel aggregation diff --git a/SECURITY.md b/SECURITY.md index 9745b3a..22a4fd5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -16,7 +16,7 @@ or by email to **gabrielandremarquez.matte@gmail.com**. Include, where possible: - A minimal reproduction: the model and the input that triggers it -- The reading or writing path involved (stream, async, `PipeReader`, generated or runtime) +- The reading or writing path involved (file or stream, sync, async or parallel, generated or runtime) - The impact you observed (crash, excessive memory/CPU, incorrect data, etc.) - The affected version(s) diff --git a/STYLEGUIDE.md b/STYLEGUIDE.md index 747ee43..1077816 100644 --- a/STYLEGUIDE.md +++ b/STYLEGUIDE.md @@ -278,9 +278,10 @@ captured and rethrown from the instance constructor, never left to surface as a Files and request bodies are attacker-controlled. These rules are not optional in reader code. - **Bound every buffer a source can grow.** A source without line breaks must hit `MaxLineLength` - before it drives an allocation, on every reader: stream, async stream and `PipeReader`. -- **When you add a guard to one reader, apply it to every sibling reader.** The three readers split - lines independently; a limit enforced on two of them is a hole in the third. + before it drives an allocation, on every reader: sync, async and both parallel aggregators. +- **Split lines in one place.** Every reader goes through `LineBufferState`. The parallel stream + producer also grows a chunk for an unterminated line, so a guard added to `LineBufferState.GrowIfFull` + belongs there too; a limit enforced in one and not the other is a hole. - **Validate before you allocate, not before you use.** Renting a bigger buffer first and discovering the line is too long afterwards still hands the attacker the allocation. - **Never let a malformed value silently change semantics.** A column that does not parse fails the @@ -314,8 +315,9 @@ keeps them honest: - **Runtime and generated.** `FixedWidthParser`/`FixedWidthWriter` and the generator's `TryParse`/`TryFormat` call the same `FixedWidthRuntime` helpers. Put shared behavior there, not in either side. -- **Stream, async stream and pipe readers.** Line semantics — the byte order mark, `\r\n`, empty - lines, a last line without a break, `MaxLineLength` — must match across all three. +- **Sequential and parallel reads.** `AggregateFileAsync` and `AggregateAsync` must return what a + sequential read folded into one accumulator would, with the same line numbers in errors. Their + per-line work shares `ParallelAggregator.TryAdd`. - **Sync and async writers.** `WriteMany` and `WriteManyAsync` batch the same way. So: @@ -325,7 +327,7 @@ So: - Behavior shared by twins belongs in a helper none of them duplicates — the split should cover only the awaiting or the code generation, not the logic around it. - Changes to a twinned path need a parity test asserting identical output, including at buffer - boundaries (a line straddling a refill or a pipe segment) — that is exactly where implementations drift. + boundaries (a line straddling a refill or a partition) — that is exactly where implementations drift. - A synchronous method does not take a `CancellationToken`. --- diff --git a/docs/README.md b/docs/README.md index 62e633e..b393c73 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,8 +6,10 @@ a constraint; the numbers that justified it live here, so they can be re-run and - [writer-performance.md](writer-performance.md) — where a written line spends its time, and the formatting shortcuts that came out of it. -- [parallel-aggregate.md](parallel-aggregate.md) — how `AggregateFileAsync` scales with the number of - workers. +- [parallel-aggregate.md](parallel-aggregate.md) — how `AggregateFileAsync` and `AggregateAsync` + scale with the number of workers. +- [pipe-reader.md](pipe-reader.md) — why a `PipeReader` is read through `AsStream()` rather than + its own enumerator. Usage documentation is the [README](../README.md); the shape of the code is [ARCHITECTURE.md](../ARCHITECTURE.md). diff --git a/docs/pipe-reader.md b/docs/pipe-reader.md new file mode 100644 index 0000000..438a0d6 --- /dev/null +++ b/docs/pipe-reader.md @@ -0,0 +1,28 @@ +# Reading a `PipeReader` + +Through 2.0 development the library read a `PipeReader` with its own enumerator, +`PipeRecordEnumerator`, which split lines over `ReadOnlySequence` and parsed a line in place when +it sat in one segment. It was removed in favour of `pipeReader.AsStream()` into the `Stream` reader. + +## Machine and method + +AMD Ryzen 7 5700X (8C/16T), Windows 10 Pro 19045, `net10.0`, Release. A Stopwatch console app reads +a 104 MB file of 4,000,000 lines, held in memory as a `MemoryStream`, through the generated path and +sums two columns. Best of seven runs after one warm-up run. + +## Result + +| Read | 4 KB segments | 64 KB segments | +|---|--:|--:| +| `ReadAsync(PipeReader)`, parsed in the segment | 472.0 ms | 474.6 ms | +| `ReadAsync(pipeReader.AsStream())` | 230.4 ms | 187.3 ms | +| `ReadAsync(Stream)` over the `MemoryStream` | 174.4 ms | | + +Avoiding the copy did not pay for itself. The dedicated enumerator's `MoveNextAsync` was `async` on +every line, with no synchronous path for a line already in the buffer, and built a `SequenceReader` +and several `SequencePosition`s per line. The `Stream` path copies a block at a time and then only +slices spans. Which of those costs dominated was not profiled. + +The enumerator also duplicated every line rule — the byte order mark, `\r\n`, empty lines, a last +line without a break, `MaxLineLength` — that `LineBufferState` already implements, so removing it +leaves one line splitter for every reader. diff --git a/src/FixedWidthParser/FixedWidth.cs b/src/FixedWidthParser/FixedWidth.cs index 908b356..65445fd 100644 --- a/src/FixedWidthParser/FixedWidth.cs +++ b/src/FixedWidthParser/FixedWidth.cs @@ -1,4 +1,3 @@ -using System.IO.Pipelines; using FixedWidthParser.Readers; namespace FixedWidthParser @@ -49,17 +48,6 @@ public static FixedWidthAsyncRecordEnumerable - /// Reads from a via await foreach, letting the pipe own buffering. - /// controls whether the reader is completed when iteration ends. - /// - public static FixedWidthPipeRecordEnumerable> ReadAsync(PipeReader reader, bool leaveOpen = false, FixedWidthReaderOptions? options = null) - where TModel : IFixedWidthModel - { - ArgumentNullException.ThrowIfNull(reader); - return new(default, reader, completeReader: !leaveOpen, FixedWidthReaderOptions.Validate(options)); - } - /// Reads from a file via await foreach with asynchronous I/O, reopening it on each enumeration. public static FixedWidthAsyncRecordEnumerable> ReadFileAsync(string path, FixedWidthReaderOptions? options = null) where TModel : IFixedWidthModel diff --git a/src/FixedWidthParser/PublicAPI.Unshipped.txt b/src/FixedWidthParser/PublicAPI.Unshipped.txt index 54928f6..b95b012 100644 --- a/src/FixedWidthParser/PublicAPI.Unshipped.txt +++ b/src/FixedWidthParser/PublicAPI.Unshipped.txt @@ -101,12 +101,9 @@ FixedWidthParser.Readers.AsyncRecordEnumerator.DisposeAsync() - FixedWidthParser.Readers.AsyncRecordEnumerator.MoveNextAsync() -> System.Threading.Tasks.ValueTask FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable.GetAsyncEnumerator(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> FixedWidthParser.Readers.AsyncRecordEnumerator! -FixedWidthParser.Readers.FixedWidthPipeRecordEnumerable -FixedWidthParser.Readers.FixedWidthPipeRecordEnumerable.GetAsyncEnumerator(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> FixedWidthParser.Readers.PipeRecordEnumerator! FixedWidthParser.Readers.FixedWidthReader FixedWidthParser.Readers.FixedWidthReader.FixedWidthReader(FixedWidthParser.FixedWidthReaderOptions? options = null) -> void FixedWidthParser.Readers.FixedWidthReader.Read(System.IO.Stream! stream, bool leaveOpen = false) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! -FixedWidthParser.Readers.FixedWidthReader.ReadAsync(System.IO.Pipelines.PipeReader! reader, bool leaveOpen = false) -> FixedWidthParser.Readers.FixedWidthPipeRecordEnumerable>! FixedWidthParser.Readers.FixedWidthReader.ReadAsync(System.IO.Stream! stream, bool leaveOpen = false) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! FixedWidthParser.Readers.FixedWidthReader.ReadFile(string! path) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! FixedWidthParser.Readers.FixedWidthReader.ReadFileAsync(string! path) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! @@ -118,10 +115,6 @@ FixedWidthParser.Readers.GeneratedLineParser.GeneratedLineParser() -> vo FixedWidthParser.Readers.GeneratedLineParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool FixedWidthParser.Readers.IRecordLineParser FixedWidthParser.Readers.IRecordLineParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool -FixedWidthParser.Readers.PipeRecordEnumerator -FixedWidthParser.Readers.PipeRecordEnumerator.Current.get -> TModel -FixedWidthParser.Readers.PipeRecordEnumerator.DisposeAsync() -> System.Threading.Tasks.ValueTask -FixedWidthParser.Readers.PipeRecordEnumerator.MoveNextAsync() -> System.Threading.Tasks.ValueTask FixedWidthParser.Readers.RecordEnumerator FixedWidthParser.Readers.RecordEnumerator.Current.get -> TModel FixedWidthParser.Readers.RecordEnumerator.Dispose() -> void @@ -142,7 +135,6 @@ FixedWidthParser.Writers.FixedWidthWriter.WriteManyAsync(System.IO.Strea override FixedWidthParser.FixedWidthContext.GetHashCode() -> int override FixedWidthParser.Formatters.ColumnFormatOptions.GetHashCode() -> int static FixedWidthParser.FixedWidth.Read(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! -static FixedWidthParser.FixedWidth.ReadAsync(System.IO.Pipelines.PipeReader! reader, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthPipeRecordEnumerable>! static FixedWidthParser.FixedWidth.ReadAsync(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! static FixedWidthParser.FixedWidth.ReadFile(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! static FixedWidthParser.FixedWidth.ReadFileAsync(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! diff --git a/src/FixedWidthParser/Readers/FixedWidthPipeRecordEnumerable.cs b/src/FixedWidthParser/Readers/FixedWidthPipeRecordEnumerable.cs deleted file mode 100644 index 4d13298..0000000 --- a/src/FixedWidthParser/Readers/FixedWidthPipeRecordEnumerable.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System.IO.Pipelines; - -namespace FixedWidthParser.Readers -{ - /// - /// Records read from a via await foreach, letting the pipe own - /// buffering and read-ahead. Single-pass: a pipe is consumed as it is read. - /// - public sealed class FixedWidthPipeRecordEnumerable : IAsyncEnumerable - where TParser : struct, IRecordLineParser - { - private readonly TParser _parser; - private readonly PipeReader _reader; - private readonly bool _completeReader; - private readonly FixedWidthReaderOptions _options; - - internal FixedWidthPipeRecordEnumerable(TParser parser, PipeReader reader, bool completeReader, FixedWidthReaderOptions options) - { - _parser = parser; - _reader = reader; - _completeReader = completeReader; - _options = options; - } - -#pragma warning disable HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type - public PipeRecordEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) -#pragma warning restore HLQ006 - { - return new(_parser, _reader, _completeReader, _options, cancellationToken); - } - - IAsyncEnumerator IAsyncEnumerable.GetAsyncEnumerator(CancellationToken cancellationToken) - { - return GetAsyncEnumerator(cancellationToken); - } - } -} diff --git a/src/FixedWidthParser/Readers/FixedWidthReader.cs b/src/FixedWidthParser/Readers/FixedWidthReader.cs index a7ddf9c..2c123bf 100644 --- a/src/FixedWidthParser/Readers/FixedWidthReader.cs +++ b/src/FixedWidthParser/Readers/FixedWidthReader.cs @@ -1,10 +1,9 @@ -using System.IO.Pipelines; using FixedWidthParser.Parsers; namespace FixedWidthParser.Readers { /// - /// Reads attribute-mapped models from a stream, a file or a pipe, lazily and with low allocation, + /// Reads attribute-mapped models from a stream or a file, lazily and with low allocation, /// through the runtime (reflection-built) parser. Configured once; reusable across reads. For /// source-generated models use . /// Column positions are bytes. @@ -49,18 +48,6 @@ public FixedWidthAsyncRecordEnumerable> Rea return new(new ReflectionLineParser(_parser), stream, ownsStream: !leaveOpen, _options); } - /// - /// Reads from a via await foreach, letting the pipe own buffering - /// ( does not apply). Use it when the source is - /// already a pipe, such as a Kestrel request body. controls whether - /// the reader is completed when iteration ends. - /// - public FixedWidthPipeRecordEnumerable> ReadAsync(PipeReader reader, bool leaveOpen = false) - { - ArgumentNullException.ThrowIfNull(reader); - return new(new ReflectionLineParser(_parser), reader, completeReader: !leaveOpen, _options); - } - /// Reads from a file via await foreach with asynchronous I/O, reopening it on each enumeration. public FixedWidthAsyncRecordEnumerable> ReadFileAsync(string path) { diff --git a/src/FixedWidthParser/Readers/PipeRecordEnumerator.cs b/src/FixedWidthParser/Readers/PipeRecordEnumerator.cs deleted file mode 100644 index 47a25c8..0000000 --- a/src/FixedWidthParser/Readers/PipeRecordEnumerator.cs +++ /dev/null @@ -1,262 +0,0 @@ -using System.Buffers; -using System.IO.Pipelines; - -namespace FixedWidthParser.Readers -{ - /// - /// -based counterpart of : - /// lets the pipe own buffering and read-ahead. A line is parsed in place when it sits in a single - /// segment, and copied into a pooled scratch buffer only when it straddles segments. Line semantics - /// match the stream path: a UTF-8 byte order mark is skipped once, lines split on \n (a - /// preceding \r is dropped), empty lines are counted but not yielded, and a final line without - /// a line break is yielded. - /// - public sealed class PipeRecordEnumerator : IAsyncEnumerator - where TParser : struct, IRecordLineParser - { - private const byte Cr = (byte)'\r'; - private const byte Lf = (byte)'\n'; - - private readonly TParser _parser; - private readonly bool _completeReader; - private readonly FixedWidthContext _context; - private readonly Func? _onParseError; - private readonly int _maxLineLength; - private readonly CancellationToken _cancellationToken; - private PipeReader? _reader; - private byte[]? _scratch; - private long _lineNumber; - private bool _preambleChecked; - private TModel _current; - - private ReadResult _currentResult; - private bool _hasResult; - private SequencePosition _consumed; - private SequencePosition _examined; - private long _scannedBytes; - - internal PipeRecordEnumerator(TParser parser, PipeReader reader, bool completeReader, FixedWidthReaderOptions options, CancellationToken cancellationToken) - { - _parser = parser; - _reader = reader; - _completeReader = completeReader; - _context = options.Context; - _onParseError = options.OnParseError; - _maxLineLength = options.MaxLineLength; - _preambleChecked = !options.SkipsByteOrderMark; - _cancellationToken = cancellationToken; - _current = default!; - } - - /// - public TModel Current => _current; - - /// - public async ValueTask MoveNextAsync() - { - ObjectDisposedException.ThrowIf(_reader is null, nameof(PipeRecordEnumerator<,>)); - var reader = _reader; - - try - { - while (true) - { - _cancellationToken.ThrowIfCancellationRequested(); - - if (!_hasResult) - { - _currentResult = await reader.ReadAsync(_cancellationToken).ConfigureAwait(false); - if (_currentResult.IsCanceled) - { - throw new OperationCanceledException(_cancellationToken); - } - _hasResult = true; - _consumed = _currentResult.Buffer.Start; - _examined = _currentResult.Buffer.Start; - } - - var status = TryReadFromResult(out _consumed, out _examined); - if (status == LineStatus.Line) - { - return true; - } - - reader.AdvanceTo(_consumed, _examined); - _hasResult = false; - - if (status == LineStatus.End || _currentResult.IsCompleted) - { - return false; - } - } - } - catch (Exception) - { - if (_hasResult) - { - reader.AdvanceTo(_consumed, _examined); - _hasResult = false; - } - throw; - } - } - - private LineStatus TryReadFromResult(out SequencePosition consumed, out SequencePosition examined) - { - ReadOnlySequence buffer = _currentResult.Buffer; - bool isCompleted = _currentResult.IsCompleted; - - if (!_preambleChecked) - { - if (buffer.Length >= 3) - { - Span head = stackalloc byte[3]; - buffer.Slice(0, 3).CopyTo(head); - if (head is [0xEF, 0xBB, 0xBF]) - { - _consumed = buffer.GetPosition(3); - } - _preambleChecked = true; - } - else if (isCompleted) - { - _preambleChecked = true; - } - else - { - consumed = _consumed; - examined = buffer.End; - return LineStatus.NeedData; - } - } - - while (true) - { - ReadOnlySequence remaining = buffer.Slice(_consumed); - var scanStart = remaining.GetPosition(_scannedBytes); - var seqReader = new SequenceReader(remaining.Slice(scanStart)); - - if (seqReader.TryReadTo(out ReadOnlySpan _, Lf, advancePastDelimiter: true)) - { - _lineNumber++; - var positionOfLf = remaining.GetPosition(seqReader.Consumed - 1, scanStart); - var line = TrimTrailingCr(remaining.Slice(0, positionOfLf)); - var nextPosition = remaining.GetPosition(seqReader.Consumed, scanStart); - _consumed = nextPosition; - _examined = nextPosition; - _scannedBytes = 0; - - if (!line.IsEmpty && Parse(in line)) - { - consumed = _consumed; - examined = _consumed; - return LineStatus.Line; - } - continue; - } - - if (!isCompleted) - { - // An unterminated line longer than the limit (+1 for its '\r') can never become valid. - if (remaining.Length > _maxLineLength + 1L) - { - throw ParseErrors.LineTooLong(_lineNumber + 1, _maxLineLength); - } - _scannedBytes = remaining.Length; - consumed = _consumed; - examined = buffer.End; - return LineStatus.NeedData; - } - - consumed = buffer.End; - examined = buffer.End; - _consumed = buffer.End; - _examined = buffer.End; - _scannedBytes = 0; - if (remaining.Length <= 0) - { - return LineStatus.End; - } - _lineNumber++; - var last = TrimTrailingCr(in remaining); - return !last.IsEmpty && Parse(in last) ? LineStatus.Line : LineStatus.End; - } - } - - // Returns false when the line failed to parse and OnParseError asked to skip it. - private bool Parse(in ReadOnlySequence line) - { - if (line.Length > _maxLineLength) - { - throw ParseErrors.LineTooLong(_lineNumber, _maxLineLength); - } - if (line.IsSingleSegment) - { - return ParseSpan(line.FirstSpan); - } - - int length = (int)line.Length; - if (_scratch is null || _scratch.Length < length) - { - if (_scratch is not null) - { - var oldScratch = _scratch; - _scratch = null; - ArrayPool.Shared.Return(oldScratch); - } - _scratch = ArrayPool.Shared.Rent(length); - } - line.CopyTo(_scratch); - return ParseSpan(_scratch.AsSpan(0, length)); - } - - private bool ParseSpan(ReadOnlySpan line) - { - if (_parser.TryParse(line, in _context, out _current)) - { - return true; - } - ParseErrors.HandleFailure(_parser, line, _lineNumber, in _context, _onParseError); - return false; - } - - private static ReadOnlySequence TrimTrailingCr(in ReadOnlySequence line) - { - long length = line.Length; - if (length == 0) - { - return line; - } - byte last = line.Slice(length - 1).FirstSpan[0]; - return last == Cr ? line.Slice(0, length - 1) : line; - } - - /// - public async ValueTask DisposeAsync() - { - if (_scratch is not null) - { - var oldScratch = _scratch; - _scratch = null; - ArrayPool.Shared.Return(oldScratch); - } - - var reader = _reader; - _reader = null; - if (reader is null) - { - return; - } - if (_hasResult) - { - reader.AdvanceTo(_consumed, _examined); - _hasResult = false; - } - - if (_completeReader) - { - await reader.CompleteAsync().ConfigureAwait(false); - } - } - } -} diff --git a/tests/Benchmarks/Perf/PipeReaderBenchmarks.cs b/tests/Benchmarks/Perf/PipeReaderBenchmarks.cs deleted file mode 100644 index dd72651..0000000 --- a/tests/Benchmarks/Perf/PipeReaderBenchmarks.cs +++ /dev/null @@ -1,63 +0,0 @@ -using System.IO.Pipelines; -using BenchmarkDotNet.Attributes; -using FixedWidthParser.Readers; - -namespace Benchmarks.Perf -{ - /// - /// Reading the same bytes through a (the reader owns buffering) versus a - /// (the pipe owns buffering/read-ahead). The source is in memory, so the - /// comparison isolates buffering and line splitting from I/O. - /// - [Config(typeof(RegressionConfig))] - public class PipeReaderBenchmarks - { - private readonly FixedWidthReader _reader = new(); - private byte[] _bytes = []; - - [Params(1000)] - public int Count { get; set; } - - [GlobalSetup] - public void Setup() - { - _bytes = SampleData.Bytes(Count); - } - - [Benchmark(Baseline = true)] - public async Task Stream_ReadAsync() - { - int sum = 0; - await foreach (var model in _reader.ReadAsync(new MemoryStream(_bytes, writable: false)).ConfigureAwait(false)) - { - sum += model.Age; - } - return sum; - } - - /// Default segment sizing: lines are typically contiguous (parsed in place). - [Benchmark] - public async Task Pipe_ReadAsync() - { - int sum = 0; - await foreach (var model in _reader.ReadAsync(PipeReader.Create(new MemoryStream(_bytes, writable: false))).ConfigureAwait(false)) - { - sum += model.Age; - } - return sum; - } - - /// A tiny buffer: most lines span segments, so they go through the pooled scratch copy. - [Benchmark] - public async Task Pipe_ReadAsync_SmallSegments() - { - int sum = 0; - var pipe = PipeReader.Create(new MemoryStream(_bytes, writable: false), new StreamPipeReaderOptions(bufferSize: 16)); - await foreach (var model in _reader.ReadAsync(pipe).ConfigureAwait(false)) - { - sum += model.Age; - } - return sum; - } - } -} diff --git a/tests/FixedWidthParser.Tests/PipeReaderTests.cs b/tests/FixedWidthParser.Tests/PipeReaderTests.cs deleted file mode 100644 index 696811b..0000000 --- a/tests/FixedWidthParser.Tests/PipeReaderTests.cs +++ /dev/null @@ -1,271 +0,0 @@ -using System.IO.Pipelines; -using System.Text; -using FixedWidthParser.Readers; -using static FixedWidthParser.Tests.TestHelpers; - -namespace FixedWidthParser.Tests -{ - /// - /// Covers the overloads of - /// (reflection) and (source-generated): line-ending - /// handling, BOM skipping, empty-line skipping, multi-segment lines (the pooled scratch-copy path), - /// cancellation, reader-completion behavior, and parity with the stream reader. - /// - public class PipeReaderTests - { - private const string TwoPeople = - "John Doe 30 60000.00 \n" + - "Jane 28 55000.00 "; - -#pragma warning disable IDISP001 // ownership of the stream transfers to the PipeReader / reader - private static PipeReader Pipe(string text, int bufferSize = -1) - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); - return bufferSize > 0 - ? PipeReader.Create(stream, new StreamPipeReaderOptions(bufferSize: bufferSize)) - : PipeReader.Create(stream); - } - - private static MemoryStream StreamOf(string text) - { - return new MemoryStream(Encoding.UTF8.GetBytes(text)); - } -#pragma warning restore IDISP001 - - private static async Task> CollectAsync(IAsyncEnumerable source) - { - var list = new List(); - await foreach (var item in source.ConfigureAwait(false)) - { - list.Add(item); - } - return list; - } - - [Fact] - public async Task ReadAsync_Pipe_ParsesAll() - { - var reader = new FixedWidthReader(); - - var people = await CollectAsync(reader.ReadAsync(Pipe(TwoPeople))).ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(30, people[0].Age); - Assert.Equal(60000.00, people[0].Salary, 2); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public async Task ReadAsync_Pipe_Generated_ParsesAll() - { - var people = await CollectAsync( - FixedWidth.ReadAsync(Pipe(TwoPeople))) - .ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Theory] - [InlineData("ABC\nDEF\n")] // LF + trailing newline - [InlineData("ABC\r\nDEF\r\n")] // CRLF - [InlineData("ABC\nDEF")] // no trailing newline on the last line - public async Task ReadAsync_Pipe_HandlesLineEndingsAndTrailingNewline(string text) - { - var reader = new FixedWidthReader(); - - var codes = (await CollectAsync(reader.ReadAsync(Pipe(text))).ConfigureAwait(true)).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public async Task ReadAsync_Pipe_SkipsEmptyLines() - { - var reader = new FixedWidthReader(); - - var codes = (await CollectAsync(reader.ReadAsync(Pipe("ABC\n\n\nDEF\n"))).ConfigureAwait(true)).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public async Task ReadAsync_Pipe_SkipsLeadingBom() - { - var reader = new FixedWidthReader(); - - var codes = (await CollectAsync(reader.ReadAsync(Pipe("ABC\nDEF\n"))).ConfigureAwait(true)).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public async Task ReadAsync_Pipe_MultiSegmentLine_ParsesViaScratch() - { - // A tiny pipe buffer (8 bytes) forces each 25-byte record to span several segments, so the - // line is non-contiguous and must be copied into the pooled scratch buffer before parsing. - var reader = new FixedWidthReader(); - - var people = await CollectAsync(reader.ReadAsync(Pipe(TwoPeople, bufferSize: 8))).ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(60000.00, people[0].Salary, 2); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(28, people[1].Age); - } - - [Fact] - public async Task ReadAsync_Pipe_MatchesStreamReader() - { - // Same content, two sources: the pipe path must produce exactly what the stream path does. - var byteReader = new FixedWidthReader(); - -#pragma warning disable IDISP004 // the reader owns and disposes the stream when iteration completes - var fromStream = await CollectAsync(byteReader.ReadAsync(StreamOf(TwoPeople))).ConfigureAwait(true); -#pragma warning restore IDISP004 - var fromPipe = await CollectAsync(byteReader.ReadAsync(Pipe(TwoPeople))).ConfigureAwait(true); - - Assert.Equal(fromStream.Count, fromPipe.Count); - for (int i = 0; i < fromStream.Count; i++) - { - Assert.Equal(fromStream[i].Name, fromPipe[i].Name); - Assert.Equal(fromStream[i].Age, fromPipe[i].Age); - Assert.Equal(fromStream[i].Salary, fromPipe[i].Salary, 2); - } - } - - [Fact] - public async Task ReadAsync_Pipe_HonorsCancellation() - { - var reader = new FixedWidthReader(); - using var cts = new CancellationTokenSource(); - await cts.CancelAsync().ConfigureAwait(true); - - await Assert.ThrowsAnyAsync(async () => - { - await foreach (var _ in reader.ReadAsync(Pipe("ABC\nDEF\n")).WithCancellation(cts.Token).ConfigureAwait(false)) - { - Assert.Fail("Enumeration should have been cancelled before yielding a record."); - } - }).ConfigureAwait(true); - } - - [Fact] - public async Task ReadAsync_Pipe_CompletesReaderByDefault() - { - var reader = new FixedWidthReader(); - var pipe = Pipe("ABC\nDEF\n"); - - _ = await CollectAsync(reader.ReadAsync(pipe)).ConfigureAwait(true); - - // The reader was completed, so a further read is no longer allowed. - await Assert.ThrowsAsync(async () => await pipe.ReadAsync().ConfigureAwait(false)).ConfigureAwait(true); - } - - [Fact] - public async Task ReadAsync_Pipe_LeaveOpen_DoesNotCompleteReader() - { - var reader = new FixedWidthReader(); - var pipe = Pipe("ABC\nDEF\n"); - - _ = await CollectAsync(reader.ReadAsync(pipe, leaveOpen: true)).ConfigureAwait(true); - - // Not completed: a further read succeeds and reports end-of-data. - var result = await pipe.ReadAsync().ConfigureAwait(true); - Assert.True(result.IsCompleted); - pipe.AdvanceTo(result.Buffer.End); - await pipe.CompleteAsync().ConfigureAwait(true); - } - - [Fact] - public void ReadAsync_Pipe_NullReader_Throws() - { - var reader = new FixedWidthReader(); - - Assert.Throws(() => reader.ReadAsync((PipeReader)null!)); - } - - [Fact] - [System.Diagnostics.CodeAnalysis.SuppressMessage("IDisposableAnalyzers.Correctness", "IDISP016:Don't use disposed instance", - Justification = "The test deliberately uses the enumerator after disposal to verify it throws.")] - public async Task ReadAsync_Pipe_MoveNextAfterDispose_Throws() - { - var reader = new FixedWidthReader(); - var enumerator = reader.ReadAsync(Pipe("ABC\nDEF\n")).GetAsyncEnumerator(); - await enumerator.DisposeAsync().ConfigureAwait(true); - - await Assert.ThrowsAsync(async () => await enumerator.MoveNextAsync().ConfigureAwait(false)).ConfigureAwait(true); - } - - [Fact] - public async Task ReadAsync_Pipe_AssemblesAcrossManySmallWrites() - { - // Feeding one byte at a time forces the BOM check and every line to span multiple reads, - // exercising the "need more data" paths (both before and within a line). - var pipe = new Pipe(); - var reader = new FixedWidthReader(); - - var produce = Task.Run(async () => - { - foreach (var b in Encoding.UTF8.GetBytes("ABC\nDEF\n")) - { - await pipe.Writer.WriteAsync(new[] { b }).ConfigureAwait(false); - await pipe.Writer.FlushAsync().ConfigureAwait(false); - } - await pipe.Writer.CompleteAsync().ConfigureAwait(false); - }); - - var codes = (await CollectAsync(reader.ReadAsync(pipe.Reader)).ConfigureAwait(true)).Select(m => m.Code).ToList(); - await produce.ConfigureAwait(true); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public Task ReadAsync_Pipe_UnparseableLine_ThrowsFormat() - { - var reader = new FixedWidthReader(); - - // "A" is shorter than CodeModel's 3-wide column, so the trailing line fails to parse. - return Assert.ThrowsAsync(async () => - await CollectAsync(reader.ReadAsync(Pipe("A"))).ConfigureAwait(false)); - } - - [Fact] - public async Task ReadAsync_Pipe_CancelPendingRead_ThrowsOperationCanceledException() - { - var pipe = new Pipe(); - var reader = new FixedWidthReader(); - - var readTask = CollectAsync(reader.ReadAsync(pipe.Reader)); - - pipe.Reader.CancelPendingRead(); - - await Assert.ThrowsAnyAsync(async () => await readTask.ConfigureAwait(false)).ConfigureAwait(true); - await pipe.Writer.CompleteAsync().ConfigureAwait(true); - } - - [Fact] - public async Task ReadAsync_Pipe_UnparseableLine_WithLeaveOpen_LeavesReaderUsable() - { - var reader = new FixedWidthReader(); - var pipe = Pipe("A\nDEF\n"); - - var enumerator = reader.ReadAsync(pipe, leaveOpen: true).GetAsyncEnumerator(); - - await Assert.ThrowsAsync(async () => await enumerator.MoveNextAsync().ConfigureAwait(false)).ConfigureAwait(true); - - await enumerator.DisposeAsync().ConfigureAwait(true); - - var result = await pipe.ReadAsync().ConfigureAwait(true); - var text = Encoding.UTF8.GetString(result.Buffer); - Assert.Contains("DEF", text); - pipe.AdvanceTo(result.Buffer.End); - await pipe.CompleteAsync().ConfigureAwait(true); - } - } -} diff --git a/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs b/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs index 9293665..d26ef34 100644 --- a/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs +++ b/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs @@ -1,4 +1,3 @@ -using System.IO.Pipelines; using System.Text; using FixedWidthParser.Readers; using static FixedWidthParser.Tests.TestHelpers; @@ -88,16 +87,14 @@ public void OnParseError_SkippingTheLastLine_EndsCleanly() } [Fact] - public async Task OnParseError_SkipsOnAsyncAndPipeReads() + public async Task OnParseError_SkipsOnAsyncReads() { var options = new FixedWidthReaderOptions { OnParseError = _ => true }; var reader = new FixedWidthReader(options); var fromStream = await reader.ReadAsync(StreamOf(BadAge)).Select(p => p.Name).ToListAsync(TestContext.Current.CancellationToken); - var fromPipe = await reader.ReadAsync(PipeReader.Create(StreamOf(BadAge))).Select(p => p.Name).ToListAsync(TestContext.Current.CancellationToken); Assert.Equal(["John Doe", "Max"], fromStream); - Assert.Equal(fromStream, fromPipe); } [Fact] @@ -146,20 +143,6 @@ await Assert.ThrowsAsync(async () => }); } - [Fact] - public async Task MaxLineLength_UnterminatedLine_ThrowsOnPipeRead() - { - var options = new FixedWidthReaderOptions { MaxLineLength = 100 }; - var pipe = PipeReader.Create(new MemoryStream(new byte[10_000]), new StreamPipeReaderOptions(bufferSize: 64)); - - await Assert.ThrowsAsync(async () => - { - await foreach (var _ in FixedWidth.ReadAsync(pipe, options: options)) - { - } - }); - } - [Theory] [InlineData(0, 10)] [InlineData(10, 0)] From ff98723c2c6b01c38b964ef6ac1de1082d2828d2 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 23 Oct 2026 22:17:29 -0300 Subject: [PATCH 17/85] feat(layout): build attribute layouts as FixedWidthLayout objects FixedWidthLayout.FromAttributes() builds a model's column parsers and formatters once and caches them; FixedWidth.TryParse and TryFormat take a layout. The runtime parser and writer now sit on top of it. A model that only one side can handle keeps that side's error until the side is first used. --- .../Attributes/ColumnLayoutValidator.cs | 14 +- .../Attributes/ModelColumns.cs | 7 +- src/FixedWidthParser/Delegates.cs | 7 +- src/FixedWidthParser/FixedWidth.cs | 35 ++++ .../Formatters/ColumnFormatters.cs | 19 +- .../Formatters/FormatterBuilder.cs | 107 ++++++++++ .../Layouts/FixedWidthLayout.cs | 194 ++++++++++++++++++ .../Parsers/FixedWidthParser.cs | 55 +---- src/FixedWidthParser/Parsers/ParserBuilder.cs | 2 +- .../Processors/ColumnParserFactory.cs | 38 ++-- src/FixedWidthParser/PublicAPI.Unshipped.txt | 9 + .../Writers/FixedWidthWriter.cs | 145 ++----------- tests/FixedWidthParser.Tests/LayoutTests.cs | 55 +++++ tests/FixedWidthParser.Tests/TestHelpers.cs | 8 +- 14 files changed, 475 insertions(+), 220 deletions(-) create mode 100644 src/FixedWidthParser/Formatters/FormatterBuilder.cs create mode 100644 src/FixedWidthParser/Layouts/FixedWidthLayout.cs create mode 100644 tests/FixedWidthParser.Tests/LayoutTests.cs diff --git a/src/FixedWidthParser/Attributes/ColumnLayoutValidator.cs b/src/FixedWidthParser/Attributes/ColumnLayoutValidator.cs index cb0811a..9af3a65 100644 --- a/src/FixedWidthParser/Attributes/ColumnLayoutValidator.cs +++ b/src/FixedWidthParser/Attributes/ColumnLayoutValidator.cs @@ -7,25 +7,19 @@ namespace FixedWidthParser.Attributes /// internal static class ColumnLayoutValidator { - public static void Validate(Span<(int Start, int Length, string Name)> columns, Type modelType) + public static void Validate(Span<(int Start, int Length, string Name)> columns, string modelName) { - if (columns.Length == 0) - { - throw new InvalidOperationException( - $"Type {modelType.Name} has no [FixedColumn] members; a fixed-width model must define at least one column."); - } - foreach (var (Start, Length, Name) in columns) { if (Start < 0) { throw new InvalidOperationException( - $"Column \"{Name}\" in {modelType.Name} has a negative Start ({Start})."); + $"Column \"{Name}\" in {modelName} has a negative Start ({Start})."); } if (Length < 1) { throw new InvalidOperationException( - $"Column \"{Name}\" in {modelType.Name} has an invalid Length ({Length}); it must be >= 1."); + $"Column \"{Name}\" in {modelName} has an invalid Length ({Length}); it must be >= 1."); } } @@ -43,7 +37,7 @@ public static void Validate(Span<(int Start, int Length, string Name)> columns, if (current.Start < maxEnd) { throw new InvalidOperationException( - $"Overlapping columns in {modelType.Name}: \"{farthest.Name}\" [{farthest.Start}, {farthest.Start + farthest.Length}) " + + $"Overlapping columns in {modelName}: \"{farthest.Name}\" [{farthest.Start}, {farthest.Start + farthest.Length}) " + $"and \"{current.Name}\" [{current.Start}, {current.Start + current.Length})."); } int end = current.Start + current.Length; diff --git a/src/FixedWidthParser/Attributes/ModelColumns.cs b/src/FixedWidthParser/Attributes/ModelColumns.cs index ae1658d..9699fa4 100644 --- a/src/FixedWidthParser/Attributes/ModelColumns.cs +++ b/src/FixedWidthParser/Attributes/ModelColumns.cs @@ -28,7 +28,12 @@ public static void ForEachColumn(Type modelType, Action(); diff --git a/src/FixedWidthParser/Delegates.cs b/src/FixedWidthParser/Delegates.cs index d63bce5..169e07b 100644 --- a/src/FixedWidthParser/Delegates.cs +++ b/src/FixedWidthParser/Delegates.cs @@ -1,5 +1,8 @@ namespace FixedWidthParser { - internal delegate void RefAction(ref TModel model, TProperty value) where TModel : allows ref struct; - internal delegate TProperty RefGetter(in TModel model) where TModel : allows ref struct; + /// Assigns a parsed column value to its member of . + public delegate void ColumnSetter(ref TModel model, TValue value) where TModel : allows ref struct; + + /// Reads the value a column is written from. + public delegate TValue ColumnGetter(in TModel model) where TModel : allows ref struct; } diff --git a/src/FixedWidthParser/FixedWidth.cs b/src/FixedWidthParser/FixedWidth.cs index 65445fd..471a7a9 100644 --- a/src/FixedWidthParser/FixedWidth.cs +++ b/src/FixedWidthParser/FixedWidth.cs @@ -17,6 +17,20 @@ public static bool TryParse(ReadOnlySpan line, in FixedWidthContex return TModel.TryParse(line, in context, out model); } + /// + /// Parses a single line, without its line break, into through + /// . Returns when the line is shorter than the layout + /// or a non-string column fails to parse, after reporting why through . + /// + /// has a column without a setter. + public static bool TryParse(ReadOnlySpan line, FixedWidthLayout layout, in FixedWidthContext context, out TModel model) + where TModel : allows ref struct + { + ArgumentNullException.ThrowIfNull(layout); + layout.EnsureCanRead(); + return layout.TryParse(line, in context, out model); + } + /// Formats into , without a line break. public static bool TryFormat(in TModel model, Span destination, in FixedWidthContext context, out int bytesWritten) where TModel : IFixedWidthModel, allows ref struct @@ -24,6 +38,27 @@ public static bool TryFormat(in TModel model, Span destination, in return TModel.TryFormat(in model, destination, in context, out bytesWritten); } + /// + /// Formats into through , + /// without a line break. Returns only when is + /// shorter than the line. + /// + /// has a column without a getter, or a value does not fit a column whose overflow throws. + public static bool TryFormat(in TModel model, FixedWidthLayout layout, Span destination, in FixedWidthContext context, out int bytesWritten) + where TModel : allows ref struct + { + ArgumentNullException.ThrowIfNull(layout); + layout.EnsureCanWrite(); + if (destination.Length < layout.LineLength) + { + bytesWritten = 0; + return false; + } + layout.FormatLine(in model, destination[..layout.LineLength], in context); + bytesWritten = layout.LineLength; + return true; + } + /// Reads from a stream, once. controls whether the stream is disposed when iteration ends. public static FixedWidthRecordEnumerable> Read(Stream stream, bool leaveOpen = false, FixedWidthReaderOptions? options = null) where TModel : IFixedWidthModel diff --git a/src/FixedWidthParser/Formatters/ColumnFormatters.cs b/src/FixedWidthParser/Formatters/ColumnFormatters.cs index 58bb9b1..b101f25 100644 --- a/src/FixedWidthParser/Formatters/ColumnFormatters.cs +++ b/src/FixedWidthParser/Formatters/ColumnFormatters.cs @@ -4,15 +4,16 @@ namespace FixedWidthParser.Formatters { /// Formats one column of a model into its slice of the line, for the runtime writer. internal interface IColumnFormatter + where TModel : allows ref struct { void Format(in TModel model, Span line, in FixedWidthContext context); } - // The formatters are instantiated via reflection (Activator.CreateInstance) in FixedWidthWriter. + // The attribute path instantiates the formatters through reflection (Activator.CreateInstance). #pragma warning disable CA1812 // Avoid uninstantiated internal classes internal sealed class StringColumnFormatter( - int start, int length, ColumnFormatOptions options, string columnName, RefGetter getter) - : IColumnFormatter + int start, int length, ColumnFormatOptions options, string columnName, ColumnGetter getter) + : IColumnFormatter where TModel : allows ref struct { public void Format(in TModel model, Span line, in FixedWidthContext context) { @@ -21,8 +22,9 @@ public void Format(in TModel model, Span line, in FixedWidthContext contex } internal sealed class SpanFormattableColumnFormatter( - int start, int length, ColumnFormatOptions options, string columnName, RefGetter getter) + int start, int length, ColumnFormatOptions options, string columnName, ColumnGetter getter) : IColumnFormatter where TProperty : ISpanFormattable + where TModel : allows ref struct { public void Format(in TModel model, Span line, in FixedWidthContext context) { @@ -31,8 +33,9 @@ public void Format(in TModel model, Span line, in FixedWidthContext contex } internal sealed class Utf8SpanFormattableColumnFormatter( - int start, int length, ColumnFormatOptions options, string columnName, RefGetter getter) + int start, int length, ColumnFormatOptions options, string columnName, ColumnGetter getter) : IColumnFormatter where TProperty : IUtf8SpanFormattable, ISpanFormattable + where TModel : allows ref struct { public void Format(in TModel model, Span line, in FixedWidthContext context) { @@ -41,8 +44,9 @@ public void Format(in TModel model, Span line, in FixedWidthContext contex } internal sealed class ConverterColumnFormatter( - int start, int length, ColumnFormatOptions options, string columnName, RefGetter getter, TConverter converter) + int start, int length, ColumnFormatOptions options, string columnName, ColumnGetter getter, TConverter converter) : IColumnFormatter where TConverter : IFixedWidthConverter + where TModel : allows ref struct { public void Format(in TModel model, Span line, in FixedWidthContext context) { @@ -52,8 +56,9 @@ public void Format(in TModel model, Span line, in FixedWidthContext contex /// A T? column: fills the column with its trim character; a value formats through the T formatter. internal sealed class NullableColumnFormatter( - int start, int length, byte fill, RefGetter getter, IColumnFormatter inner) + int start, int length, byte fill, ColumnGetter getter, IColumnFormatter inner) : IColumnFormatter where TUnderlying : struct + where TModel : allows ref struct { public void Format(in TModel model, Span line, in FixedWidthContext context) { diff --git a/src/FixedWidthParser/Formatters/FormatterBuilder.cs b/src/FixedWidthParser/Formatters/FormatterBuilder.cs new file mode 100644 index 0000000..1921198 --- /dev/null +++ b/src/FixedWidthParser/Formatters/FormatterBuilder.cs @@ -0,0 +1,107 @@ +using System.Linq.Expressions; +using System.Reflection; +using FixedWidthParser.Attributes; +using FixedWidthParser.Processors; + +namespace FixedWidthParser.Formatters +{ + /// Builds the attribute path's column formatters: a compiled getter per [FixedColumn] member, wrapped in the formatter its type needs. + internal static class FormatterBuilder + { + public static IColumnFormatter[] Build() + where TModel : allows ref struct + { + var formatters = new List>(); + ModelColumns.ForEachColumn(typeof(TModel), (member, attribute) => formatters.Add(CreateFormatter(member, attribute))); + return [.. formatters]; + } + + private static IColumnFormatter CreateFormatter(MemberInfo member, FixedColumnAttribute attribute) + where TModel : allows ref struct + { + var (memberType, model, access) = ModelColumns.MemberAccess(typeof(TModel), member); + FixedWidthRuntime.ToAsciiByte(attribute.Padding, member.Name, nameof(FixedColumnAttribute.Padding)); + byte fill = FixedWidthRuntime.ToAsciiByte(attribute.TrimChar, member.Name, nameof(FixedColumnAttribute.TrimChar)); + + if (attribute.Overflow == OverflowBehavior.Truncate && memberType != typeof(string)) + { + throw new InvalidOperationException( + $"OverflowBehavior.Truncate is not supported for non-string column '{member.Name}' of type '{memberType.Name}'."); + } + + var getter = Expression.Lambda(typeof(ColumnGetter<,>).MakeGenericType(typeof(TModel), memberType), access, model).Compile(); + var options = new ColumnFormatOptions(attribute.Alignment, attribute.Padding, attribute.Format, ResolveOverflow(attribute.Overflow, memberType)); + + var underlyingType = Nullable.GetUnderlyingType(memberType); + if (underlyingType is null) + { + return BuildValueFormatter(member, attribute, memberType, options, getter); + } + + var inner = BuildValueFormatter(member, attribute, underlyingType, options, BuildUnderlyingGetter(underlyingType, getter)); + return (IColumnFormatter)Activator.CreateInstance( + typeof(NullableColumnFormatter<,>).MakeGenericType(typeof(TModel), underlyingType), + attribute.Start, attribute.Length, fill, getter, inner)!; + } + + // Default overflow resolves per type: string truncates, everything else throws. + internal static OverflowBehavior ResolveOverflow(OverflowBehavior overflow, Type valueType) + { + if (overflow != OverflowBehavior.Default) + { + return overflow; + } + if (valueType == typeof(string)) + { + return OverflowBehavior.Truncate; + } + return OverflowBehavior.Throw; + } + + // Adapts a ColumnGetter into a ColumnGetter; only called after HasValue is confirmed. + private static Delegate BuildUnderlyingGetter(Type underlyingType, Delegate nullableGetter) + where TModel : allows ref struct + { + var modelParam = Expression.Parameter(typeof(TModel).MakeByRefType(), "model"); + var value = Expression.Property(Expression.Invoke(Expression.Constant(nullableGetter), modelParam), "Value"); + return Expression.Lambda(typeof(ColumnGetter<,>).MakeGenericType(typeof(TModel), underlyingType), value, modelParam).Compile(); + } + + private static IColumnFormatter BuildValueFormatter( + MemberInfo member, FixedColumnAttribute attribute, Type valueType, ColumnFormatOptions options, Delegate getter) + where TModel : allows ref struct + { + if (attribute.Converter is { } converterType) + { + var requiredInterface = typeof(IFixedWidthConverter<>).MakeGenericType(valueType); + if (!requiredInterface.IsAssignableFrom(converterType)) + { + throw new InvalidOperationException( + $"Converter '{converterType}' for column '{member.Name}' must implement '{requiredInterface}'."); + } + var converter = Activator.CreateInstance(converterType) + ?? throw new InvalidOperationException($"Could not instantiate converter '{converterType}' for column '{member.Name}'."); + return (IColumnFormatter)Activator.CreateInstance( + typeof(ConverterColumnFormatter<,,>).MakeGenericType(typeof(TModel), valueType, converterType), + attribute.Start, attribute.Length, options, member.Name, getter, converter)!; + } + + if (valueType == typeof(string)) + { + return new StringColumnFormatter(attribute.Start, attribute.Length, options, member.Name, (ColumnGetter)getter); + } + + if (!typeof(ISpanFormattable).IsAssignableFrom(valueType)) + { + throw new InvalidOperationException( + $"Column '{member.Name}' has type '{valueType}', which is not string or ISpanFormattable; set a Converter for it."); + } + var formatterType = typeof(IUtf8SpanFormattable).IsAssignableFrom(valueType) + ? typeof(Utf8SpanFormattableColumnFormatter<,>) + : typeof(SpanFormattableColumnFormatter<,>); + return (IColumnFormatter)Activator.CreateInstance( + formatterType.MakeGenericType(typeof(TModel), valueType), + attribute.Start, attribute.Length, options, member.Name, getter)!; + } + } +} diff --git a/src/FixedWidthParser/Layouts/FixedWidthLayout.cs b/src/FixedWidthParser/Layouts/FixedWidthLayout.cs new file mode 100644 index 0000000..b997c93 --- /dev/null +++ b/src/FixedWidthParser/Layouts/FixedWidthLayout.cs @@ -0,0 +1,194 @@ +using System.Diagnostics.CodeAnalysis; +using System.Runtime.ExceptionServices; +using FixedWidthParser.Attributes; +using FixedWidthParser.Formatters; +using FixedWidthParser.Parsers; + +namespace FixedWidthParser +{ + /// + /// How maps to the columns of a fixed-width line, for reading and for + /// writing. Immutable and safe to share across threads. Create one with + /// and pass it to the methods. + /// Column positions are bytes. + /// + /// The model type. + public sealed class FixedWidthLayout + where TModel : allows ref struct + { + private readonly ColumnParserInfo[]? _parsers; + private readonly Func? _factory; + private readonly IColumnFormatter[]? _formatters; + private readonly ExceptionDispatchInfo? _readError; + private readonly ExceptionDispatchInfo? _writeError; + + internal FixedWidthLayout(ColumnParserInfo[]? parsers, Func? factory, IColumnFormatter[]? formatters, int lineLength, + ExceptionDispatchInfo? readError = null, ExceptionDispatchInfo? writeError = null) + { + _parsers = parsers; + _factory = factory; + _formatters = formatters; + LineLength = lineLength; + _readError = readError; + _writeError = writeError; + } + + /// The end of the last column: the length of a written line, and the shortest line that parses. + internal int LineLength { get; } + + internal bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TModel model) + { + if (line.Length < LineLength) + { + context.ReportShortLine(LineLength); + model = default!; + return false; + } + + model = _factory!(); + foreach (ref readonly var processor in _parsers.AsSpan()) + { + if (!processor.Parse(line.Slice(processor.Start, processor.Length), in context, ref model)) + { + context.ReportFailedColumn(processor.Name, processor.Start, processor.Length); + return false; + } + } + return true; + } + + internal void FormatLine(in TModel model, Span line, in FixedWidthContext context) + { + line.Fill((byte)' '); + foreach (var formatter in _formatters!) + { + formatter.Format(in model, line, in context); + } + } + + internal void EnsureCanRead() + { + _readError?.Throw(); + if (_parsers is null || _factory is null) + { + throw new InvalidOperationException( + $"The layout for {typeof(TModel).Name} has a column without a setter; it cannot be used to read."); + } + } + + internal void EnsureCanWrite() + { + _writeError?.Throw(); + if (_formatters is null) + { + throw new InvalidOperationException( + $"The layout for {typeof(TModel).Name} has a column without a getter; it cannot be used to write."); + } + } + } + + /// Creates instances. + public static class FixedWidthLayout + { + /// + /// Returns the layout declared by 's [FixedColumn] properties + /// and fields, built once per type by reflection. + /// + /// + /// Not compatible with Native AOT, and trimming can remove the members it binds to. A + /// model's layout reads but cannot write. + /// + /// The layout is invalid: no columns, a negative start, a length below 1, overlapping columns, a non-ASCII padding or trim character, or a column type with no parser or formatter. + [RequiresUnreferencedCode("Attribute layouts reflect over TModel's public properties and fields, which trimming may remove.")] + [RequiresDynamicCode("Attribute layouts compile member accessors at runtime (Expression.Compile / MakeGenericType).")] + public static FixedWidthLayout FromAttributes() + where TModel : new(), allows ref struct + { + AttributeLayout.BuildError?.Throw(); + return AttributeLayout.Layout!; + } + + private static class AttributeLayout + where TModel : new(), allows ref struct + { + internal static readonly FixedWidthLayout? Layout; + internal static readonly ExceptionDispatchInfo? BuildError; + +#pragma warning disable CA1810, S3963 // The try/catch needs a static constructor. + static AttributeLayout() + { + try + { + Layout = Build(); + } +#pragma warning disable CA1031 // Rethrown from FromAttributes on every call. + catch (Exception ex) +#pragma warning restore CA1031 + { + BuildError = ExceptionDispatchInfo.Capture(ex); + } + } +#pragma warning restore CA1810, S3963 + + // A model may be readable and not writable (or the reverse): one side's error waits for that + // side's first use. Only a model neither side can use fails here. + private static FixedWidthLayout Build() + { + var readError = TryBuildReader(out var parsers, out var factory); + IColumnFormatter[]? formatters = null; + var writeError = ExceptionDispatchInfo.Capture(new InvalidOperationException( + $"{typeof(TModel).Name} is a ref struct; an attribute layout can read it but not write it.")); + if (!typeof(TModel).IsByRefLike) + { + writeError = TryBuildWriter(out formatters); + } + if (readError is not null && formatters is null) + { + readError.Throw(); + } + return new(parsers, factory, formatters, LineLength(), readError, writeError); + } + + private static ExceptionDispatchInfo? TryBuildReader(out ColumnParserInfo[]? parsers, out Func? factory) + { + parsers = null; + factory = null; + try + { + parsers = ParserBuilder.BuildProcessors(); + factory = ParserBuilder.BuildModelFactory(); + return null; + } +#pragma warning disable CA1031 // Rethrown when the layout is first used to read. + catch (Exception ex) +#pragma warning restore CA1031 + { + return ExceptionDispatchInfo.Capture(ex); + } + } + + private static ExceptionDispatchInfo? TryBuildWriter(out IColumnFormatter[]? formatters) + { + formatters = null; + try + { + formatters = FormatterBuilder.Build(); + return null; + } +#pragma warning disable CA1031 // Rethrown when the layout is first used to write. + catch (Exception ex) +#pragma warning restore CA1031 + { + return ExceptionDispatchInfo.Capture(ex); + } + } + + private static int LineLength() + { + int lineLength = 0; + ModelColumns.ForEachColumn(typeof(TModel), (_, attribute) => lineLength = Math.Max(lineLength, attribute.Start + attribute.Length)); + return lineLength; + } + } + } +} diff --git a/src/FixedWidthParser/Parsers/FixedWidthParser.cs b/src/FixedWidthParser/Parsers/FixedWidthParser.cs index 95594aa..57e1851 100644 --- a/src/FixedWidthParser/Parsers/FixedWidthParser.cs +++ b/src/FixedWidthParser/Parsers/FixedWidthParser.cs @@ -1,4 +1,4 @@ -using System.Runtime.ExceptionServices; +using System.Diagnostics.CodeAnalysis; namespace FixedWidthParser.Parsers { @@ -9,40 +9,15 @@ namespace FixedWidthParser.Parsers /// public sealed class FixedWidthParser where TModel : new(), allows ref struct { - private static readonly Func _modelFactory; - private static readonly ColumnParserInfo[] _processors; - private static readonly int _requiredLineLength; - private static readonly ExceptionDispatchInfo? _buildError; - - // The build is static (once per type). Any layout/configuration error is captured and - // rethrown from the instance constructor, so the caller gets a clean exception instead of - // a TypeInitializationException on first use. -#pragma warning disable CA1810, S3963 // The try/catch needs a static constructor. - static FixedWidthParser() - { - try - { - _modelFactory = ParserBuilder.BuildModelFactory(); - _processors = ParserBuilder.BuildProcessors(); - _requiredLineLength = ParserBuilder.ComputeRequiredLineLength(_processors); - _buildError = null; - } -#pragma warning disable CA1031 // Rethrown from the instance constructor. - catch (Exception ex) -#pragma warning restore CA1031 - { - _modelFactory = null!; - _processors = []; - _requiredLineLength = 0; - _buildError = ExceptionDispatchInfo.Capture(ex); - } - } -#pragma warning restore CA1810, S3963 + private readonly FixedWidthLayout _layout; /// Builds the parser, throwing when the model's layout is invalid. + [RequiresUnreferencedCode("Attribute layouts reflect over TModel's public properties and fields, which trimming may remove.")] + [RequiresDynamicCode("Attribute layouts compile member accessors at runtime (Expression.Compile / MakeGenericType).")] public FixedWidthParser() { - _buildError?.Throw(); + _layout = FixedWidthLayout.FromAttributes(); + _layout.EnsureCanRead(); } /// @@ -52,23 +27,7 @@ public FixedWidthParser() /// public bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TModel model) { - if (line.Length < _requiredLineLength) - { - context.ReportShortLine(_requiredLineLength); - model = default!; - return false; - } - - model = _modelFactory(); - foreach (ref readonly var processor in _processors.AsSpan()) - { - if (!processor.Parse(line.Slice(processor.Start, processor.Length), in context, ref model)) - { - context.ReportFailedColumn(processor.Name, processor.Start, processor.Length); - return false; - } - } - return true; + return _layout.TryParse(line, in context, out model); } } } diff --git a/src/FixedWidthParser/Parsers/ParserBuilder.cs b/src/FixedWidthParser/Parsers/ParserBuilder.cs index 79ed001..47f03fa 100644 --- a/src/FixedWidthParser/Parsers/ParserBuilder.cs +++ b/src/FixedWidthParser/Parsers/ParserBuilder.cs @@ -26,7 +26,7 @@ public static ColumnParserInfo[] BuildProcessors() where TModel { var (memberType, model, access) = ModelColumns.MemberAccess(typeof(TModel), member); var value = Expression.Parameter(memberType, "value"); - var actionType = typeof(RefAction<,>).MakeGenericType(typeof(TModel), memberType); + var actionType = typeof(ColumnSetter<,>).MakeGenericType(typeof(TModel), memberType); var setter = Expression.Lambda(actionType, Expression.Assign(access, value), model, value).Compile(); var parse = ColumnParserFactory.Create(memberType, setter, attribute.Converter, member.Name, attribute.TrimChar, attribute.TrimMode, attribute.Format); processors.Add(new(attribute.Start, attribute.Length, member.Name, parse)); diff --git a/src/FixedWidthParser/Processors/ColumnParserFactory.cs b/src/FixedWidthParser/Processors/ColumnParserFactory.cs index 413d6c7..96312a8 100644 --- a/src/FixedWidthParser/Processors/ColumnParserFactory.cs +++ b/src/FixedWidthParser/Processors/ColumnParserFactory.cs @@ -48,33 +48,33 @@ private static ColumnParser CreateValue(Type valueType, Delegate { if (valueType == typeof(DateTime)) { - return BuildDateTime((RefAction)setter, format, trim, trimMode); + return BuildDateTime((ColumnSetter)setter, format, trim, trimMode); } if (valueType == typeof(DateOnly)) { - return BuildDateOnly((RefAction)setter, format, trim, trimMode); + return BuildDateOnly((ColumnSetter)setter, format, trim, trimMode); } if (valueType == typeof(TimeOnly)) { - return BuildTimeOnly((RefAction)setter, format, trim, trimMode); + return BuildTimeOnly((ColumnSetter)setter, format, trim, trimMode); } if (valueType == typeof(DateTimeOffset)) { - return BuildDateTimeOffset((RefAction)setter, format, trim, trimMode); + return BuildDateTimeOffset((ColumnSetter)setter, format, trim, trimMode); } } if (valueType == typeof(string)) { - return BuildString((RefAction)setter, trim, trimMode); + return BuildString((ColumnSetter)setter, trim, trimMode); } if (valueType == typeof(double)) { - return BuildDouble((RefAction)setter, trim, trimMode); + return BuildDouble((ColumnSetter)setter, trim, trimMode); } if (valueType == typeof(float)) { - return BuildFloat((RefAction)setter, trim, trimMode); + return BuildFloat((ColumnSetter)setter, trim, trimMode); } if (Implements(valueType, typeof(IUtf8SpanParsable<>))) { @@ -106,8 +106,8 @@ private static ColumnParser BuildNullable(Delegate where TModel : allows ref struct where TUnderlying : struct { - var nullableSetter = (RefAction)setter; - RefAction adapted = (ref TModel model, TUnderlying value) => nullableSetter(ref model, value); + var nullableSetter = (ColumnSetter)setter; + ColumnSetter adapted = (ref TModel model, TUnderlying value) => nullableSetter(ref model, value); var inner = CreateValue(typeof(TUnderlying), adapted, converterType, memberName, trim, trimMode, format); return (column, in context, ref model) => @@ -121,7 +121,7 @@ private static ColumnParser BuildNullable(Delegate }; } - private static ColumnParser BuildUtf8Parsable(RefAction setter, byte trim, TrimMode trimMode) + private static ColumnParser BuildUtf8Parsable(ColumnSetter setter, byte trim, TrimMode trimMode) where TModel : allows ref struct where TValue : IUtf8SpanParsable { @@ -136,7 +136,7 @@ private static ColumnParser BuildUtf8Parsable(RefAction< }; } - private static ColumnParser BuildDecodedParsable(RefAction setter, byte trim, TrimMode trimMode) + private static ColumnParser BuildDecodedParsable(ColumnSetter setter, byte trim, TrimMode trimMode) where TModel : allows ref struct where TValue : ISpanParsable { @@ -151,7 +151,7 @@ private static ColumnParser BuildDecodedParsable(RefActi }; } - private static ColumnParser BuildConverter(RefAction setter, TConverter converter, byte trim, TrimMode trimMode) + private static ColumnParser BuildConverter(ColumnSetter setter, TConverter converter, byte trim, TrimMode trimMode) where TModel : allows ref struct where TConverter : IFixedWidthConverter { @@ -166,7 +166,7 @@ private static ColumnParser BuildConverter(R }; } - private static ColumnParser BuildString(RefAction setter, byte trim, TrimMode trimMode) + private static ColumnParser BuildString(ColumnSetter setter, byte trim, TrimMode trimMode) where TModel : allows ref struct { return (column, in context, ref model) => @@ -176,7 +176,7 @@ private static ColumnParser BuildString(RefAction BuildDouble(RefAction setter, byte trim, TrimMode trimMode) + private static ColumnParser BuildDouble(ColumnSetter setter, byte trim, TrimMode trimMode) where TModel : allows ref struct { return (column, in context, ref model) => @@ -190,7 +190,7 @@ private static ColumnParser BuildDouble(RefAction BuildFloat(RefAction setter, byte trim, TrimMode trimMode) + private static ColumnParser BuildFloat(ColumnSetter setter, byte trim, TrimMode trimMode) where TModel : allows ref struct { return (column, in context, ref model) => @@ -204,7 +204,7 @@ private static ColumnParser BuildFloat(RefAction }; } - private static ColumnParser BuildDateTime(RefAction setter, string format, byte trim, TrimMode trimMode) + private static ColumnParser BuildDateTime(ColumnSetter setter, string format, byte trim, TrimMode trimMode) where TModel : allows ref struct { return (column, in context, ref model) => @@ -218,7 +218,7 @@ private static ColumnParser BuildDateTime(RefAction BuildDateOnly(RefAction setter, string format, byte trim, TrimMode trimMode) + private static ColumnParser BuildDateOnly(ColumnSetter setter, string format, byte trim, TrimMode trimMode) where TModel : allows ref struct { return (column, in context, ref model) => @@ -232,7 +232,7 @@ private static ColumnParser BuildDateOnly(RefAction BuildTimeOnly(RefAction setter, string format, byte trim, TrimMode trimMode) + private static ColumnParser BuildTimeOnly(ColumnSetter setter, string format, byte trim, TrimMode trimMode) where TModel : allows ref struct { return (column, in context, ref model) => @@ -246,7 +246,7 @@ private static ColumnParser BuildTimeOnly(RefAction BuildDateTimeOffset(RefAction setter, string format, byte trim, TrimMode trimMode) + private static ColumnParser BuildDateTimeOffset(ColumnSetter setter, string format, byte trim, TrimMode trimMode) where TModel : allows ref struct { return (column, in context, ref model) => diff --git a/src/FixedWidthParser/PublicAPI.Unshipped.txt b/src/FixedWidthParser/PublicAPI.Unshipped.txt index b95b012..bc6ebd4 100644 --- a/src/FixedWidthParser/PublicAPI.Unshipped.txt +++ b/src/FixedWidthParser/PublicAPI.Unshipped.txt @@ -175,3 +175,12 @@ FixedWidthParser.Readers.FixedWidthReader.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! FixedWidthParser.Readers.FixedWidthReader.AggregateAsync(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! static FixedWidthParser.FixedWidth.AggregateAsync(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +FixedWidthParser.ColumnGetter +FixedWidthParser.ColumnSetter +FixedWidthParser.FixedWidthLayout +FixedWidthParser.FixedWidthLayout +static FixedWidthParser.FixedWidth.TryFormat(in TModel model, FixedWidthParser.FixedWidthLayout! layout, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int bytesWritten) -> bool +static FixedWidthParser.FixedWidth.TryParse(System.ReadOnlySpan line, FixedWidthParser.FixedWidthLayout! layout, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool +static FixedWidthParser.FixedWidthLayout.FromAttributes() -> FixedWidthParser.FixedWidthLayout! +virtual FixedWidthParser.ColumnGetter.Invoke(in TModel model) -> TValue +virtual FixedWidthParser.ColumnSetter.Invoke(ref TModel model, TValue value) -> void diff --git a/src/FixedWidthParser/Writers/FixedWidthWriter.cs b/src/FixedWidthParser/Writers/FixedWidthWriter.cs index 4cf8eea..4e916ee 100644 --- a/src/FixedWidthParser/Writers/FixedWidthWriter.cs +++ b/src/FixedWidthParser/Writers/FixedWidthWriter.cs @@ -1,62 +1,30 @@ using System.Buffers; -using System.Linq.Expressions; -using System.Reflection; +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; -using System.Runtime.ExceptionServices; -using FixedWidthParser.Attributes; -using FixedWidthParser.Formatters; -using FixedWidthParser.Processors; namespace FixedWidthParser.Writers { /// /// Writes attribute-mapped models as fixed-width lines of bytes through the runtime /// (reflection-built) formatter. Batches lines into one pooled buffer, so writing many records - /// allocates nothing per record. For source-generated models use . + /// allocates nothing per record. For source-generated models use . /// Column positions are bytes. /// - public sealed class FixedWidthWriter + public sealed class FixedWidthWriter where TModel : new() { private const int BatchSize = 64 * 1024; - private static readonly IColumnFormatter[] _formatters; - private static readonly int _lineLength; - private static readonly ExceptionDispatchInfo? _buildError; - -#pragma warning disable CA1810, S3963 // The try/catch needs a static constructor. - static FixedWidthWriter() - { - try - { - var formatters = new List>(); - int lineLength = 0; - ModelColumns.ForEachColumn(typeof(TModel), (member, attribute) => - { - lineLength = Math.Max(lineLength, attribute.Start + attribute.Length); - formatters.Add(CreateFormatter(member, attribute)); - }); - _formatters = [.. formatters]; - _lineLength = lineLength; - _buildError = null; - } -#pragma warning disable CA1031 // Rethrown from the instance constructor. - catch (Exception ex) -#pragma warning restore CA1031 - { - _formatters = []; - _lineLength = 0; - _buildError = ExceptionDispatchInfo.Capture(ex); - } - } -#pragma warning restore CA1810, S3963 - + private readonly FixedWidthLayout _layout; private readonly FixedWidthContext _context; private readonly byte[] _newLine; /// Creates a writer; options use every default. + [RequiresUnreferencedCode("Attribute layouts reflect over TModel's public properties and fields, which trimming may remove.")] + [RequiresDynamicCode("Attribute layouts compile member accessors at runtime (Expression.Compile / MakeGenericType).")] public FixedWidthWriter(FixedWidthWriterOptions? options = null) { - _buildError?.Throw(); + _layout = FixedWidthLayout.FromAttributes(); + _layout.EnsureCanWrite(); options ??= FixedWidthWriterOptions.Default; if (options.NewLine is null) { @@ -66,7 +34,9 @@ public FixedWidthWriter(FixedWidthWriterOptions? options = null) _newLine = _context.Encoding.GetBytes(options.NewLine); } - private int RecordLength => _lineLength + _newLine.Length; + private int LineLength => _layout.LineLength; + + private int RecordLength => LineLength + _newLine.Length; /// /// Formats into , without a line break. @@ -74,13 +44,13 @@ public FixedWidthWriter(FixedWidthWriterOptions? options = null) /// public bool TryFormat(in TModel model, Span destination, out int bytesWritten) { - if (destination.Length < _lineLength) + if (destination.Length < LineLength) { bytesWritten = 0; return false; } - FormatLine(in model, destination[.._lineLength]); - bytesWritten = _lineLength; + FormatLine(in model, destination[..LineLength]); + bytesWritten = LineLength; return true; } @@ -205,95 +175,14 @@ private int Append(Stream stream, byte[] buffer, int used, in TModel model) private int FormatRecord(in TModel model, Span destination) { - FormatLine(in model, destination[.._lineLength]); - _newLine.CopyTo(destination[_lineLength..]); + FormatLine(in model, destination[..LineLength]); + _newLine.CopyTo(destination[LineLength..]); return RecordLength; } private void FormatLine(in TModel model, Span line) { - line.Fill((byte)' '); - foreach (var formatter in _formatters) - { - formatter.Format(in model, line, in _context); - } - } - - private static IColumnFormatter CreateFormatter(MemberInfo member, FixedColumnAttribute attribute) - { - var (memberType, model, access) = ModelColumns.MemberAccess(typeof(TModel), member); - FixedWidthRuntime.ToAsciiByte(attribute.Padding, member.Name, nameof(FixedColumnAttribute.Padding)); - byte fill = FixedWidthRuntime.ToAsciiByte(attribute.TrimChar, member.Name, nameof(FixedColumnAttribute.TrimChar)); - - if (attribute.Overflow == OverflowBehavior.Truncate && memberType != typeof(string)) - { - throw new InvalidOperationException( - $"OverflowBehavior.Truncate is not supported for non-string column '{member.Name}' of type '{memberType.Name}'."); - } - - var getter = Expression.Lambda(typeof(RefGetter<,>).MakeGenericType(typeof(TModel), memberType), access, model).Compile(); - // Default overflow resolves per type: string truncates, everything else throws. - var overflow = attribute.Overflow; - if (overflow == OverflowBehavior.Default) - { - overflow = memberType == typeof(string) ? OverflowBehavior.Truncate : OverflowBehavior.Throw; - } - var options = new ColumnFormatOptions(attribute.Alignment, attribute.Padding, attribute.Format, overflow); - - var underlyingType = Nullable.GetUnderlyingType(memberType); - if (underlyingType is null) - { - return BuildValueFormatter(member, attribute, memberType, options, getter); - } - - var inner = BuildValueFormatter(member, attribute, underlyingType, options, BuildUnderlyingGetter(underlyingType, getter)); - return (IColumnFormatter)Activator.CreateInstance( - typeof(NullableColumnFormatter<,>).MakeGenericType(typeof(TModel), underlyingType), - attribute.Start, attribute.Length, fill, getter, inner)!; - } - - // Adapts a RefGetter into a RefGetter; only called after HasValue is confirmed. - private static Delegate BuildUnderlyingGetter(Type underlyingType, Delegate nullableGetter) - { - var modelParam = Expression.Parameter(typeof(TModel).MakeByRefType(), "model"); - var value = Expression.Property(Expression.Invoke(Expression.Constant(nullableGetter), modelParam), "Value"); - return Expression.Lambda(typeof(RefGetter<,>).MakeGenericType(typeof(TModel), underlyingType), value, modelParam).Compile(); - } - - private static IColumnFormatter BuildValueFormatter( - MemberInfo member, FixedColumnAttribute attribute, Type valueType, ColumnFormatOptions options, Delegate getter) - { - if (attribute.Converter is { } converterType) - { - var requiredInterface = typeof(IFixedWidthConverter<>).MakeGenericType(valueType); - if (!requiredInterface.IsAssignableFrom(converterType)) - { - throw new InvalidOperationException( - $"Converter '{converterType}' for column '{member.Name}' must implement '{requiredInterface}'."); - } - var converter = Activator.CreateInstance(converterType) - ?? throw new InvalidOperationException($"Could not instantiate converter '{converterType}' for column '{member.Name}'."); - return (IColumnFormatter)Activator.CreateInstance( - typeof(ConverterColumnFormatter<,,>).MakeGenericType(typeof(TModel), valueType, converterType), - attribute.Start, attribute.Length, options, member.Name, getter, converter)!; - } - - if (valueType == typeof(string)) - { - return new StringColumnFormatter(attribute.Start, attribute.Length, options, member.Name, (RefGetter)getter); - } - - if (!typeof(ISpanFormattable).IsAssignableFrom(valueType)) - { - throw new InvalidOperationException( - $"Column '{member.Name}' has type '{valueType}', which is not string or ISpanFormattable; set a Converter for it."); - } - var formatterType = typeof(IUtf8SpanFormattable).IsAssignableFrom(valueType) - ? typeof(Utf8SpanFormattableColumnFormatter<,>) - : typeof(SpanFormattableColumnFormatter<,>); - return (IColumnFormatter)Activator.CreateInstance( - formatterType.MakeGenericType(typeof(TModel), valueType), - attribute.Start, attribute.Length, options, member.Name, getter)!; + _layout.FormatLine(in model, line, in _context); } } } diff --git a/tests/FixedWidthParser.Tests/LayoutTests.cs b/tests/FixedWidthParser.Tests/LayoutTests.cs new file mode 100644 index 0000000..64038a0 --- /dev/null +++ b/tests/FixedWidthParser.Tests/LayoutTests.cs @@ -0,0 +1,55 @@ +using System.Text; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// A layout built from attributes parses and formats through the overloads. + public class LayoutTests + { + [Fact] + public void FromAttributes_TryParse_MatchesParser() + { + var layout = FixedWidthLayout.FromAttributes(); + + Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, layout, Ctx, out var p)); + Assert.Equal(("John Doe", 30, 6000000.00), (p.Name, p.Age, p.Salary)); + } + + [Fact] + public void FromAttributes_TryFormat_WritesLine() + { + var layout = FixedWidthLayout.FromAttributes(); + Span buffer = stackalloc byte[25]; + + Assert.True(FixedWidth.TryFormat(new PersonModel { Name = "Ann", Age = 7, Salary = 1.5 }, layout, buffer, Ctx, out int written)); + Assert.Equal("Ann 7 1.5 ", Encoding.UTF8.GetString(buffer[..written])); + Assert.False(FixedWidth.TryFormat(new PersonModel(), layout, stackalloc byte[24], Ctx, out _)); + } + + [Fact] + public void FromAttributes_IsCachedPerType() + { + Assert.Same(FixedWidthLayout.FromAttributes(), FixedWidthLayout.FromAttributes()); + } + + [Fact] + public void FromAttributes_InvalidLayout_ThrowsEveryCall() + { + for (int i = 0; i < 2; i++) + { + var ex = Assert.Throws(() => FixedWidthLayout.FromAttributes()); + Assert.Contains("Overlapping", ex.Message, StringComparison.Ordinal); + } + } + + [Fact] + public void FromAttributes_RefStruct_ParsesButCannotFormat() + { + var layout = FixedWidthLayout.FromAttributes(); + + Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, layout, Ctx, out var r)); + Assert.Equal(30, r.Age); + Assert.Throws(() => FixedWidth.TryFormat(new RefPersonModel(), layout, new byte[25], Ctx, out _)); + } + } +} diff --git a/tests/FixedWidthParser.Tests/TestHelpers.cs b/tests/FixedWidthParser.Tests/TestHelpers.cs index 3cd1c23..b236964 100644 --- a/tests/FixedWidthParser.Tests/TestHelpers.cs +++ b/tests/FixedWidthParser.Tests/TestHelpers.cs @@ -36,28 +36,28 @@ public static string Format(in T model, in FixedWidthContext context = defaul } /// Writes a model and returns the line without the trailing newline. - public static string WriteOne(FixedWidthWriter writer, in T model) + public static string WriteOne(FixedWidthWriter writer, in T model) where T : new() { using var ms = new MemoryStream(); writer.Write(ms, in model); return Decode(ms).TrimEnd('\r', '\n'); } - public static async Task WriteOneAsync(FixedWidthWriter writer, T model) + public static async Task WriteOneAsync(FixedWidthWriter writer, T model) where T : new() { await using var ms = new MemoryStream(); await writer.WriteAsync(ms, model, TestContext.Current.CancellationToken).ConfigureAwait(false); return Decode(ms).TrimEnd('\r', '\n'); } - public static string WriteMany(FixedWidthWriter writer, ReadOnlySpan models) + public static string WriteMany(FixedWidthWriter writer, ReadOnlySpan models) where T : new() { using var ms = new MemoryStream(); writer.WriteMany(ms, models); return Decode(ms); } - public static string WriteMany(FixedWidthWriter writer, IEnumerable models) + public static string WriteMany(FixedWidthWriter writer, IEnumerable models) where T : new() { using var ms = new MemoryStream(); writer.WriteMany(ms, models); From 0eb52a70459f8fa777fe6942ee47f2a46fca9355 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 23 Oct 2026 22:22:26 -0300 Subject: [PATCH 18/85] feat(reader): read and aggregate through a FixedWidthLayout FixedWidth.Read, ReadFile, ReadAsync, ReadFileAsync, AggregateFileAsync and AggregateAsync gain a layout overload next to each generated one. A LayoutLineParser strategy replaces ReflectionLineParser, so the enumerators and both parallel aggregators are unchanged. --- src/FixedWidthParser/FixedWidth.cs | 70 +++++++++++++++ src/FixedWidthParser/PublicAPI.Unshipped.txt | 20 +++-- .../Readers/FixedWidthReader.cs | 34 +++---- src/FixedWidthParser/Readers/ILineParser.cs | 14 ++- .../FixedWidthParser.Tests/LayoutReadTests.cs | 90 +++++++++++++++++++ tests/FixedWidthParser.Tests/ReaderTests.cs | 2 +- 6 files changed, 199 insertions(+), 31 deletions(-) create mode 100644 tests/FixedWidthParser.Tests/LayoutReadTests.cs diff --git a/src/FixedWidthParser/FixedWidth.cs b/src/FixedWidthParser/FixedWidth.cs index 471a7a9..97a3d69 100644 --- a/src/FixedWidthParser/FixedWidth.cs +++ b/src/FixedWidthParser/FixedWidth.cs @@ -67,6 +67,14 @@ public static FixedWidthRecordEnumerable> Re return new(default, stream, ownsStream: !leaveOpen, FixedWidthReaderOptions.Validate(options)); } + /// Reads from a stream through , once. controls whether the stream is disposed when iteration ends. + /// has a column without a setter. + public static FixedWidthRecordEnumerable> Read(Stream stream, FixedWidthLayout layout, bool leaveOpen = false, FixedWidthReaderOptions? options = null) + { + ArgumentNullException.ThrowIfNull(stream); + return new(ReadableLayout(layout), stream, ownsStream: !leaveOpen, FixedWidthReaderOptions.Validate(options)); + } + /// Reads from a file, reopening it on each enumeration. public static FixedWidthRecordEnumerable> ReadFile(string path, FixedWidthReaderOptions? options = null) where TModel : IFixedWidthModel @@ -75,6 +83,14 @@ public static FixedWidthRecordEnumerable> Re return new(default, path, FixedWidthReaderOptions.Validate(options)); } + /// Reads from a file through , reopening it on each enumeration. + /// has a column without a setter. + public static FixedWidthRecordEnumerable> ReadFile(string path, FixedWidthLayout layout, FixedWidthReaderOptions? options = null) + { + ArgumentException.ThrowIfNullOrEmpty(path); + return new(ReadableLayout(layout), path, FixedWidthReaderOptions.Validate(options)); + } + /// Reads from a stream via await foreach, once. controls whether the stream is disposed when iteration ends. public static FixedWidthAsyncRecordEnumerable> ReadAsync(Stream stream, bool leaveOpen = false, FixedWidthReaderOptions? options = null) where TModel : IFixedWidthModel @@ -83,6 +99,14 @@ public static FixedWidthAsyncRecordEnumerableReads from a stream through via await foreach, once. controls whether the stream is disposed when iteration ends. + /// has a column without a setter. + public static FixedWidthAsyncRecordEnumerable> ReadAsync(Stream stream, FixedWidthLayout layout, bool leaveOpen = false, FixedWidthReaderOptions? options = null) + { + ArgumentNullException.ThrowIfNull(stream); + return new(ReadableLayout(layout), stream, ownsStream: !leaveOpen, FixedWidthReaderOptions.Validate(options)); + } + /// Reads from a file via await foreach with asynchronous I/O, reopening it on each enumeration. public static FixedWidthAsyncRecordEnumerable> ReadFileAsync(string path, FixedWidthReaderOptions? options = null) where TModel : IFixedWidthModel @@ -91,6 +115,14 @@ public static FixedWidthAsyncRecordEnumerableReads from a file through via await foreach with asynchronous I/O, reopening it on each enumeration. + /// has a column without a setter. + public static FixedWidthAsyncRecordEnumerable> ReadFileAsync(string path, FixedWidthLayout layout, FixedWidthReaderOptions? options = null) + { + ArgumentException.ThrowIfNullOrEmpty(path); + return new(ReadableLayout(layout), path, FixedWidthReaderOptions.Validate(options)); + } + /// /// Reads a file across several threads and folds every record into a , /// one instance per partition, merged in file order. @@ -106,6 +138,21 @@ public static Task AggregateFileAsync( default, path, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), ct); } + /// + /// Reads a file through across several threads and folds every record into + /// a , one instance per partition, merged in file order. + /// + /// Carries the same contract as . + /// has a column without a setter. + public static Task AggregateFileAsync( + string path, FixedWidthLayout layout, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken ct = default) + where TAccumulator : IRecordAccumulator, new() + { + ArgumentException.ThrowIfNullOrEmpty(path); + return ParallelAggregator.RunAsync, TAccumulator>( + ReadableLayout(layout), path, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), ct); + } + /// /// Reads a stream once and folds every record into a across /// several threads. controls whether the stream is disposed afterwards. @@ -120,5 +167,28 @@ public static Task AggregateAsync( return ParallelStreamAggregator.RunAsync, TAccumulator>( default, stream, leaveOpen, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), ct); } + + /// + /// Reads a stream once through and folds every record into a + /// across several threads. controls + /// whether the stream is disposed afterwards. + /// + /// Carries the same contract as . + /// has a column without a setter. + public static Task AggregateAsync( + Stream stream, FixedWidthLayout layout, bool leaveOpen = false, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken ct = default) + where TAccumulator : IRecordAccumulator, new() + { + ArgumentNullException.ThrowIfNull(stream); + return ParallelStreamAggregator.RunAsync, TAccumulator>( + ReadableLayout(layout), stream, leaveOpen, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), ct); + } + + private static LayoutLineParser ReadableLayout(FixedWidthLayout layout) + { + ArgumentNullException.ThrowIfNull(layout); + layout.EnsureCanRead(); + return new LayoutLineParser(layout); + } } } diff --git a/src/FixedWidthParser/PublicAPI.Unshipped.txt b/src/FixedWidthParser/PublicAPI.Unshipped.txt index bc6ebd4..4f8144f 100644 --- a/src/FixedWidthParser/PublicAPI.Unshipped.txt +++ b/src/FixedWidthParser/PublicAPI.Unshipped.txt @@ -103,10 +103,6 @@ FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable.GetAsyncEnumerator(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> FixedWidthParser.Readers.AsyncRecordEnumerator! FixedWidthParser.Readers.FixedWidthReader FixedWidthParser.Readers.FixedWidthReader.FixedWidthReader(FixedWidthParser.FixedWidthReaderOptions? options = null) -> void -FixedWidthParser.Readers.FixedWidthReader.Read(System.IO.Stream! stream, bool leaveOpen = false) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! -FixedWidthParser.Readers.FixedWidthReader.ReadAsync(System.IO.Stream! stream, bool leaveOpen = false) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! -FixedWidthParser.Readers.FixedWidthReader.ReadFile(string! path) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! -FixedWidthParser.Readers.FixedWidthReader.ReadFileAsync(string! path) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! FixedWidthParser.Readers.FixedWidthReader.TryParse(System.ReadOnlySpan line, out TModel model) -> bool FixedWidthParser.Readers.FixedWidthRecordEnumerable FixedWidthParser.Readers.FixedWidthRecordEnumerable.GetEnumerator() -> FixedWidthParser.Readers.RecordEnumerator @@ -121,9 +117,6 @@ FixedWidthParser.Readers.RecordEnumerator.Dispose() -> void FixedWidthParser.Readers.RecordEnumerator.MoveNext() -> bool FixedWidthParser.Readers.RecordEnumerator.RecordEnumerator() -> void FixedWidthParser.Readers.RecordEnumerator.Reset() -> void -FixedWidthParser.Readers.ReflectionLineParser -FixedWidthParser.Readers.ReflectionLineParser.ReflectionLineParser() -> void -FixedWidthParser.Readers.ReflectionLineParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool FixedWidthParser.Writers.FixedWidthWriter FixedWidthParser.Writers.FixedWidthWriter.FixedWidthWriter(FixedWidthParser.FixedWidthWriterOptions? options = null) -> void FixedWidthParser.Writers.FixedWidthWriter.TryFormat(in TModel model, System.Span destination, out int bytesWritten) -> bool @@ -184,3 +177,16 @@ static FixedWidthParser.FixedWidth.TryParse(System.ReadOnlySpan li static FixedWidthParser.FixedWidthLayout.FromAttributes() -> FixedWidthParser.FixedWidthLayout! virtual FixedWidthParser.ColumnGetter.Invoke(in TModel model) -> TValue virtual FixedWidthParser.ColumnSetter.Invoke(ref TModel model, TValue value) -> void +FixedWidthParser.Readers.FixedWidthReader.Read(System.IO.Stream! stream, bool leaveOpen = false) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! +FixedWidthParser.Readers.FixedWidthReader.ReadAsync(System.IO.Stream! stream, bool leaveOpen = false) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! +FixedWidthParser.Readers.FixedWidthReader.ReadFile(string! path) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! +FixedWidthParser.Readers.FixedWidthReader.ReadFileAsync(string! path) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! +FixedWidthParser.Readers.LayoutLineParser +FixedWidthParser.Readers.LayoutLineParser.LayoutLineParser() -> void +FixedWidthParser.Readers.LayoutLineParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool +static FixedWidthParser.FixedWidth.AggregateAsync(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.Read(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadAsync(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadFile(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadFileAsync(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! diff --git a/src/FixedWidthParser/Readers/FixedWidthReader.cs b/src/FixedWidthParser/Readers/FixedWidthReader.cs index 2c123bf..9038c41 100644 --- a/src/FixedWidthParser/Readers/FixedWidthReader.cs +++ b/src/FixedWidthParser/Readers/FixedWidthReader.cs @@ -1,4 +1,4 @@ -using FixedWidthParser.Parsers; +using System.Diagnostics.CodeAnalysis; namespace FixedWidthParser.Readers { @@ -10,13 +10,17 @@ namespace FixedWidthParser.Readers /// public sealed class FixedWidthReader where TModel : new() { - private readonly FixedWidthParser _parser = new(); + private readonly FixedWidthLayout _layout; private readonly FixedWidthReaderOptions _options; private readonly FixedWidthContext _context; /// Creates a reader; options use every default. + [RequiresUnreferencedCode("Attribute layouts reflect over TModel's public properties and fields, which trimming may remove.")] + [RequiresDynamicCode("Attribute layouts compile member accessors at runtime (Expression.Compile / MakeGenericType).")] public FixedWidthReader(FixedWidthReaderOptions? options = null) { + _layout = FixedWidthLayout.FromAttributes(); + _layout.EnsureCanRead(); _options = FixedWidthReaderOptions.Validate(options); _context = _options.Context; } @@ -24,35 +28,35 @@ public FixedWidthReader(FixedWidthReaderOptions? options = null) /// Parses a single line, without its line break, using the reader's options. public bool TryParse(ReadOnlySpan line, out TModel model) { - return _parser.TryParse(line, in _context, out model); + return _layout.TryParse(line, in _context, out model); } /// Reads from a stream, once. controls whether the stream is disposed when iteration ends. - public FixedWidthRecordEnumerable> Read(Stream stream, bool leaveOpen = false) + public FixedWidthRecordEnumerable> Read(Stream stream, bool leaveOpen = false) { ArgumentNullException.ThrowIfNull(stream); - return new(new ReflectionLineParser(_parser), stream, ownsStream: !leaveOpen, _options); + return new(new LayoutLineParser(_layout), stream, ownsStream: !leaveOpen, _options); } /// Reads from a file, reopening it on each enumeration. - public FixedWidthRecordEnumerable> ReadFile(string path) + public FixedWidthRecordEnumerable> ReadFile(string path) { ArgumentException.ThrowIfNullOrEmpty(path); - return new(new ReflectionLineParser(_parser), path, _options); + return new(new LayoutLineParser(_layout), path, _options); } /// Reads from a stream via await foreach, once. controls whether the stream is disposed when iteration ends. - public FixedWidthAsyncRecordEnumerable> ReadAsync(Stream stream, bool leaveOpen = false) + public FixedWidthAsyncRecordEnumerable> ReadAsync(Stream stream, bool leaveOpen = false) { ArgumentNullException.ThrowIfNull(stream); - return new(new ReflectionLineParser(_parser), stream, ownsStream: !leaveOpen, _options); + return new(new LayoutLineParser(_layout), stream, ownsStream: !leaveOpen, _options); } /// Reads from a file via await foreach with asynchronous I/O, reopening it on each enumeration. - public FixedWidthAsyncRecordEnumerable> ReadFileAsync(string path) + public FixedWidthAsyncRecordEnumerable> ReadFileAsync(string path) { ArgumentException.ThrowIfNullOrEmpty(path); - return new(new ReflectionLineParser(_parser), path, _options); + return new(new LayoutLineParser(_layout), path, _options); } /// @@ -69,8 +73,8 @@ public Task AggregateFileAsync(string path, FixedWid where TAccumulator : IRecordAccumulator, new() { ArgumentException.ThrowIfNullOrEmpty(path); - return ParallelAggregator.RunAsync, TAccumulator>( - new ReflectionLineParser(_parser), path, _options, FixedWidthParallelOptions.Validate(parallel), ct); + return ParallelAggregator.RunAsync, TAccumulator>( + new LayoutLineParser(_layout), path, _options, FixedWidthParallelOptions.Validate(parallel), ct); } /// @@ -84,8 +88,8 @@ public Task AggregateAsync(Stream stream, bool leave where TAccumulator : IRecordAccumulator, new() { ArgumentNullException.ThrowIfNull(stream); - return ParallelStreamAggregator.RunAsync, TAccumulator>( - new ReflectionLineParser(_parser), stream, leaveOpen, _options, FixedWidthParallelOptions.Validate(parallel), ct); + return ParallelStreamAggregator.RunAsync, TAccumulator>( + new LayoutLineParser(_layout), stream, leaveOpen, _options, FixedWidthParallelOptions.Validate(parallel), ct); } } } diff --git a/src/FixedWidthParser/Readers/ILineParser.cs b/src/FixedWidthParser/Readers/ILineParser.cs index 12011f7..0fcb850 100644 --- a/src/FixedWidthParser/Readers/ILineParser.cs +++ b/src/FixedWidthParser/Readers/ILineParser.cs @@ -1,5 +1,3 @@ -using FixedWidthParser.Parsers; - namespace FixedWidthParser.Readers { /// @@ -17,20 +15,20 @@ public interface IRecordLineParser // Strategy structs are plumbing for the generic enumerators; they are never compared, so // value equality is meaningless for them. #pragma warning disable CA1815 // Override equals and operator equals on value types - /// Reflection-based parse strategy: forwards to a runtime . - public readonly struct ReflectionLineParser : IRecordLineParser where TModel : new() + /// Layout parse strategy: forwards to a , built from attributes or fluently. + public readonly struct LayoutLineParser : IRecordLineParser { - private readonly FixedWidthParser _parser; + private readonly FixedWidthLayout _layout; - internal ReflectionLineParser(FixedWidthParser parser) + internal LayoutLineParser(FixedWidthLayout layout) { - _parser = parser; + _layout = layout; } /// public bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TModel model) { - return _parser.TryParse(line, in context, out model); + return _layout.TryParse(line, in context, out model); } } diff --git a/tests/FixedWidthParser.Tests/LayoutReadTests.cs b/tests/FixedWidthParser.Tests/LayoutReadTests.cs new file mode 100644 index 0000000..293a1c7 --- /dev/null +++ b/tests/FixedWidthParser.Tests/LayoutReadTests.cs @@ -0,0 +1,90 @@ +using System.Text; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// The read and aggregate overloads that take a layout. + public sealed class LayoutReadTests : IDisposable + { + private const string People = "John Doe 30 6000000.00\nJane 28 55000.00 \nAnn 7 1.5 \n"; + + private readonly string _path = Path.GetTempFileName(); + + public void Dispose() + { + File.Delete(_path); + } + + [Fact] + public void ReadFile_WithLayout_YieldsSameRecordsAsReader() + { + File.WriteAllText(_path, People); + + var records = FixedWidth.ReadFile(_path, FixedWidthLayout.FromAttributes()) + .Select(p => (p.Name, p.Age, p.Salary)) + .ToList(); + + Assert.Equal([("John Doe", 30, 6000000.00), ("Jane", 28, 55000.00), ("Ann", 7, 1.5)], records); + } + + [Fact] + public async Task ReadAsync_WithLayout_ParseError_ReportsLineAndColumn() + { + using var stream = StreamOf("John Doe 30 6000000.00\nJane abc 55000.00 \n"); + + var ex = await Assert.ThrowsAsync(async () => + { + await foreach (var _ in FixedWidth.ReadAsync(stream, FixedWidthLayout.FromAttributes())) + { + } + }); + + Assert.Equal(2, ex.LineNumber); + Assert.Equal("Age", ex.ColumnName); + Assert.Equal(10, ex.ColumnStart); + Assert.Equal(5, ex.ColumnLength); + } + + [Fact] + public async Task AggregateFileAsync_WithLayout_MatchesSequentialFold() + { + File.WriteAllText(_path, NameAges(200_000)); + var layout = FixedWidthLayout.FromAttributes(); + + var parallel = await FixedWidth.AggregateFileAsync( + _path, layout, parallel: new FixedWidthParallelOptions { DegreeOfParallelism = 4 }, ct: TestContext.Current.CancellationToken); + + Assert.Equal(SequentialAges(layout), parallel.Values); + } + + [Fact] + public async Task AggregateAsync_WithLayout_MatchesSequentialFold() + { + File.WriteAllText(_path, NameAges(200_000)); + var layout = FixedWidthLayout.FromAttributes(); + using var stream = new MemoryStream(File.ReadAllBytes(_path)); + + var parallel = await FixedWidth.AggregateAsync( + stream, layout, parallel: new FixedWidthParallelOptions { DegreeOfParallelism = 4 }, ct: TestContext.Current.CancellationToken); + + Assert.Equal(SequentialAges(layout), parallel.Values); + } + + private List SequentialAges(FixedWidthLayout layout) + { + var ages = FixedWidth.ReadFile(_path, layout).Select(m => m.Age).ToList(); + Assert.Equal(200_000, ages.Count); + return ages; + } + + private static string NameAges(int lines) + { + var text = new StringBuilder(); + for (int i = 1; i <= lines; i++) + { + text.Append("Name").Append(i % 10).Append(' ').Append((i % 1000).ToString("D3", Inv)).Append('\n'); + } + return text.ToString(); + } + } +} diff --git a/tests/FixedWidthParser.Tests/ReaderTests.cs b/tests/FixedWidthParser.Tests/ReaderTests.cs index 8eca5d4..1a070a8 100644 --- a/tests/FixedWidthParser.Tests/ReaderTests.cs +++ b/tests/FixedWidthParser.Tests/ReaderTests.cs @@ -123,7 +123,7 @@ public void Read_WithStringPool_InternsRepeatedValues() [Fact] public void GetEnumerator_IsStruct_ForAllocationFreeForeach() { - Assert.True(typeof(RecordEnumerator>).IsValueType); + Assert.True(typeof(RecordEnumerator>).IsValueType); int count = 0; foreach (var _ in new FixedWidthReader().Read(StreamOf("ABC\nDEF\n"))) From ae1ca4a68496eaeb664f1b0a35a021ecc13c5132 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 23 Oct 2026 22:25:45 -0300 Subject: [PATCH 19/85] feat(writer): write generated and layout models through FixedWidth FixedWidth.Write, WriteMany, WriteAsync and WriteManyAsync each come as a generated overload and a layout overload. Batching moves out of FixedWidthWriter into RecordWriter, generic over an IRecordFormatter struct like the read side. A generated model has no known line length, so a record that does not fit flushes the batch and, if still too long, grows it. Generated models can now write to a stream. --- src/FixedWidthParser/FixedWidth.cs | 110 ++++++++++-- .../FixedWidthWriterOptions.cs | 15 +- src/FixedWidthParser/PublicAPI.Unshipped.txt | 10 ++ .../Writers/FixedWidthWriter.cs | 136 ++------------ .../Writers/RecordFormatters.cs | 40 +++++ src/FixedWidthParser/Writers/RecordWriter.cs | 167 ++++++++++++++++++ .../FixedWidthParser.Tests/GeneratedModels.cs | 12 ++ .../LayoutWriteTests.cs | 129 ++++++++++++++ 8 files changed, 483 insertions(+), 136 deletions(-) create mode 100644 src/FixedWidthParser/Writers/RecordFormatters.cs create mode 100644 src/FixedWidthParser/Writers/RecordWriter.cs create mode 100644 tests/FixedWidthParser.Tests/LayoutWriteTests.cs diff --git a/src/FixedWidthParser/FixedWidth.cs b/src/FixedWidthParser/FixedWidth.cs index 97a3d69..3561030 100644 --- a/src/FixedWidthParser/FixedWidth.cs +++ b/src/FixedWidthParser/FixedWidth.cs @@ -1,4 +1,5 @@ using FixedWidthParser.Readers; +using FixedWidthParser.Writers; namespace FixedWidthParser { @@ -47,16 +48,7 @@ public static bool TryFormat(in TModel model, Span destination, in public static bool TryFormat(in TModel model, FixedWidthLayout layout, Span destination, in FixedWidthContext context, out int bytesWritten) where TModel : allows ref struct { - ArgumentNullException.ThrowIfNull(layout); - layout.EnsureCanWrite(); - if (destination.Length < layout.LineLength) - { - bytesWritten = 0; - return false; - } - layout.FormatLine(in model, destination[..layout.LineLength], in context); - bytesWritten = layout.LineLength; - return true; + return WritableLayout(layout).TryFormat(in model, destination, in context, out bytesWritten); } /// Reads from a stream, once. controls whether the stream is disposed when iteration ends. @@ -184,6 +176,104 @@ public static Task AggregateAsync( ReadableLayout(layout), stream, leaveOpen, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), ct); } + + /// Writes one record and its line break. + /// A value does not fit a column whose overflow throws. + public static void Write(Stream stream, in TModel model, FixedWidthWriterOptions? options = null) + where TModel : IFixedWidthModel + { + ArgumentNullException.ThrowIfNull(stream); + RecordWriter.Write(stream, in model, default(GeneratedRecordFormatter), FixedWidthWriterOptions.Validate(options)); + } + + /// Writes one record and its line break through . + /// has a column without a getter, or a value does not fit a column whose overflow throws. + public static void Write(Stream stream, in TModel model, FixedWidthLayout layout, FixedWidthWriterOptions? options = null) + { + ArgumentNullException.ThrowIfNull(stream); + RecordWriter.Write(stream, in model, WritableLayout(layout), FixedWidthWriterOptions.Validate(options)); + } + + /// Writes every record, each followed by a line break. + /// A value does not fit a column whose overflow throws. + public static void WriteMany(Stream stream, IEnumerable models, FixedWidthWriterOptions? options = null) + where TModel : IFixedWidthModel + { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(models); + RecordWriter.WriteMany(stream, models, default(GeneratedRecordFormatter), FixedWidthWriterOptions.Validate(options)); + } + + /// Writes every record through , each followed by a line break. + /// has a column without a getter, or a value does not fit a column whose overflow throws. + public static void WriteMany(Stream stream, IEnumerable models, FixedWidthLayout layout, FixedWidthWriterOptions? options = null) + { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(models); + RecordWriter.WriteMany(stream, models, WritableLayout(layout), FixedWidthWriterOptions.Validate(options)); + } + + /// Writes every record, each followed by a line break, reading each one by reference. + /// A value does not fit a column whose overflow throws. + public static void WriteMany(Stream stream, ReadOnlySpan models, FixedWidthWriterOptions? options = null) + where TModel : IFixedWidthModel + { + ArgumentNullException.ThrowIfNull(stream); + RecordWriter.WriteMany(stream, models, default(GeneratedRecordFormatter), FixedWidthWriterOptions.Validate(options)); + } + + /// Writes every record through , each followed by a line break, reading each one by reference. + /// has a column without a getter, or a value does not fit a column whose overflow throws. + public static void WriteMany(Stream stream, ReadOnlySpan models, FixedWidthLayout layout, FixedWidthWriterOptions? options = null) + { + ArgumentNullException.ThrowIfNull(stream); + RecordWriter.WriteMany(stream, models, WritableLayout(layout), FixedWidthWriterOptions.Validate(options)); + } + + /// Writes one record and its line break. + /// A value does not fit a column whose overflow throws. + public static Task WriteAsync(Stream stream, TModel model, FixedWidthWriterOptions? options = null, CancellationToken cancellationToken = default) + where TModel : IFixedWidthModel + { + ArgumentNullException.ThrowIfNull(stream); + return RecordWriter.WriteAsync(stream, model, default(GeneratedRecordFormatter), FixedWidthWriterOptions.Validate(options), cancellationToken); + } + + /// Writes one record and its line break through . + /// has a column without a getter, or a value does not fit a column whose overflow throws. + public static Task WriteAsync(Stream stream, TModel model, FixedWidthLayout layout, FixedWidthWriterOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(stream); + return RecordWriter.WriteAsync(stream, model, WritableLayout(layout), FixedWidthWriterOptions.Validate(options), cancellationToken); + } + + /// Writes every record, each followed by a line break. + /// A value does not fit a column whose overflow throws. + public static Task WriteManyAsync(Stream stream, IEnumerable models, FixedWidthWriterOptions? options = null, CancellationToken cancellationToken = default) + where TModel : IFixedWidthModel + { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(models); + return RecordWriter.WriteManyAsync(stream, models, default(GeneratedRecordFormatter), FixedWidthWriterOptions.Validate(options), cancellationToken); + } + + /// Writes every record through , each followed by a line break. + /// has a column without a getter, or a value does not fit a column whose overflow throws. + public static Task WriteManyAsync(Stream stream, IEnumerable models, FixedWidthLayout layout, FixedWidthWriterOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(models); + return RecordWriter.WriteManyAsync(stream, models, WritableLayout(layout), FixedWidthWriterOptions.Validate(options), cancellationToken); + } + + private static LayoutRecordFormatter WritableLayout(FixedWidthLayout layout) + where TModel : allows ref struct + { + ArgumentNullException.ThrowIfNull(layout); + layout.EnsureCanWrite(); + return new LayoutRecordFormatter(layout); + } + private static LayoutLineParser ReadableLayout(FixedWidthLayout layout) { ArgumentNullException.ThrowIfNull(layout); diff --git a/src/FixedWidthParser/FixedWidthWriterOptions.cs b/src/FixedWidthParser/FixedWidthWriterOptions.cs index 624ceaa..d567cae 100644 --- a/src/FixedWidthParser/FixedWidthWriterOptions.cs +++ b/src/FixedWidthParser/FixedWidthWriterOptions.cs @@ -2,7 +2,7 @@ namespace FixedWidthParser { - /// Settings for . Every property has a default. + /// Settings for the write methods. Every property has a default. public sealed class FixedWidthWriterOptions { internal static readonly FixedWidthWriterOptions Default = new(); @@ -17,5 +17,18 @@ public sealed class FixedWidthWriterOptions public string NewLine { get; init; } = Environment.NewLine; internal FixedWidthContext Context => new() { FormatProvider = FormatProvider!, Encoding = Encoding! }; + + /// in the file's encoding, encoded once per options instance. + internal byte[] NewLineBytes => field ??= Context.Encoding.GetBytes(NewLine); + + internal static FixedWidthWriterOptions Validate(FixedWidthWriterOptions? options) + { + options ??= Default; + if (options.NewLine is null) + { + throw new ArgumentException("NewLine must not be null.", nameof(options)); + } + return options; + } } } diff --git a/src/FixedWidthParser/PublicAPI.Unshipped.txt b/src/FixedWidthParser/PublicAPI.Unshipped.txt index 4f8144f..cce3f07 100644 --- a/src/FixedWidthParser/PublicAPI.Unshipped.txt +++ b/src/FixedWidthParser/PublicAPI.Unshipped.txt @@ -190,3 +190,13 @@ static FixedWidthParser.FixedWidth.Read(System.IO.Stream! stream, FixedW static FixedWidthParser.FixedWidth.ReadAsync(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! static FixedWidthParser.FixedWidth.ReadFile(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! static FixedWidthParser.FixedWidth.ReadFileAsync(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! +static FixedWidthParser.FixedWidth.Write(System.IO.Stream! stream, in TModel model, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthWriterOptions? options = null) -> void +static FixedWidthParser.FixedWidth.Write(System.IO.Stream! stream, in TModel model, FixedWidthParser.FixedWidthWriterOptions? options = null) -> void +static FixedWidthParser.FixedWidth.WriteAsync(System.IO.Stream! stream, TModel model, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthWriterOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.WriteAsync(System.IO.Stream! stream, TModel model, FixedWidthParser.FixedWidthWriterOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.WriteMany(System.IO.Stream! stream, System.Collections.Generic.IEnumerable! models, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthWriterOptions? options = null) -> void +static FixedWidthParser.FixedWidth.WriteMany(System.IO.Stream! stream, System.Collections.Generic.IEnumerable! models, FixedWidthParser.FixedWidthWriterOptions? options = null) -> void +static FixedWidthParser.FixedWidth.WriteMany(System.IO.Stream! stream, System.ReadOnlySpan models, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthWriterOptions? options = null) -> void +static FixedWidthParser.FixedWidth.WriteMany(System.IO.Stream! stream, System.ReadOnlySpan models, FixedWidthParser.FixedWidthWriterOptions? options = null) -> void +static FixedWidthParser.FixedWidth.WriteManyAsync(System.IO.Stream! stream, System.Collections.Generic.IEnumerable! models, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthWriterOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.WriteManyAsync(System.IO.Stream! stream, System.Collections.Generic.IEnumerable! models, FixedWidthParser.FixedWidthWriterOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! diff --git a/src/FixedWidthParser/Writers/FixedWidthWriter.cs b/src/FixedWidthParser/Writers/FixedWidthWriter.cs index 4e916ee..0860346 100644 --- a/src/FixedWidthParser/Writers/FixedWidthWriter.cs +++ b/src/FixedWidthParser/Writers/FixedWidthWriter.cs @@ -1,6 +1,4 @@ -using System.Buffers; using System.Diagnostics.CodeAnalysis; -using System.Runtime.CompilerServices; namespace FixedWidthParser.Writers { @@ -12,11 +10,8 @@ namespace FixedWidthParser.Writers /// public sealed class FixedWidthWriter where TModel : new() { - private const int BatchSize = 64 * 1024; - private readonly FixedWidthLayout _layout; - private readonly FixedWidthContext _context; - private readonly byte[] _newLine; + private readonly FixedWidthWriterOptions _options; /// Creates a writer; options use every default. [RequiresUnreferencedCode("Attribute layouts reflect over TModel's public properties and fields, which trimming may remove.")] @@ -25,18 +20,10 @@ public FixedWidthWriter(FixedWidthWriterOptions? options = null) { _layout = FixedWidthLayout.FromAttributes(); _layout.EnsureCanWrite(); - options ??= FixedWidthWriterOptions.Default; - if (options.NewLine is null) - { - throw new ArgumentException("NewLine must not be null.", nameof(options)); - } - _context = options.Context; - _newLine = _context.Encoding.GetBytes(options.NewLine); + _options = FixedWidthWriterOptions.Validate(options); } - private int LineLength => _layout.LineLength; - - private int RecordLength => LineLength + _newLine.Length; + private LayoutRecordFormatter Formatter => new(_layout); /// /// Formats into , without a line break. @@ -44,35 +31,14 @@ public FixedWidthWriter(FixedWidthWriterOptions? options = null) /// public bool TryFormat(in TModel model, Span destination, out int bytesWritten) { - if (destination.Length < LineLength) - { - bytesWritten = 0; - return false; - } - FormatLine(in model, destination[..LineLength]); - bytesWritten = LineLength; - return true; + return Formatter.TryFormat(in model, destination, _options.Context, out bytesWritten); } /// Writes one record and its line break. - [SkipLocalsInit] public void Write(Stream stream, in TModel model) { ArgumentNullException.ThrowIfNull(stream); - int size = RecordLength; - byte[]? rented = size <= 1024 ? null : ArrayPool.Shared.Rent(size); - Span record = rented is null ? stackalloc byte[1024] : rented; - try - { - stream.Write(record[..FormatRecord(in model, record)]); - } - finally - { - if (rented is not null) - { - ArrayPool.Shared.Return(rented); - } - } + RecordWriter.Write(stream, in model, Formatter, _options); } /// Writes every record, each followed by a line break. @@ -80,109 +46,29 @@ public void WriteMany(Stream stream, IEnumerable models) { ArgumentNullException.ThrowIfNull(stream); ArgumentNullException.ThrowIfNull(models); - byte[] buffer = RentBatch(); - try - { - int used = 0; - foreach (var model in models) - { - used = Append(stream, buffer, used, in model); - } - stream.Write(buffer, 0, used); - } - finally - { - ArrayPool.Shared.Return(buffer); - } + RecordWriter.WriteMany(stream, models, Formatter, _options); } /// Writes every record, each followed by a line break, reading each one by reference. public void WriteMany(Stream stream, ReadOnlySpan models) { ArgumentNullException.ThrowIfNull(stream); - byte[] buffer = RentBatch(); - try - { - int used = 0; - foreach (ref readonly var model in models) - { - used = Append(stream, buffer, used, in model); - } - stream.Write(buffer, 0, used); - } - finally - { - ArrayPool.Shared.Return(buffer); - } + RecordWriter.WriteMany(stream, models, Formatter, _options); } /// Writes one record and its line break. - public async Task WriteAsync(Stream stream, TModel model, CancellationToken cancellationToken = default) + public Task WriteAsync(Stream stream, TModel model, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(stream); - byte[] buffer = ArrayPool.Shared.Rent(RecordLength); - try - { - int written = FormatRecord(in model, buffer); - await stream.WriteAsync(buffer.AsMemory(0, written), cancellationToken).ConfigureAwait(false); - } - finally - { - ArrayPool.Shared.Return(buffer); - } + return RecordWriter.WriteAsync(stream, model, Formatter, _options, cancellationToken); } /// Writes every record, each followed by a line break. - public async Task WriteManyAsync(Stream stream, IEnumerable models, CancellationToken cancellationToken = default) + public Task WriteManyAsync(Stream stream, IEnumerable models, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(stream); ArgumentNullException.ThrowIfNull(models); - byte[] buffer = RentBatch(); - try - { - int used = 0; - foreach (var model in models) - { - if (used + RecordLength > buffer.Length) - { - await stream.WriteAsync(buffer.AsMemory(0, used), cancellationToken).ConfigureAwait(false); - used = 0; - } - used += FormatRecord(in model, buffer.AsSpan(used)); - } - await stream.WriteAsync(buffer.AsMemory(0, used), cancellationToken).ConfigureAwait(false); - } - finally - { - ArrayPool.Shared.Return(buffer); - } - } - - private byte[] RentBatch() - { - return ArrayPool.Shared.Rent(Math.Max(BatchSize, RecordLength)); - } - - private int Append(Stream stream, byte[] buffer, int used, in TModel model) - { - if (used + RecordLength > buffer.Length) - { - stream.Write(buffer, 0, used); - used = 0; - } - return used + FormatRecord(in model, buffer.AsSpan(used)); - } - - private int FormatRecord(in TModel model, Span destination) - { - FormatLine(in model, destination[..LineLength]); - _newLine.CopyTo(destination[LineLength..]); - return RecordLength; - } - - private void FormatLine(in TModel model, Span line) - { - _layout.FormatLine(in model, line, in _context); + return RecordWriter.WriteManyAsync(stream, models, Formatter, _options, cancellationToken); } } } diff --git a/src/FixedWidthParser/Writers/RecordFormatters.cs b/src/FixedWidthParser/Writers/RecordFormatters.cs new file mode 100644 index 0000000..52ba08a --- /dev/null +++ b/src/FixedWidthParser/Writers/RecordFormatters.cs @@ -0,0 +1,40 @@ +namespace FixedWidthParser.Writers +{ + /// + /// Format strategy is specialized by, the write-side twin of + /// : implemented by types so each + /// (TModel, TFormatter) pair is compiled once and the call devirtualizes. + /// + internal interface IRecordFormatter + where TModel : allows ref struct + { + /// Formats one line, without its line break. Returns only when is shorter than the line. + bool TryFormat(in TModel model, Span destination, in FixedWidthContext context, out int bytesWritten); + } + + internal readonly struct LayoutRecordFormatter(FixedWidthLayout layout) : IRecordFormatter + where TModel : allows ref struct + { + public bool TryFormat(in TModel model, Span destination, in FixedWidthContext context, out int bytesWritten) + { + int lineLength = layout.LineLength; + if (destination.Length < lineLength) + { + bytesWritten = 0; + return false; + } + layout.FormatLine(in model, destination[..lineLength], in context); + bytesWritten = lineLength; + return true; + } + } + + internal readonly struct GeneratedRecordFormatter : IRecordFormatter + where TModel : IFixedWidthModel + { + public bool TryFormat(in TModel model, Span destination, in FixedWidthContext context, out int bytesWritten) + { + return TModel.TryFormat(in model, destination, in context, out bytesWritten); + } + } +} diff --git a/src/FixedWidthParser/Writers/RecordWriter.cs b/src/FixedWidthParser/Writers/RecordWriter.cs new file mode 100644 index 0000000..ff739a1 --- /dev/null +++ b/src/FixedWidthParser/Writers/RecordWriter.cs @@ -0,0 +1,167 @@ +using System.Buffers; +using System.Runtime.CompilerServices; + +namespace FixedWidthParser.Writers +{ + /// + /// Writes records, each followed by a line break, batched into one pooled buffer. The line length is + /// not known up front (a generated model formats itself), so a record that does not fit the rest of the + /// batch flushes it, and one that does not fit an empty batch grows it. + /// + internal static class RecordWriter + { + private const int BatchSize = 64 * 1024; + private const int StackRecordSize = 1024; + + [SkipLocalsInit] + public static void Write(Stream stream, in TModel model, TFormatter formatter, FixedWidthWriterOptions options) + where TFormatter : struct, IRecordFormatter + { + var context = options.Context; + Span record = stackalloc byte[StackRecordSize]; + int written = TryFormatRecord(in model, formatter, record, in context, options.NewLineBytes); + if (written > 0) + { + stream.Write(record[..written]); + return; + } + + byte[] buffer = ArrayPool.Shared.Rent(StackRecordSize * 2); + try + { + written = FormatGrowing(in model, formatter, ref buffer, in context, options.NewLineBytes); + stream.Write(buffer, 0, written); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + public static void WriteMany(Stream stream, IEnumerable models, TFormatter formatter, FixedWidthWriterOptions options) + where TFormatter : struct, IRecordFormatter + { + var context = options.Context; + byte[] buffer = ArrayPool.Shared.Rent(BatchSize); + try + { + int used = 0; + foreach (var model in models) + { + used = Append(stream, in model, formatter, ref buffer, used, in context, options.NewLineBytes); + } + stream.Write(buffer, 0, used); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + public static void WriteMany(Stream stream, ReadOnlySpan models, TFormatter formatter, FixedWidthWriterOptions options) + where TFormatter : struct, IRecordFormatter + { + var context = options.Context; + byte[] buffer = ArrayPool.Shared.Rent(BatchSize); + try + { + int used = 0; + foreach (ref readonly var model in models) + { + used = Append(stream, in model, formatter, ref buffer, used, in context, options.NewLineBytes); + } + stream.Write(buffer, 0, used); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + public static async Task WriteAsync(Stream stream, TModel model, TFormatter formatter, FixedWidthWriterOptions options, CancellationToken cancellationToken) + where TFormatter : struct, IRecordFormatter + { + var context = options.Context; + byte[] buffer = ArrayPool.Shared.Rent(StackRecordSize); + try + { + int written = FormatGrowing(in model, formatter, ref buffer, in context, options.NewLineBytes); + await stream.WriteAsync(buffer.AsMemory(0, written), cancellationToken).ConfigureAwait(false); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + public static async Task WriteManyAsync(Stream stream, IEnumerable models, TFormatter formatter, FixedWidthWriterOptions options, CancellationToken cancellationToken) + where TFormatter : struct, IRecordFormatter + { + var context = options.Context; + byte[] buffer = ArrayPool.Shared.Rent(BatchSize); + try + { + int used = 0; + foreach (var model in models) + { + int written = TryFormatRecord(in model, formatter, buffer.AsSpan(used), in context, options.NewLineBytes); + if (written == 0) + { + await stream.WriteAsync(buffer.AsMemory(0, used), cancellationToken).ConfigureAwait(false); + used = 0; + written = FormatGrowing(in model, formatter, ref buffer, in context, options.NewLineBytes); + } + used += written; + } + await stream.WriteAsync(buffer.AsMemory(0, used), cancellationToken).ConfigureAwait(false); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + private static int Append( + Stream stream, in TModel model, TFormatter formatter, ref byte[] buffer, int used, in FixedWidthContext context, byte[] newLine) + where TFormatter : struct, IRecordFormatter + { + int written = TryFormatRecord(in model, formatter, buffer.AsSpan(used), in context, newLine); + if (written > 0) + { + return used + written; + } + stream.Write(buffer, 0, used); + return FormatGrowing(in model, formatter, ref buffer, in context, newLine); + } + + // Formats into the whole of buffer, replacing it with one twice the size until the record fits. + // The new buffer is rented before the old one is returned, so the caller's finally never returns a buffer twice. + private static int FormatGrowing(in TModel model, TFormatter formatter, ref byte[] buffer, in FixedWidthContext context, byte[] newLine) + where TFormatter : struct, IRecordFormatter + { + while (true) + { + int written = TryFormatRecord(in model, formatter, buffer, in context, newLine); + if (written > 0) + { + return written; + } + byte[] larger = ArrayPool.Shared.Rent(checked(buffer.Length * 2)); + ArrayPool.Shared.Return(buffer); + buffer = larger; + } + } + + /// Formats a record and its line break into ; returns 0 when they do not fit. + private static int TryFormatRecord(in TModel model, TFormatter formatter, Span destination, in FixedWidthContext context, byte[] newLine) + where TFormatter : struct, IRecordFormatter + { + if (!formatter.TryFormat(in model, destination, in context, out int written) || destination.Length - written < newLine.Length) + { + return 0; + } + newLine.CopyTo(destination[written..]); + return written + newLine.Length; + } + } +} diff --git a/tests/FixedWidthParser.Tests/GeneratedModels.cs b/tests/FixedWidthParser.Tests/GeneratedModels.cs index e12126a..a9fd364 100644 --- a/tests/FixedWidthParser.Tests/GeneratedModels.cs +++ b/tests/FixedWidthParser.Tests/GeneratedModels.cs @@ -162,4 +162,16 @@ public partial struct GenProductModel : IFixedWidthModel { [FixedColumn(0, 8, Alignment = Alignment.Right, Format = "C")] public decimal Price { get; init; } } + + /// A line longer than the single-record stack buffer. + public readonly partial record struct GenLongNameModel : IFixedWidthModel + { + [FixedColumn(0, 1200)] public string Name { get; init; } + } + + /// A line longer than a whole write batch. + public readonly partial record struct GenHugeNameModel : IFixedWidthModel + { + [FixedColumn(0, 70_000)] public string Name { get; init; } + } } diff --git a/tests/FixedWidthParser.Tests/LayoutWriteTests.cs b/tests/FixedWidthParser.Tests/LayoutWriteTests.cs new file mode 100644 index 0000000..ef3e278 --- /dev/null +++ b/tests/FixedWidthParser.Tests/LayoutWriteTests.cs @@ -0,0 +1,129 @@ +using System.Text; +using FixedWidthParser.Writers; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// The write overloads, for generated models and for layouts. + public class LayoutWriteTests + { + private static readonly FixedWidthWriterOptions Lf = new() { NewLine = "\n" }; + + private static readonly GenPersonModel[] GenPeople = + [ + new() { Name = "John Doe", Age = 30, Salary = 6000000 }, + new() { Name = "Jane", Age = 28, Salary = 55000 }, + new() { Name = "Ann", Age = 7, Salary = 1.5 }, + ]; + + private static readonly PersonModel[] People = + [ + new() { Name = "John Doe", Age = 30, Salary = 6000000 }, + new() { Name = "Jane", Age = 28, Salary = 55000 }, + new() { Name = "Ann", Age = 7, Salary = 1.5 }, + ]; + + [Fact] + public void Generated_WriteMany_MatchesTryFormatPlusNewLine() + { + using var stream = new MemoryStream(); + + FixedWidth.WriteMany(stream, GenPeople, Lf); + + Assert.Equal(string.Concat(GenPeople.Select(p => Format(p) + "\n")), Decode(stream)); + } + + [Fact] + public void Generated_WriteMany_CrossesBatchBoundary() + { + int count = (3 * 64 * 1024 / 26) + 17; + var models = Enumerable.Range(0, count).Select(i => new GenPersonModel { Name = "N" + i, Age = i % 1000, Salary = i }).ToList(); + using var stream = new MemoryStream(); + + FixedWidth.WriteMany(stream, models, Lf); + + byte[] bytes = stream.ToArray(); + Assert.Equal(count * 26, bytes.Length); + for (int i = 0; i < count; i++) + { + Assert.True(FixedWidth.TryParse(bytes.AsSpan(i * 26, 25), Ctx, out var parsed)); + Assert.Equal(models[i], parsed); + } + } + + [Fact] + public void Generated_Write_RecordLongerThanStackBuffer() + { + using var stream = new MemoryStream(); + + FixedWidth.Write(stream, new GenLongNameModel { Name = "Wide" }, Lf); + + Assert.Equal("Wide".PadRight(1200) + "\n", Decode(stream)); + } + + [Fact] + public void Generated_WriteMany_RecordLongerThanBatch() + { + using var stream = new MemoryStream(); + + FixedWidth.WriteMany(stream, [new GenHugeNameModel { Name = "A" }, new GenHugeNameModel { Name = "B" }], Lf); + + Assert.Equal("A".PadRight(70_000) + "\n" + "B".PadRight(70_000) + "\n", Decode(stream)); + } + + [Fact] + public void Layout_WriteMany_MatchesWriter() + { + using var stream = new MemoryStream(); + + FixedWidth.WriteMany(stream, People, FixedWidthLayout.FromAttributes(), Lf); + + Assert.Equal(WriteMany(new FixedWidthWriter(Lf), (IEnumerable)People), Decode(stream)); + } + + [Fact] + public async Task WriteManyAsync_MatchesWriteMany() + { + var layout = FixedWidthLayout.FromAttributes(); + using var sync = new MemoryStream(); + await using var async = new MemoryStream(); + using var genSync = new MemoryStream(); + await using var genAsync = new MemoryStream(); + + FixedWidth.WriteMany(sync, People, layout, Lf); + await FixedWidth.WriteManyAsync(async, People, layout, Lf, TestContext.Current.CancellationToken); + FixedWidth.WriteMany(genSync, GenPeople, Lf); + await FixedWidth.WriteManyAsync(genAsync, GenPeople, Lf, TestContext.Current.CancellationToken); + + Assert.Equal(Decode(sync), Decode(async)); + Assert.Equal(Decode(genSync), Decode(genAsync)); + Assert.Equal(Decode(sync), Decode(genSync)); + } + + [Fact] + public async Task WriteAsync_MatchesWrite() + { + using var sync = new MemoryStream(); + await using var async = new MemoryStream(); + + FixedWidth.Write(sync, GenPeople[0], Lf); + await FixedWidth.WriteAsync(async, GenPeople[0], Lf, TestContext.Current.CancellationToken); + + Assert.Equal(Format(GenPeople[0]) + "\n", Decode(sync)); + Assert.Equal(Decode(sync), Decode(async)); + } + + [Fact] + public void Write_NullNewLine_ThrowsArgumentException() + { + using var stream = new MemoryStream(); + + Assert.Throws(() => FixedWidth.Write(stream, GenPeople[0], new FixedWidthWriterOptions { NewLine = null! })); + } + + private static string Decode(MemoryStream stream) + { + return Encoding.UTF8.GetString(stream.ToArray()); + } + } +} From 1b09d6ea09226db28aa81df2e9e2262e7f28beae Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 23 Oct 2026 22:31:13 -0300 Subject: [PATCH 20/85] feat(layout): describe a layout fluently with FixedWidthLayout.Build FixedWidthLayoutBuilder binds each column with a ColumnGetter and a ColumnSetter: Column (string or UTF-8 parsable), DecodedColumn, NullableColumn, ConvertedColumn and Factory. Per-column settings come in FixedWidthColumnOptions, with FixedColumnAttribute's defaults. The builder reaches the same typed parse and format builders as the attribute path, by generics instead of reflection, so the two produce identical results and Build stays trimming- and AOT-safe. --- .../Formatters/FormatterBuilder.cs | 24 +- .../Layouts/FixedWidthColumnOptions.cs | 31 ++ .../Layouts/FixedWidthLayout.cs | 23 +- .../Layouts/FixedWidthLayoutBuilder.cs | 238 ++++++++++++++ .../Processors/ColumnParserFactory.cs | 118 +++++-- src/FixedWidthParser/PublicAPI.Unshipped.txt | 22 ++ .../FluentLayoutTests.cs | 296 ++++++++++++++++++ tests/FixedWidthParser.Tests/TestModels.cs | 29 ++ 8 files changed, 741 insertions(+), 40 deletions(-) create mode 100644 src/FixedWidthParser/Layouts/FixedWidthColumnOptions.cs create mode 100644 src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs create mode 100644 tests/FixedWidthParser.Tests/FluentLayoutTests.cs diff --git a/src/FixedWidthParser/Formatters/FormatterBuilder.cs b/src/FixedWidthParser/Formatters/FormatterBuilder.cs index 1921198..a4de1c1 100644 --- a/src/FixedWidthParser/Formatters/FormatterBuilder.cs +++ b/src/FixedWidthParser/Formatters/FormatterBuilder.cs @@ -20,17 +20,9 @@ private static IColumnFormatter CreateFormatter(MemberInfo membe where TModel : allows ref struct { var (memberType, model, access) = ModelColumns.MemberAccess(typeof(TModel), member); - FixedWidthRuntime.ToAsciiByte(attribute.Padding, member.Name, nameof(FixedColumnAttribute.Padding)); + var options = ResolveOptions(member.Name, memberType, attribute.Alignment, attribute.Padding, attribute.Format, attribute.Overflow); byte fill = FixedWidthRuntime.ToAsciiByte(attribute.TrimChar, member.Name, nameof(FixedColumnAttribute.TrimChar)); - - if (attribute.Overflow == OverflowBehavior.Truncate && memberType != typeof(string)) - { - throw new InvalidOperationException( - $"OverflowBehavior.Truncate is not supported for non-string column '{member.Name}' of type '{memberType.Name}'."); - } - var getter = Expression.Lambda(typeof(ColumnGetter<,>).MakeGenericType(typeof(TModel), memberType), access, model).Compile(); - var options = new ColumnFormatOptions(attribute.Alignment, attribute.Padding, attribute.Format, ResolveOverflow(attribute.Overflow, memberType)); var underlyingType = Nullable.GetUnderlyingType(memberType); if (underlyingType is null) @@ -44,8 +36,20 @@ private static IColumnFormatter CreateFormatter(MemberInfo membe attribute.Start, attribute.Length, fill, getter, inner)!; } + /// Validates a column's write options and resolves its overflow; shared by attribute and fluent layouts. + internal static ColumnFormatOptions ResolveOptions(string columnName, Type valueType, Alignment alignment, char padding, string? format, OverflowBehavior overflow) + { + FixedWidthRuntime.ToAsciiByte(padding, columnName, nameof(FixedColumnAttribute.Padding)); + if (overflow == OverflowBehavior.Truncate && valueType != typeof(string)) + { + throw new InvalidOperationException( + $"OverflowBehavior.Truncate is not supported for non-string column '{columnName}' of type '{valueType.Name}'."); + } + return new ColumnFormatOptions(alignment, padding, format, ResolveOverflow(overflow, valueType)); + } + // Default overflow resolves per type: string truncates, everything else throws. - internal static OverflowBehavior ResolveOverflow(OverflowBehavior overflow, Type valueType) + private static OverflowBehavior ResolveOverflow(OverflowBehavior overflow, Type valueType) { if (overflow != OverflowBehavior.Default) { diff --git a/src/FixedWidthParser/Layouts/FixedWidthColumnOptions.cs b/src/FixedWidthParser/Layouts/FixedWidthColumnOptions.cs new file mode 100644 index 0000000..281b955 --- /dev/null +++ b/src/FixedWidthParser/Layouts/FixedWidthColumnOptions.cs @@ -0,0 +1,31 @@ +using FixedWidthParser.Attributes; + +namespace FixedWidthParser +{ + /// + /// How one column of a layout is trimmed, parsed and + /// written: the settings carries, with the same defaults. + /// + public sealed class FixedWidthColumnOptions + { + internal static readonly FixedWidthColumnOptions Default = new(); + + /// Which side a written value is aligned to. Defaults to . + public Alignment Alignment { get; init; } = Alignment.Left; + + /// The ASCII character that fills a written column around its value. Defaults to a space. + public char Padding { get; init; } = ' '; + + /// The ASCII character trimmed from a column before it is parsed, and written for a value. Defaults to a space. + public char TrimChar { get; init; } = ' '; + + /// Which ends of a column is trimmed from. Defaults to . + public TrimMode TrimMode { get; init; } = TrimMode.Trailing; + + /// The format for writing, and for parsing a date or time exactly. means the type's default. + public string? Format { get; init; } + + /// What happens when a written value is wider than its column. truncates a string and throws for anything else. + public OverflowBehavior Overflow { get; init; } = OverflowBehavior.Default; + } +} diff --git a/src/FixedWidthParser/Layouts/FixedWidthLayout.cs b/src/FixedWidthParser/Layouts/FixedWidthLayout.cs index b997c93..e16dfe9 100644 --- a/src/FixedWidthParser/Layouts/FixedWidthLayout.cs +++ b/src/FixedWidthParser/Layouts/FixedWidthLayout.cs @@ -90,6 +90,25 @@ internal void EnsureCanWrite() /// Creates instances. public static class FixedWidthLayout { + /// + /// Builds a layout column by column through , binding each column with + /// delegates. Trimming and AOT safe. + /// + /// + /// For a type you cannot annotate, or a layout known only at run time. Each call builds its own + /// layout, so differently configured layouts for the same coexist. + /// + /// is . + /// The layout is invalid: no columns, a negative start, a length below 1, overlapping columns, a non-ASCII padding or trim character, a truncated non-string column, or a reference-type model with setters and no factory. + public static FixedWidthLayout Build(Action> configure) + where TModel : allows ref struct + { + ArgumentNullException.ThrowIfNull(configure); + var builder = new FixedWidthLayoutBuilder(); + configure(builder); + return builder.Build(); + } + /// /// Returns the layout declared by 's [FixedColumn] properties /// and fields, built once per type by reflection. @@ -119,7 +138,7 @@ static AttributeLayout() { try { - Layout = Build(); + Layout = BuildLayout(); } #pragma warning disable CA1031 // Rethrown from FromAttributes on every call. catch (Exception ex) @@ -132,7 +151,7 @@ static AttributeLayout() // A model may be readable and not writable (or the reverse): one side's error waits for that // side's first use. Only a model neither side can use fails here. - private static FixedWidthLayout Build() + private static FixedWidthLayout BuildLayout() { var readError = TryBuildReader(out var parsers, out var factory); IColumnFormatter[]? formatters = null; diff --git a/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs b/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs new file mode 100644 index 0000000..7c4a858 --- /dev/null +++ b/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs @@ -0,0 +1,238 @@ +using System.Runtime.InteropServices; +using FixedWidthParser.Attributes; +using FixedWidthParser.Formatters; +using FixedWidthParser.Parsers; +using FixedWidthParser.Processors; + +namespace FixedWidthParser +{ + /// + /// Describes a column by column, for + /// . Each column binds to the model through a getter, used + /// to write, and a setter, used to read; either may be , which leaves the layout + /// unable to write or to read. No reflection: trimming and AOT safe. + /// Column positions are bytes. + /// + /// The model type. + public sealed class FixedWidthLayoutBuilder + where TModel : allows ref struct + { + private readonly List<(int Start, int Length, string Name)> _columns = []; + private readonly List> _parsers = []; + private readonly List> _formatters = []; + private bool _missingSetter; + private bool _missingGetter; + private Func? _factory; + + internal FixedWidthLayoutBuilder() + { + } + + /// Adds a column, decoded with the configured encoding. + /// The column's name in error messages. + /// The 0-based offset of the column, in bytes. + /// The width of the column, in bytes. + /// Reads the value to write; for a layout that only reads. + /// Assigns the parsed value; for a layout that only writes. + /// Trimming, alignment, padding, format and overflow. uses every default. + /// This builder. + public FixedWidthLayoutBuilder Column( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, FixedWidthColumnOptions? options = null) + { + var column = Begin(name, start, length, typeof(string), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForString(set, column.Trim, column.TrimMode); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + formatter = new StringColumnFormatter(start, length, column.FormatOptions, name, get); + } + return Add(column, parser, formatter); + } + + /// + /// Adds a column of a type that parses from and formats to UTF-8: the numeric types, + /// and the like. A date or time with a parses exactly + /// with that format. + /// + /// + public FixedWidthLayoutBuilder Column( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, FixedWidthColumnOptions? options = null) + where TValue : IUtf8SpanParsable, IUtf8SpanFormattable, ISpanFormattable + { + var column = Begin(name, start, length, typeof(TValue), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForUtf8Parsable(set, column.Trim, column.TrimMode, column.Format); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + formatter = new Utf8SpanFormattableColumnFormatter(start, length, column.FormatOptions, name, get); + } + return Add(column, parser, formatter); + } + + /// + /// Adds a column of a type that parses only from characters: the column is decoded with the + /// configured encoding first. A date or time with a + /// parses exactly with that format. + /// + /// + public FixedWidthLayoutBuilder DecodedColumn( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, FixedWidthColumnOptions? options = null) + where TValue : ISpanParsable, ISpanFormattable + { + var column = Begin(name, start, length, typeof(TValue), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForDecodedParsable(set, column.Trim, column.TrimMode, column.Format); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + formatter = new SpanFormattableColumnFormatter(start, length, column.FormatOptions, name, get); + } + return Add(column, parser, formatter); + } + + /// + /// Adds a column: a column holding only its trim character reads as + /// , and writes the column filled with it. + /// + /// + public FixedWidthLayoutBuilder NullableColumn( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, FixedWidthColumnOptions? options = null) + where TValue : struct, IUtf8SpanParsable, IUtf8SpanFormattable, ISpanFormattable + { + var column = Begin(name, start, length, typeof(TValue?), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForNullable(set, column.Trim, + adapted => ColumnParserFactory.ForUtf8Parsable(adapted, column.Trim, column.TrimMode, column.Format)); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + var inner = new Utf8SpanFormattableColumnFormatter( + start, length, column.FormatOptions, name, (in TModel model) => get(in model).GetValueOrDefault()); + formatter = new NullableColumnFormatter(start, length, column.Trim, get, inner); + } + return Add(column, parser, formatter); + } + + /// Adds a column parsed and formatted by . + /// The column's name in error messages. + /// The 0-based offset of the column, in bytes. + /// The width of the column, in bytes. + /// Reads the value to write; for a layout that only reads. + /// Assigns the parsed value; for a layout that only writes. + /// Parses and formats the column's value; shared by every line, and by every thread reading in parallel. + /// Trimming, alignment, padding and overflow. uses every default. + /// This builder. + public FixedWidthLayoutBuilder ConvertedColumn( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, + IFixedWidthConverter converter, FixedWidthColumnOptions? options = null) + { + ArgumentNullException.ThrowIfNull(converter); + var column = Begin(name, start, length, typeof(TValue), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForConverter(set, converter, column.Trim, column.TrimMode); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + formatter = new ConverterColumnFormatter>(start, length, column.FormatOptions, name, get, converter); + } + return Add(column, parser, formatter); + } + + /// + /// Sets how a read creates the model before its columns are assigned. Required to read a reference + /// type; a value type without one starts from . + /// + /// Returns a new model for each line. + /// This builder. + public FixedWidthLayoutBuilder Factory(Func factory) + { + ArgumentNullException.ThrowIfNull(factory); + _factory = factory; + return this; + } + + internal FixedWidthLayout Build() + { + if (_columns.Count == 0) + { + throw new InvalidOperationException( + $"The layout for {typeof(TModel).Name} has no columns; a fixed-width layout must define at least one column."); + } + ColumnLayoutValidator.Validate(CollectionsMarshal.AsSpan(_columns), typeof(TModel).Name); + + ColumnParserInfo[]? parsers = null; + Func? factory = null; + if (!_missingSetter) + { + parsers = [.. _parsers]; + factory = ResolveFactory(); + } + IColumnFormatter[]? formatters = null; + if (!_missingGetter) + { + formatters = [.. _formatters]; + } + return new(parsers, factory, formatters, _columns.Max(static column => column.Start + column.Length)); + } + + private Func ResolveFactory() + { + if (_factory is not null) + { + return _factory; + } + if (typeof(TModel).IsValueType) + { + return static () => default!; + } + throw new InvalidOperationException( + $"The layout for {typeof(TModel).Name} has no factory: {typeof(TModel).Name} is a reference type, so reading needs a new instance per line. Call Factory(...)."); + } + + private PendingColumn Begin(string name, int start, int length, Type valueType, FixedWidthColumnOptions? options) + { + ArgumentException.ThrowIfNullOrEmpty(name); + options ??= FixedWidthColumnOptions.Default; + var formatOptions = FormatterBuilder.ResolveOptions(name, valueType, options.Alignment, options.Padding, options.Format, options.Overflow); + byte trim = FixedWidthRuntime.ToAsciiByte(options.TrimChar, name, nameof(FixedWidthColumnOptions.TrimChar)); + _columns.Add((start, length, name)); + return new(name, start, length, trim, options.TrimMode, options.Format, formatOptions); + } + + private FixedWidthLayoutBuilder Add(in PendingColumn column, ColumnParser? parser, IColumnFormatter? formatter) + { + _missingSetter |= parser is null; + _missingGetter |= formatter is null; + if (parser is not null) + { + _parsers.Add(new(column.Start, column.Length, column.Name, parser)); + } + if (formatter is not null) + { + _formatters.Add(formatter); + } + return this; + } + + [StructLayout(LayoutKind.Auto)] + private readonly record struct PendingColumn( + string Name, int Start, int Length, byte Trim, TrimMode TrimMode, string? Format, ColumnFormatOptions FormatOptions); + } +} diff --git a/src/FixedWidthParser/Processors/ColumnParserFactory.cs b/src/FixedWidthParser/Processors/ColumnParserFactory.cs index 96312a8..37cdf21 100644 --- a/src/FixedWidthParser/Processors/ColumnParserFactory.cs +++ b/src/FixedWidthParser/Processors/ColumnParserFactory.cs @@ -44,30 +44,76 @@ private static ColumnParser CreateValue(Type valueType, Delegate .Invoke(null, [setter, converter, trim, trimMode])!; } - if (format is not null) + var special = TryBuildSpecial(valueType, setter, trim, trimMode, format); + if (special is not null) { - if (valueType == typeof(DateTime)) - { - return BuildDateTime((ColumnSetter)setter, format, trim, trimMode); - } - if (valueType == typeof(DateOnly)) - { - return BuildDateOnly((ColumnSetter)setter, format, trim, trimMode); - } - if (valueType == typeof(TimeOnly)) - { - return BuildTimeOnly((ColumnSetter)setter, format, trim, trimMode); - } - if (valueType == typeof(DateTimeOffset)) - { - return BuildDateTimeOffset((ColumnSetter)setter, format, trim, trimMode); - } + return special; } - if (valueType == typeof(string)) { return BuildString((ColumnSetter)setter, trim, trimMode); } + if (Implements(valueType, typeof(IUtf8SpanParsable<>))) + { + return (ColumnParser)Method(nameof(BuildUtf8Parsable), typeof(TModel), valueType).Invoke(null, [setter, trim, trimMode])!; + } + if (Implements(valueType, typeof(ISpanParsable<>))) + { + return (ColumnParser)Method(nameof(BuildDecodedParsable), typeof(TModel), valueType).Invoke(null, [setter, trim, trimMode])!; + } + + throw new InvalidOperationException( + $"Column '{memberName}' has type '{valueType}', which is not string, double, float, IUtf8SpanParsable or ISpanParsable; set a Converter for it."); + } + + internal static ColumnParser ForString(ColumnSetter setter, byte trim, TrimMode trimMode) + where TModel : allows ref struct + { + return BuildString(setter, trim, trimMode); + } + + internal static ColumnParser ForUtf8Parsable(ColumnSetter setter, byte trim, TrimMode trimMode, string? format) + where TModel : allows ref struct + where TValue : IUtf8SpanParsable + { + return TryBuildSpecial(typeof(TValue), setter, trim, trimMode, format) ?? BuildUtf8Parsable(setter, trim, trimMode); + } + + internal static ColumnParser ForDecodedParsable(ColumnSetter setter, byte trim, TrimMode trimMode, string? format) + where TModel : allows ref struct + where TValue : ISpanParsable + { + return TryBuildSpecial(typeof(TValue), setter, trim, trimMode, format) ?? BuildDecodedParsable(setter, trim, trimMode); + } + + internal static ColumnParser ForConverter(ColumnSetter setter, IFixedWidthConverter converter, byte trim, TrimMode trimMode) + where TModel : allows ref struct + { + return BuildConverter(setter, converter, trim, trimMode); + } + + internal static ColumnParser ForNullable( + ColumnSetter setter, byte trim, Func, ColumnParser> buildInner) + where TModel : allows ref struct + where TValue : struct + { + ColumnSetter adapted = (ref TModel model, TValue value) => setter(ref model, value); + return WrapNullable(setter, buildInner(adapted), trim); + } + + // The types the attribute path special-cases ahead of the generic parsers, in its order: a date or + // time with a Format, then double and float. + private static ColumnParser? TryBuildSpecial(Type valueType, Delegate setter, byte trim, TrimMode trimMode, string? format) + where TModel : allows ref struct + { + if (format is not null) + { + var date = TryBuildDate(valueType, setter, trim, trimMode, format); + if (date is not null) + { + return date; + } + } if (valueType == typeof(double)) { return BuildDouble((ColumnSetter)setter, trim, trimMode); @@ -76,17 +122,29 @@ private static ColumnParser CreateValue(Type valueType, Delegate { return BuildFloat((ColumnSetter)setter, trim, trimMode); } - if (Implements(valueType, typeof(IUtf8SpanParsable<>))) + return null; + } + + private static ColumnParser? TryBuildDate(Type valueType, Delegate setter, byte trim, TrimMode trimMode, string format) + where TModel : allows ref struct + { + if (valueType == typeof(DateTime)) { - return (ColumnParser)Method(nameof(BuildUtf8Parsable), typeof(TModel), valueType).Invoke(null, [setter, trim, trimMode])!; + return BuildDateTime((ColumnSetter)setter, format, trim, trimMode); } - if (Implements(valueType, typeof(ISpanParsable<>))) + if (valueType == typeof(DateOnly)) { - return (ColumnParser)Method(nameof(BuildDecodedParsable), typeof(TModel), valueType).Invoke(null, [setter, trim, trimMode])!; + return BuildDateOnly((ColumnSetter)setter, format, trim, trimMode); } - - throw new InvalidOperationException( - $"Column '{memberName}' has type '{valueType}', which is not string, double, float, IUtf8SpanParsable or ISpanParsable; set a Converter for it."); + if (valueType == typeof(TimeOnly)) + { + return BuildTimeOnly((ColumnSetter)setter, format, trim, trimMode); + } + if (valueType == typeof(DateTimeOffset)) + { + return BuildDateTimeOffset((ColumnSetter)setter, format, trim, trimMode); + } + return null; } private static bool Implements(Type valueType, Type openInterface) @@ -106,10 +164,14 @@ private static ColumnParser BuildNullable(Delegate where TModel : allows ref struct where TUnderlying : struct { - var nullableSetter = (ColumnSetter)setter; - ColumnSetter adapted = (ref TModel model, TUnderlying value) => nullableSetter(ref model, value); - var inner = CreateValue(typeof(TUnderlying), adapted, converterType, memberName, trim, trimMode, format); + return ForNullable((ColumnSetter)setter, trim, + adapted => CreateValue(typeof(TUnderlying), adapted, converterType, memberName, trim, trimMode, format)); + } + private static ColumnParser WrapNullable(ColumnSetter nullableSetter, ColumnParser inner, byte trim) + where TModel : allows ref struct + where TUnderlying : struct + { return (column, in context, ref model) => { if (FixedWidthRuntime.IsBlank(column, trim)) diff --git a/src/FixedWidthParser/PublicAPI.Unshipped.txt b/src/FixedWidthParser/PublicAPI.Unshipped.txt index cce3f07..aaa44d1 100644 --- a/src/FixedWidthParser/PublicAPI.Unshipped.txt +++ b/src/FixedWidthParser/PublicAPI.Unshipped.txt @@ -200,3 +200,25 @@ static FixedWidthParser.FixedWidth.WriteMany(System.IO.Stream! stream, S static FixedWidthParser.FixedWidth.WriteMany(System.IO.Stream! stream, System.ReadOnlySpan models, FixedWidthParser.FixedWidthWriterOptions? options = null) -> void static FixedWidthParser.FixedWidth.WriteManyAsync(System.IO.Stream! stream, System.Collections.Generic.IEnumerable! models, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthWriterOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! static FixedWidthParser.FixedWidth.WriteManyAsync(System.IO.Stream! stream, System.Collections.Generic.IEnumerable! models, FixedWidthParser.FixedWidthWriterOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +FixedWidthParser.FixedWidthColumnOptions +FixedWidthParser.FixedWidthColumnOptions.Alignment.get -> FixedWidthParser.Attributes.Alignment +FixedWidthParser.FixedWidthColumnOptions.Alignment.init -> void +FixedWidthParser.FixedWidthColumnOptions.FixedWidthColumnOptions() -> void +FixedWidthParser.FixedWidthColumnOptions.Format.get -> string? +FixedWidthParser.FixedWidthColumnOptions.Format.init -> void +FixedWidthParser.FixedWidthColumnOptions.Overflow.get -> FixedWidthParser.Attributes.OverflowBehavior +FixedWidthParser.FixedWidthColumnOptions.Overflow.init -> void +FixedWidthParser.FixedWidthColumnOptions.Padding.get -> char +FixedWidthParser.FixedWidthColumnOptions.Padding.init -> void +FixedWidthParser.FixedWidthColumnOptions.TrimChar.get -> char +FixedWidthParser.FixedWidthColumnOptions.TrimChar.init -> void +FixedWidthParser.FixedWidthColumnOptions.TrimMode.get -> FixedWidthParser.Attributes.TrimMode +FixedWidthParser.FixedWidthColumnOptions.TrimMode.init -> void +FixedWidthParser.FixedWidthLayoutBuilder +FixedWidthParser.FixedWidthLayoutBuilder.Column(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.FixedWidthLayoutBuilder.Column(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.FixedWidthLayoutBuilder.ConvertedColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.Processors.IFixedWidthConverter! converter, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.FixedWidthLayoutBuilder.DecodedColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.FixedWidthLayoutBuilder.Factory(System.Func! factory) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.FixedWidthLayoutBuilder.NullableColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +static FixedWidthParser.FixedWidthLayout.Build(System.Action!>! configure) -> FixedWidthParser.FixedWidthLayout! diff --git a/tests/FixedWidthParser.Tests/FluentLayoutTests.cs b/tests/FixedWidthParser.Tests/FluentLayoutTests.cs new file mode 100644 index 0000000..8f8f2a6 --- /dev/null +++ b/tests/FixedWidthParser.Tests/FluentLayoutTests.cs @@ -0,0 +1,296 @@ +using System.Text; +using FixedWidthParser.Attributes; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// A layout from reads and writes exactly like the same layout declared with attributes. + public sealed class FluentLayoutTests : IDisposable + { + private static readonly FixedWidthWriterOptions Lf = new() { NewLine = "\n" }; + + private readonly string _path = Path.GetTempFileName(); + + public void Dispose() + { + File.Delete(_path); + } + + private static FixedWidthLayout Person() + { + return FixedWidthLayout.Build(b => b + .Factory(() => new MutablePersonModel()) + .Column("Name", 0, 10, (in MutablePersonModel p) => p.Name, (ref MutablePersonModel p, string v) => p.Name = v) + .Column("Age", 10, 5, (in MutablePersonModel p) => p.Age, (ref MutablePersonModel p, int v) => p.Age = v) + .Column("Salary", 15, 10, (in MutablePersonModel p) => p.Salary, (ref MutablePersonModel p, double v) => p.Salary = v)); + } + + private static FixedWidthLayout Options() + { + return FixedWidthLayout.Build(b => b + .Factory(() => new MutableOptionsModel()) + .Column("ZeroPadded", 0, 5, (in MutableOptionsModel m) => m.ZeroPadded, (ref MutableOptionsModel m, int v) => m.ZeroPadded = v, + new FixedWidthColumnOptions { Alignment = Alignment.Right, Padding = '0' }) + .Column("Fixed2", 5, 8, (in MutableOptionsModel m) => m.Fixed2, (ref MutableOptionsModel m, double v) => m.Fixed2 = v, + new FixedWidthColumnOptions { Format = "F2" }) + .Column("Truncated", 13, 4, (in MutableOptionsModel m) => m.Truncated, (ref MutableOptionsModel m, string v) => m.Truncated = v, + new FixedWidthColumnOptions { Alignment = Alignment.Right, Overflow = OverflowBehavior.Truncate }) + .Column("Starred", 17, 5, (in MutableOptionsModel m) => m.Starred, (ref MutableOptionsModel m, int v) => m.Starred = v, + new FixedWidthColumnOptions { TrimChar = '*' }) + .NullableColumn("NullableAge", 22, 5, (in MutableOptionsModel m) => m.NullableAge, (ref MutableOptionsModel m, int? v) => m.NullableAge = v) + .NullableColumn("NullableAmount", 27, 10, (in MutableOptionsModel m) => m.NullableAmount, (ref MutableOptionsModel m, decimal? v) => m.NullableAmount = v) + .ConvertedColumn("Cents", 37, 8, (in MutableOptionsModel m) => m.Cents, (ref MutableOptionsModel m, CentsValue v) => m.Cents = v, new CentsConverter()) + .DecodedColumn("Date", 45, 8, (in MutableOptionsModel m) => m.Date, (ref MutableOptionsModel m, DateOnly v) => m.Date = v, + new FixedWidthColumnOptions { Format = "yyyyMMdd" })); + } + + [Theory] + [InlineData("John Doe 30 6000000.00")] + [InlineData("Jane 28 55000.00 ")] + [InlineData("John")] + [InlineData("")] + [InlineData("Ann abc 10.5 ")] + public void Fluent_TryParse_MatchesAttributes(string line) + { + bool okA = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var a); + bool okF = FixedWidth.TryParse(Bytes(line), Person(), Ctx, out var f); + + Assert.Equal(okA, okF); + if (okA) + { + Assert.Equal(a, f); + } + } + + [Fact] + public void Fluent_WriteMany_MatchesAttributes() + { + MutablePersonModel[] people = [new() { Name = "John Doe", Age = 30, Salary = 6000000 }, new() { Name = "VeryLongName", Age = 1 }, new()]; + + Assert.Equal(Write(people, FixedWidthLayout.FromAttributes()), Write(people, Person())); + } + + [Theory] + [InlineData(42, 3.14159, "ABCDEF", 7, 30, "12.5")] + [InlineData(-3, -0.5, "X", 0, null, null)] + public void Fluent_Options_MatchAttributes(int zero, double fixed2, string truncated, int starred, int? age, string? amount) + { + var model = new MutableOptionsModel + { + ZeroPadded = zero, + Fixed2 = fixed2, + Truncated = truncated, + Starred = starred, + NullableAge = age, + NullableAmount = amount is null ? null : decimal.Parse(amount, Inv), + Cents = new CentsValue(1234), + Date = new DateOnly(2026, 10, 23), + }; + var attributes = FixedWidthLayout.FromAttributes(); + + string attributeLine = Write([model], attributes); + Assert.Equal(attributeLine, Write([model], Options())); + + byte[] line = Bytes(attributeLine.TrimEnd('\n')); + Assert.True(FixedWidth.TryParse(line, attributes, Ctx, out var a)); + Assert.True(FixedWidth.TryParse(line, Options(), Ctx, out var f)); + Assert.Equal(a, f); + } + + [Fact] + public void Fluent_StarTrim_ParsesLikeAttributes() + { + byte[] line = Bytes("00042" + "3.14 " + "ABCD" + "42***" + new string(' ', 15) + "1234 " + "20261023"); + + Assert.True(FixedWidth.TryParse(line, FixedWidthLayout.FromAttributes(), Ctx, out var a)); + Assert.True(FixedWidth.TryParse(line, Options(), Ctx, out var f)); + Assert.Equal(42, f.Starred); + Assert.Equal(a, f); + } + + [Fact] + public void Fluent_ParseError_SameExceptionAsAttributes() + { + File.WriteAllText(_path, "John Doe 30 6000000.00\nJane abc 55000.00 \n"); + + var a = Assert.Throws(() => FixedWidth.ReadFile(_path, FixedWidthLayout.FromAttributes()).ToList()); + var f = Assert.Throws(() => FixedWidth.ReadFile(_path, Person()).ToList()); + + Assert.Equal((a.LineNumber, a.ColumnName, a.ColumnStart, a.ColumnLength, a.RawValue), (f.LineNumber, f.ColumnName, f.ColumnStart, f.ColumnLength, f.RawValue)); + Assert.Equal(2, f.LineNumber); + Assert.Equal("Age", f.ColumnName); + } + + [Fact] + public void Fluent_ShortLine_ReportsRequiredLength() + { + File.WriteAllText(_path, "John\n"); + + var ex = Assert.Throws(() => FixedWidth.ReadFile(_path, Person()).ToList()); + + Assert.Contains("the layout needs 25", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Fluent_StructModel_SetterMutatesResult() + { + var layout = FixedWidthLayout.Build(b => b + .Column("Name", 0, 10, (in MutablePersonStruct p) => p.Name, (ref MutablePersonStruct p, string v) => p.Name = v) + .Column("Age", 10, 5, (in MutablePersonStruct p) => p.Age, (ref MutablePersonStruct p, int v) => p.Age = v) + .Column("Salary", 15, 10, (in MutablePersonStruct p) => p.Salary, (ref MutablePersonStruct p, double v) => p.Salary = v)); + + Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, layout, Ctx, out var p)); + + Assert.Equal(new MutablePersonStruct { Name = "John Doe", Age = 30, Salary = 6000000 }, p); + } + + [Fact] + public void Fluent_RefStruct_TryParseAndTryFormat() + { + var layout = FixedWidthLayout.Build(b => b + .Column("Name", 0, 10, (in RefPersonModel p) => p.Name, (ref RefPersonModel p, string v) => p.Name = v) + .Column("Age", 10, 5, (in RefPersonModel p) => p.Age, (ref RefPersonModel p, int v) => p.Age = v) + .Column("Salary", 15, 10, (in RefPersonModel p) => p.Salary, (ref RefPersonModel p, double v) => p.Salary = v)); + Span buffer = stackalloc byte[25]; + + Assert.True(FixedWidth.TryParse("John Doe 30 6000000 "u8, layout, Ctx, out var p)); + Assert.True(FixedWidth.TryFormat(in p, layout, buffer, Ctx, out int written)); + + Assert.Equal("John Doe 30 6000000 ", Encoding.UTF8.GetString(buffer[..written])); + } + + [Fact] + public void Fluent_WriteOnlyColumn_ReadThrowsAtCall() + { + var layout = FixedWidthLayout.Build(b => b + .Factory(() => new MutablePersonModel()) + .Column("Name", 0, 10, (in MutablePersonModel p) => p.Name, null)); + File.WriteAllText(_path, "John Doe \n"); + + var ex = Assert.Throws(() => FixedWidth.ReadFile(_path, layout)); + + Assert.Contains("without a setter", ex.Message, StringComparison.Ordinal); + Assert.Equal("John Doe \n", Write([new MutablePersonModel { Name = "John Doe" }], layout)); + } + + [Fact] + public void Fluent_ReadOnlyColumn_WriteThrows() + { + var layout = FixedWidthLayout.Build(b => b + .Factory(() => new MutablePersonModel()) + .Column("Name", 0, 10, null, (ref MutablePersonModel p, string v) => p.Name = v)); + using var stream = new MemoryStream(); + + var ex = Assert.Throws(() => FixedWidth.WriteMany(stream, [new MutablePersonModel()], layout)); + + Assert.Contains("without a getter", ex.Message, StringComparison.Ordinal); + Assert.True(FixedWidth.TryParse("John Doe "u8, layout, Ctx, out var p)); + Assert.Equal("John Doe", p.Name); + } + + [Fact] + public void Fluent_Overlap_NamesColumns() + { + var attributes = Assert.Throws(() => FixedWidthLayout.FromAttributes()); + + var fluent = Assert.Throws(() => FixedWidthLayout.Build(b => b + .Column("Left", 0, 5, (in OverlapReadModel m) => m.Left, null) + .Column("Right", 2, 5, (in OverlapReadModel m) => m.Right, null))); + + Assert.Equal(attributes.Message, fluent.Message); + } + + [Fact] + public void Fluent_NoColumns_Throws() + { + var ex = Assert.Throws(() => FixedWidthLayout.Build(_ => { })); + + Assert.Contains("has no columns", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Fluent_NegativeStart_Throws() + { + var ex = Assert.Throws(() => FixedWidthLayout.Build(b => b + .Column("Name", -1, 10, (in MutablePersonStruct p) => p.Name, null))); + + Assert.Contains("negative Start", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Fluent_ZeroLength_Throws() + { + var ex = Assert.Throws(() => FixedWidthLayout.Build(b => b + .Column("Name", 0, 0, (in MutablePersonStruct p) => p.Name, null))); + + Assert.Contains("invalid Length", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Fluent_NonAsciiPadding_Throws() + { + var ex = Assert.Throws(() => FixedWidthLayout.Build(b => b + .Column("Name", 0, 10, (in MutablePersonStruct p) => p.Name, null, new FixedWidthColumnOptions { Padding = 'é' }))); + + Assert.Contains("Padding", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Fluent_TruncateOnNonString_Throws() + { + var ex = Assert.Throws(() => FixedWidthLayout.Build(b => b + .Column("Age", 0, 5, (in MutablePersonStruct p) => p.Age, null, new FixedWidthColumnOptions { Overflow = OverflowBehavior.Truncate }))); + + Assert.Contains("Truncate", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Fluent_ClassWithoutFactory_Throws() + { + var ex = Assert.Throws(() => FixedWidthLayout.Build(b => b + .Column("Name", 0, 10, (in MutablePersonModel p) => p.Name, (ref MutablePersonModel p, string v) => p.Name = v))); + + Assert.Contains("Factory", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task Fluent_AggregateFileAsync_MatchesSequential() + { + var text = new StringBuilder(); + for (int i = 0; i < 100_000; i++) + { + text.Append(("N" + (i % 10)).PadRight(10)).Append((i % 1000).ToString(Inv).PadRight(5)).Append(i.ToString(Inv).PadRight(10)).Append('\n'); + } + File.WriteAllText(_path, text.ToString()); + var layout = Person(); + + var parallel = await FixedWidth.AggregateFileAsync( + _path, layout, parallel: new FixedWidthParallelOptions { DegreeOfParallelism = 4 }, ct: TestContext.Current.CancellationToken); + + Assert.Equal(FixedWidth.ReadFile(_path, layout).Select(p => p.Age), parallel.Values); + Assert.Equal(100_000, parallel.Values.Count); + } + + public sealed class Ages : IRecordAccumulator + { + public List Values { get; } = []; + + public void Add(in MutablePersonModel model) + { + Values.Add(model.Age); + } + + public void Merge(Ages following) + { + Values.AddRange(following.Values); + } + } + + private static string Write(T[] models, FixedWidthLayout layout) + { + using var stream = new MemoryStream(); + FixedWidth.WriteMany(stream, models, layout, Lf); + return Encoding.UTF8.GetString(stream.ToArray()); + } + } +} diff --git a/tests/FixedWidthParser.Tests/TestModels.cs b/tests/FixedWidthParser.Tests/TestModels.cs index 27bb584..fb04a55 100644 --- a/tests/FixedWidthParser.Tests/TestModels.cs +++ b/tests/FixedWidthParser.Tests/TestModels.cs @@ -431,4 +431,33 @@ public readonly record struct PriceModel { [FixedColumn(0, 8, Alignment = Alignment.Right, Format = "C")] public decimal Price { get; init; } } + + /// Settable as a class, so a fluent layout and its attributes can bind the same type. + public sealed record class MutablePersonModel + { + [FixedColumn(0, 10)] public string Name { get; set; } = ""; + [FixedColumn(10, 5)] public int Age { get; set; } + [FixedColumn(15, 10)] public double Salary { get; set; } + } + + /// Settable as a struct: a fluent setter must reach the returned model, not a copy. + public record struct MutablePersonStruct + { + [FixedColumn(0, 10)] public string Name { get; set; } + [FixedColumn(10, 5)] public int Age { get; set; } + [FixedColumn(15, 10)] public double Salary { get; set; } + } + + /// One settable column per formatting and parsing option, for fluent-versus-attribute parity. + public sealed record class MutableOptionsModel + { + [FixedColumn(0, 5, Alignment = Alignment.Right, Padding = '0')] public int ZeroPadded { get; set; } + [FixedColumn(5, 8, Format = "F2")] public double Fixed2 { get; set; } + [FixedColumn(13, 4, Alignment = Alignment.Right, Overflow = OverflowBehavior.Truncate)] public string Truncated { get; set; } = ""; + [FixedColumn(17, 5, TrimChar = '*')] public int Starred { get; set; } + [FixedColumn(22, 5)] public int? NullableAge { get; set; } + [FixedColumn(27, 10)] public decimal? NullableAmount { get; set; } + [FixedColumn(37, 8, Converter = typeof(CentsConverter))] public CentsValue Cents { get; set; } + [FixedColumn(45, 8, Format = "yyyyMMdd")] public DateOnly Date { get; set; } + } } \ No newline at end of file From b640bcb51ed75ec13aa699c78bd3de4580f65358 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 23 Oct 2026 22:41:16 -0300 Subject: [PATCH 21/85] feat!: make FixedWidth the only entry point and drop the runtime classes FixedWidthParser, FixedWidthReader and FixedWidthWriter are removed. Attribute-mapped models are read and written through FixedWidth.*(..., FixedWidthLayout.FromAttributes(), ...), the same overloads fluent layouts use. Tests and benchmarks are migrated; the package validation suppressions now list the removed types. BREAKING CHANGE: replace new FixedWidthReader(options).ReadFile(path) with FixedWidth.ReadFile(path, FixedWidthLayout.FromAttributes(), options), and likewise for the parser and the writer. --- .../FixedWidthModelGenerator.cs | 4 +- .../CompatibilitySuppressions.xml | 184 +++--------------- src/FixedWidthParser/FixedWidth.cs | 16 +- .../FixedWidthReaderOptions.cs | 3 +- src/FixedWidthParser/IFixedWidthModel.cs | 4 +- .../Parsers/FixedWidthParser.cs | 33 ---- src/FixedWidthParser/Parsers/ParserBuilder.cs | 2 +- src/FixedWidthParser/PublicAPI.Unshipped.txt | 20 -- .../Readers/FixedWidthReader.cs | 95 --------- .../Writers/FixedWidthWriter.cs | 74 ------- .../Benchmarks/Perf/AsyncReaderBenchmarks.cs | 12 +- tests/Benchmarks/Perf/ComparisonBenchmarks.cs | 4 +- .../Perf/ComparisonWriterBenchmarks.cs | 6 +- tests/Benchmarks/Perf/ParserBenchmarks.cs | 7 +- tests/Benchmarks/Perf/ReaderBenchmarks.cs | 12 +- .../Perf/ReaderPoolingBenchmarks.cs | 15 +- .../Perf/ReaderStartupBenchmarks.cs | 2 +- .../Perf/SourceGenParserBenchmarks.cs | 5 +- tests/Benchmarks/Perf/WriterBenchmarks.cs | 10 +- tests/Benchmarks/SampleData.cs | 4 +- .../AllocationGuardTests.cs | 20 +- .../ColumnValidationTests.cs | 25 ++- .../ConverterAttributeTests.cs | 14 +- .../CorrectnessTests.cs | 30 ++- tests/FixedWidthParser.Tests/CoverageTests.cs | 7 +- tests/FixedWidthParser.Tests/CultureTests.cs | 52 ++--- .../FixedWidthParser.Tests/GeneratedModels.cs | 2 +- .../GeneratedReaderTests.cs | 8 +- .../GeneratedWriterTests.cs | 40 ++-- .../LayoutWriteTests.cs | 3 +- .../NullableColumnTests.cs | 22 +-- .../ParallelAggregateTests.cs | 23 ++- tests/FixedWidthParser.Tests/ParserTests.cs | 53 +++-- .../PropertyRoundTripTests.cs | 20 +- .../ReaderOptionsTests.cs | 30 +-- tests/FixedWidthParser.Tests/ReaderTests.cs | 84 ++++---- .../FixedWidthParser.Tests/RoundTripTests.cs | 14 +- .../SourceGenParityTests.cs | 21 +- tests/FixedWidthParser.Tests/TestHelpers.cs | 19 +- tests/FixedWidthParser.Tests/TrimCharTests.cs | 7 +- .../WriterAsyncTests.cs | 20 +- .../WriterEncodingTests.cs | 38 ++-- .../WriterFormattingTests.cs | 43 ++-- tests/FixedWidthParser.Tests/WriterTests.cs | 43 ++-- 44 files changed, 394 insertions(+), 756 deletions(-) delete mode 100644 src/FixedWidthParser/Parsers/FixedWidthParser.cs delete mode 100644 src/FixedWidthParser/Readers/FixedWidthReader.cs delete mode 100644 src/FixedWidthParser/Writers/FixedWidthWriter.cs diff --git a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs index ad8f30f..f312e08 100644 --- a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs +++ b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs @@ -193,7 +193,7 @@ public void Initialize(IncrementalGeneratorInitializationContext context) string? converterFqn = converterType?.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); var kind = Classify(underlying, converterType, format, compilation); var writeKind = ClassifyWrite(underlying, converterType, compilation); - // Mirrors FixedWidthWriter: an explicit Overflow wins; otherwise string truncates and everything else throws. + // Mirrors FormatterBuilder.ResolveOptions: an explicit Overflow wins; otherwise string truncates and everything else throws. int resolvedOverflow = overflow; if (resolvedOverflow == 0) { @@ -248,7 +248,7 @@ private static ColumnKind Classify(ITypeSymbol type, ITypeSymbol? converterType, return ImplementsGeneric(type, type, compilation, "System.ISpanParsable`1") ? ColumnKind.DecodedParsable : ColumnKind.Unsupported; } - // Mirrors FixedWidthWriter: converter → string → ISpanFormattable. + // Mirrors FormatterBuilder: converter → string → ISpanFormattable. private static WriteKind ClassifyWrite(ITypeSymbol type, ITypeSymbol? converterType, Compilation compilation) { if (converterType is not null) diff --git a/src/FixedWidthParser/CompatibilitySuppressions.xml b/src/FixedWidthParser/CompatibilitySuppressions.xml index 9970ad0..1055937 100644 --- a/src/FixedWidthParser/CompatibilitySuppressions.xml +++ b/src/FixedWidthParser/CompatibilitySuppressions.xml @@ -1,4 +1,4 @@ - + @@ -29,6 +29,13 @@ lib/net10.0/FixedWidthParser.dll true + + CP0001 + T:FixedWidthParser.Parsers.FixedWidthParser`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + CP0001 T:FixedWidthParser.Parsers.Utf8FixedWidthParser`1 @@ -99,6 +106,13 @@ lib/net10.0/FixedWidthParser.dll true + + CP0001 + T:FixedWidthParser.Readers.FixedWidthReader`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + CP0001 T:FixedWidthParser.Readers.FixedWidthRecordEnumerable`1 @@ -162,6 +176,13 @@ lib/net10.0/FixedWidthParser.dll true + + CP0001 + T:FixedWidthParser.Readers.ReflectionLineParser`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + CP0001 T:FixedWidthParser.Readers.ReflectionUtf8LineParser`1 @@ -225,6 +246,13 @@ lib/net10.0/FixedWidthParser.dll true + + CP0001 + T:FixedWidthParser.Writers.FixedWidthWriter`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + CP0002 M:FixedWidthParser.FixedWidth.Read``1(System.IO.Stream,System.Text.Encoding,System.Boolean,System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,System.Int32) @@ -316,62 +344,6 @@ lib/net10.0/FixedWidthParser.dll true - - CP0002 - M:FixedWidthParser.Parsers.FixedWidthParser`1.TryParse(System.ReadOnlySpan{System.Char},System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,`0@) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Readers.FixedWidthReader`1.#ctor(System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,System.Int32) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Readers.FixedWidthReader`1.Read(System.IO.Stream,System.Text.Encoding,System.Boolean) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Readers.FixedWidthReader`1.Read(System.IO.TextReader) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Readers.FixedWidthReader`1.ReadAsync(System.IO.Stream,System.Text.Encoding,System.Boolean) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Readers.FixedWidthReader`1.ReadAsync(System.IO.TextReader) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Readers.FixedWidthReader`1.ReadFile(System.String,System.Text.Encoding) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Readers.FixedWidthReader`1.ReadFileAsync(System.String,System.Text.Encoding) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - CP0002 M:FixedWidthParser.Readers.GeneratedLineParser`1.TryParse(System.ReadOnlySpan{System.Char},System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,`0@) @@ -379,97 +351,6 @@ lib/net10.0/FixedWidthParser.dll true - - CP0002 - M:FixedWidthParser.Readers.ReflectionLineParser`1.#ctor(FixedWidthParser.Parsers.FixedWidthParser{`0}) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Readers.ReflectionLineParser`1.TryParse(System.ReadOnlySpan{System.Char},System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,`0@) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Writers.FixedWidthWriter`1.#ctor - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Writers.FixedWidthWriter`1.Write(System.IO.Stream,`0@,System.IFormatProvider) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Writers.FixedWidthWriter`1.Write(System.IO.StreamWriter,`0@,System.IFormatProvider) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Writers.FixedWidthWriter`1.WriteAsync(System.IO.Stream,`0,System.IFormatProvider,System.Threading.CancellationToken) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Writers.FixedWidthWriter`1.WriteAsync(System.IO.StreamWriter,`0,System.IFormatProvider,System.Threading.CancellationToken) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Writers.FixedWidthWriter`1.WriteMany(System.IO.Stream,System.Collections.Generic.IEnumerable{`0},System.IFormatProvider) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Writers.FixedWidthWriter`1.WriteMany(System.IO.Stream,System.ReadOnlySpan{`0},System.IFormatProvider) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Writers.FixedWidthWriter`1.WriteMany(System.IO.StreamWriter,System.Collections.Generic.IEnumerable{`0},System.IFormatProvider) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Writers.FixedWidthWriter`1.WriteMany(System.IO.StreamWriter,System.ReadOnlySpan{`0},System.IFormatProvider) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Writers.FixedWidthWriter`1.WriteManyAsync(System.IO.Stream,System.Collections.Generic.IEnumerable{`0},System.IFormatProvider,System.Threading.CancellationToken) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Writers.FixedWidthWriter`1.WriteManyAsync(System.IO.StreamWriter,System.Collections.Generic.IEnumerable{`0},System.IFormatProvider,System.Threading.CancellationToken) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - CP0006 M:FixedWidthParser.IFixedWidthModel`1.TryFormat(`0@,System.Span{System.Byte},FixedWidthParser.FixedWidthContext@,System.Int32@) @@ -491,13 +372,6 @@ lib/net10.0/FixedWidthParser.dll true - - CP0008 - T:FixedWidthParser.Readers.ReflectionLineParser`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - PKV006 net8.0 diff --git a/src/FixedWidthParser/FixedWidth.cs b/src/FixedWidthParser/FixedWidth.cs index 3561030..e32305e 100644 --- a/src/FixedWidthParser/FixedWidth.cs +++ b/src/FixedWidthParser/FixedWidth.cs @@ -4,9 +4,10 @@ namespace FixedWidthParser { /// - /// Entry points for source-generated models. Dispatches to the model's generated - /// and - /// through the type parameter, so the call is statically resolved (no reflection, no delegates). + /// Parses, reads, aggregates, formats and writes fixed-width lines. Every operation comes in two + /// overloads: one for a source-generated model (), resolved + /// statically through the type parameter, and one that takes a + /// built from attributes or fluently. /// Column positions are bytes. /// public static class FixedWidth @@ -119,7 +120,12 @@ public static FixedWidthAsyncRecordEnumerable> /// Reads a file across several threads and folds every record into a , /// one instance per partition, merged in file order. /// - /// Carries the same contract as . + /// + /// The result equals one sequential pass when is + /// associative. Every line is parsed exactly once, and errors report the same line number as a + /// sequential read. runs on the worker threads, + /// possibly concurrently, and the first failure in file order is the one thrown. + /// public static Task AggregateFileAsync( string path, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken ct = default) where TModel : IFixedWidthModel @@ -149,7 +155,7 @@ public static Task AggregateFileAsync( /// Reads a stream once and folds every record into a across /// several threads. controls whether the stream is disposed afterwards. /// - /// Carries the same contract as . + /// Carries the same contract as . public static Task AggregateAsync( Stream stream, bool leaveOpen = false, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken ct = default) where TModel : IFixedWidthModel diff --git a/src/FixedWidthParser/FixedWidthReaderOptions.cs b/src/FixedWidthParser/FixedWidthReaderOptions.cs index e6d436a..1b195bc 100644 --- a/src/FixedWidthParser/FixedWidthReaderOptions.cs +++ b/src/FixedWidthParser/FixedWidthReaderOptions.cs @@ -4,8 +4,7 @@ namespace FixedWidthParser { /// - /// Settings for reading lines: and the - /// read methods. Every property has a default, so + /// Settings for the parse, read and aggregate methods. Every property has a default, so /// options and new FixedWidthReaderOptions() read the same way. /// public sealed class FixedWidthReaderOptions diff --git a/src/FixedWidthParser/IFixedWidthModel.cs b/src/FixedWidthParser/IFixedWidthModel.cs index 6114806..20c856b 100644 --- a/src/FixedWidthParser/IFixedWidthModel.cs +++ b/src/FixedWidthParser/IFixedWidthModel.cs @@ -4,7 +4,7 @@ namespace FixedWidthParser /// Implemented by models that opt into the compile-time generated parser and writer. Declaring /// this interface on a model is the trigger for the source generator, /// which fills in and with reflection-free code - /// mirroring and . + /// that reads and writes exactly like the model's layout. /// Column positions are bytes. /// public interface IFixedWidthModel where TSelf : IFixedWidthModel, allows ref struct @@ -19,7 +19,7 @@ public interface IFixedWidthModel where TSelf : IFixedWidthModel, /// /// Formats into , without a line break, - /// mirroring . Returns only + /// mirroring the model's attribute layout. Returns only /// when is shorter than the line; a value that does not fit its /// column throws or truncates per its FixedColumnAttribute.Overflow. /// diff --git a/src/FixedWidthParser/Parsers/FixedWidthParser.cs b/src/FixedWidthParser/Parsers/FixedWidthParser.cs deleted file mode 100644 index 57e1851..0000000 --- a/src/FixedWidthParser/Parsers/FixedWidthParser.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System.Diagnostics.CodeAnalysis; - -namespace FixedWidthParser.Parsers -{ - /// - /// Runtime (reflection-built) parser for one fixed-width line of bytes. Accepts ref struct - /// models; for reading many lines use . - /// Column positions are bytes. - /// - public sealed class FixedWidthParser where TModel : new(), allows ref struct - { - private readonly FixedWidthLayout _layout; - - /// Builds the parser, throwing when the model's layout is invalid. - [RequiresUnreferencedCode("Attribute layouts reflect over TModel's public properties and fields, which trimming may remove.")] - [RequiresDynamicCode("Attribute layouts compile member accessors at runtime (Expression.Compile / MakeGenericType).")] - public FixedWidthParser() - { - _layout = FixedWidthLayout.FromAttributes(); - _layout.EnsureCanRead(); - } - - /// - /// Parses a single line, without its line break, into . Returns - /// when the line is shorter than the layout or a non-string column fails - /// to parse, after reporting why through . - /// - public bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TModel model) - { - return _layout.TryParse(line, in context, out model); - } - } -} diff --git a/src/FixedWidthParser/Parsers/ParserBuilder.cs b/src/FixedWidthParser/Parsers/ParserBuilder.cs index 47f03fa..5ef734e 100644 --- a/src/FixedWidthParser/Parsers/ParserBuilder.cs +++ b/src/FixedWidthParser/Parsers/ParserBuilder.cs @@ -10,7 +10,7 @@ namespace FixedWidthParser.Parsers internal readonly record struct ColumnParserInfo(int Start, int Length, string Name, ColumnParser Parse) where TModel : allows ref struct; - /// Static-build logic for : the compiled model factory, the per-column setters and the required line length. + /// Static-build logic for : the compiled model factory, the per-column setters and the required line length. internal static class ParserBuilder { public static Func BuildModelFactory() where TModel : new(), allows ref struct diff --git a/src/FixedWidthParser/PublicAPI.Unshipped.txt b/src/FixedWidthParser/PublicAPI.Unshipped.txt index aaa44d1..8581501 100644 --- a/src/FixedWidthParser/PublicAPI.Unshipped.txt +++ b/src/FixedWidthParser/PublicAPI.Unshipped.txt @@ -89,9 +89,6 @@ FixedWidthParser.Formatters.ColumnFormatOptions.WriteInto(System.ReadOnlySpan FixedWidthParser.IFixedWidthModel.TryFormat(in TSelf model, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int bytesWritten) -> bool FixedWidthParser.IFixedWidthModel.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TSelf model) -> bool -FixedWidthParser.Parsers.FixedWidthParser -FixedWidthParser.Parsers.FixedWidthParser.FixedWidthParser() -> void -FixedWidthParser.Parsers.FixedWidthParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool FixedWidthParser.Processors.IFixedWidthConverter FixedWidthParser.Processors.IFixedWidthConverter.TryFormat(T value, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int written) -> bool FixedWidthParser.Processors.IFixedWidthConverter.TryParse(System.ReadOnlySpan field, in FixedWidthParser.FixedWidthContext context, out T value) -> bool @@ -101,9 +98,6 @@ FixedWidthParser.Readers.AsyncRecordEnumerator.DisposeAsync() - FixedWidthParser.Readers.AsyncRecordEnumerator.MoveNextAsync() -> System.Threading.Tasks.ValueTask FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable.GetAsyncEnumerator(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> FixedWidthParser.Readers.AsyncRecordEnumerator! -FixedWidthParser.Readers.FixedWidthReader -FixedWidthParser.Readers.FixedWidthReader.FixedWidthReader(FixedWidthParser.FixedWidthReaderOptions? options = null) -> void -FixedWidthParser.Readers.FixedWidthReader.TryParse(System.ReadOnlySpan line, out TModel model) -> bool FixedWidthParser.Readers.FixedWidthRecordEnumerable FixedWidthParser.Readers.FixedWidthRecordEnumerable.GetEnumerator() -> FixedWidthParser.Readers.RecordEnumerator FixedWidthParser.Readers.GeneratedLineParser @@ -117,14 +111,6 @@ FixedWidthParser.Readers.RecordEnumerator.Dispose() -> void FixedWidthParser.Readers.RecordEnumerator.MoveNext() -> bool FixedWidthParser.Readers.RecordEnumerator.RecordEnumerator() -> void FixedWidthParser.Readers.RecordEnumerator.Reset() -> void -FixedWidthParser.Writers.FixedWidthWriter -FixedWidthParser.Writers.FixedWidthWriter.FixedWidthWriter(FixedWidthParser.FixedWidthWriterOptions? options = null) -> void -FixedWidthParser.Writers.FixedWidthWriter.TryFormat(in TModel model, System.Span destination, out int bytesWritten) -> bool -FixedWidthParser.Writers.FixedWidthWriter.Write(System.IO.Stream! stream, in TModel model) -> void -FixedWidthParser.Writers.FixedWidthWriter.WriteAsync(System.IO.Stream! stream, TModel model, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! -FixedWidthParser.Writers.FixedWidthWriter.WriteMany(System.IO.Stream! stream, System.Collections.Generic.IEnumerable! models) -> void -FixedWidthParser.Writers.FixedWidthWriter.WriteMany(System.IO.Stream! stream, System.ReadOnlySpan models) -> void -FixedWidthParser.Writers.FixedWidthWriter.WriteManyAsync(System.IO.Stream! stream, System.Collections.Generic.IEnumerable! models, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! override FixedWidthParser.FixedWidthContext.GetHashCode() -> int override FixedWidthParser.Formatters.ColumnFormatOptions.GetHashCode() -> int static FixedWidthParser.FixedWidth.Read(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! @@ -164,9 +150,7 @@ FixedWidthParser.FixedWidthParallelOptions.FixedWidthParallelOptions() -> void FixedWidthParser.IRecordAccumulator FixedWidthParser.IRecordAccumulator.Add(in TModel model) -> void FixedWidthParser.IRecordAccumulator.Merge(TSelf following) -> void -FixedWidthParser.Readers.FixedWidthReader.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! static FixedWidthParser.FixedWidth.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! -FixedWidthParser.Readers.FixedWidthReader.AggregateAsync(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! static FixedWidthParser.FixedWidth.AggregateAsync(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! FixedWidthParser.ColumnGetter FixedWidthParser.ColumnSetter @@ -177,10 +161,6 @@ static FixedWidthParser.FixedWidth.TryParse(System.ReadOnlySpan li static FixedWidthParser.FixedWidthLayout.FromAttributes() -> FixedWidthParser.FixedWidthLayout! virtual FixedWidthParser.ColumnGetter.Invoke(in TModel model) -> TValue virtual FixedWidthParser.ColumnSetter.Invoke(ref TModel model, TValue value) -> void -FixedWidthParser.Readers.FixedWidthReader.Read(System.IO.Stream! stream, bool leaveOpen = false) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! -FixedWidthParser.Readers.FixedWidthReader.ReadAsync(System.IO.Stream! stream, bool leaveOpen = false) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! -FixedWidthParser.Readers.FixedWidthReader.ReadFile(string! path) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! -FixedWidthParser.Readers.FixedWidthReader.ReadFileAsync(string! path) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! FixedWidthParser.Readers.LayoutLineParser FixedWidthParser.Readers.LayoutLineParser.LayoutLineParser() -> void FixedWidthParser.Readers.LayoutLineParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool diff --git a/src/FixedWidthParser/Readers/FixedWidthReader.cs b/src/FixedWidthParser/Readers/FixedWidthReader.cs deleted file mode 100644 index 9038c41..0000000 --- a/src/FixedWidthParser/Readers/FixedWidthReader.cs +++ /dev/null @@ -1,95 +0,0 @@ -using System.Diagnostics.CodeAnalysis; - -namespace FixedWidthParser.Readers -{ - /// - /// Reads attribute-mapped models from a stream or a file, lazily and with low allocation, - /// through the runtime (reflection-built) parser. Configured once; reusable across reads. For - /// source-generated models use . - /// Column positions are bytes. - /// - public sealed class FixedWidthReader where TModel : new() - { - private readonly FixedWidthLayout _layout; - private readonly FixedWidthReaderOptions _options; - private readonly FixedWidthContext _context; - - /// Creates a reader; options use every default. - [RequiresUnreferencedCode("Attribute layouts reflect over TModel's public properties and fields, which trimming may remove.")] - [RequiresDynamicCode("Attribute layouts compile member accessors at runtime (Expression.Compile / MakeGenericType).")] - public FixedWidthReader(FixedWidthReaderOptions? options = null) - { - _layout = FixedWidthLayout.FromAttributes(); - _layout.EnsureCanRead(); - _options = FixedWidthReaderOptions.Validate(options); - _context = _options.Context; - } - - /// Parses a single line, without its line break, using the reader's options. - public bool TryParse(ReadOnlySpan line, out TModel model) - { - return _layout.TryParse(line, in _context, out model); - } - - /// Reads from a stream, once. controls whether the stream is disposed when iteration ends. - public FixedWidthRecordEnumerable> Read(Stream stream, bool leaveOpen = false) - { - ArgumentNullException.ThrowIfNull(stream); - return new(new LayoutLineParser(_layout), stream, ownsStream: !leaveOpen, _options); - } - - /// Reads from a file, reopening it on each enumeration. - public FixedWidthRecordEnumerable> ReadFile(string path) - { - ArgumentException.ThrowIfNullOrEmpty(path); - return new(new LayoutLineParser(_layout), path, _options); - } - - /// Reads from a stream via await foreach, once. controls whether the stream is disposed when iteration ends. - public FixedWidthAsyncRecordEnumerable> ReadAsync(Stream stream, bool leaveOpen = false) - { - ArgumentNullException.ThrowIfNull(stream); - return new(new LayoutLineParser(_layout), stream, ownsStream: !leaveOpen, _options); - } - - /// Reads from a file via await foreach with asynchronous I/O, reopening it on each enumeration. - public FixedWidthAsyncRecordEnumerable> ReadFileAsync(string path) - { - ArgumentException.ThrowIfNullOrEmpty(path); - return new(new LayoutLineParser(_layout), path, _options); - } - - /// - /// Reads a file across several threads and folds every record into a , - /// one instance per partition, merged in file order. - /// - /// - /// The result equals one sequential pass when is - /// associative. Every line is parsed exactly once, and errors report the same line number as a - /// sequential read. runs on the worker threads, - /// possibly concurrently, and the first failure in file order is the one thrown. - /// - public Task AggregateFileAsync(string path, FixedWidthParallelOptions? parallel = null, CancellationToken ct = default) - where TAccumulator : IRecordAccumulator, new() - { - ArgumentException.ThrowIfNullOrEmpty(path); - return ParallelAggregator.RunAsync, TAccumulator>( - new LayoutLineParser(_layout), path, _options, FixedWidthParallelOptions.Validate(parallel), ct); - } - - /// - /// Reads a stream once and folds every record into a across - /// several threads: one thread reads the stream into chunks, the others parse them, one instance - /// per chunk, merged in stream order. controls whether the stream is - /// disposed afterwards. - /// - /// Carries the same contract as . - public Task AggregateAsync(Stream stream, bool leaveOpen = false, FixedWidthParallelOptions? parallel = null, CancellationToken ct = default) - where TAccumulator : IRecordAccumulator, new() - { - ArgumentNullException.ThrowIfNull(stream); - return ParallelStreamAggregator.RunAsync, TAccumulator>( - new LayoutLineParser(_layout), stream, leaveOpen, _options, FixedWidthParallelOptions.Validate(parallel), ct); - } - } -} diff --git a/src/FixedWidthParser/Writers/FixedWidthWriter.cs b/src/FixedWidthParser/Writers/FixedWidthWriter.cs deleted file mode 100644 index 0860346..0000000 --- a/src/FixedWidthParser/Writers/FixedWidthWriter.cs +++ /dev/null @@ -1,74 +0,0 @@ -using System.Diagnostics.CodeAnalysis; - -namespace FixedWidthParser.Writers -{ - /// - /// Writes attribute-mapped models as fixed-width lines of bytes through the runtime - /// (reflection-built) formatter. Batches lines into one pooled buffer, so writing many records - /// allocates nothing per record. For source-generated models use . - /// Column positions are bytes. - /// - public sealed class FixedWidthWriter where TModel : new() - { - private readonly FixedWidthLayout _layout; - private readonly FixedWidthWriterOptions _options; - - /// Creates a writer; options use every default. - [RequiresUnreferencedCode("Attribute layouts reflect over TModel's public properties and fields, which trimming may remove.")] - [RequiresDynamicCode("Attribute layouts compile member accessors at runtime (Expression.Compile / MakeGenericType).")] - public FixedWidthWriter(FixedWidthWriterOptions? options = null) - { - _layout = FixedWidthLayout.FromAttributes(); - _layout.EnsureCanWrite(); - _options = FixedWidthWriterOptions.Validate(options); - } - - private LayoutRecordFormatter Formatter => new(_layout); - - /// - /// Formats into , without a line break. - /// Returns only when is shorter than the line. - /// - public bool TryFormat(in TModel model, Span destination, out int bytesWritten) - { - return Formatter.TryFormat(in model, destination, _options.Context, out bytesWritten); - } - - /// Writes one record and its line break. - public void Write(Stream stream, in TModel model) - { - ArgumentNullException.ThrowIfNull(stream); - RecordWriter.Write(stream, in model, Formatter, _options); - } - - /// Writes every record, each followed by a line break. - public void WriteMany(Stream stream, IEnumerable models) - { - ArgumentNullException.ThrowIfNull(stream); - ArgumentNullException.ThrowIfNull(models); - RecordWriter.WriteMany(stream, models, Formatter, _options); - } - - /// Writes every record, each followed by a line break, reading each one by reference. - public void WriteMany(Stream stream, ReadOnlySpan models) - { - ArgumentNullException.ThrowIfNull(stream); - RecordWriter.WriteMany(stream, models, Formatter, _options); - } - - /// Writes one record and its line break. - public Task WriteAsync(Stream stream, TModel model, CancellationToken cancellationToken = default) - { - ArgumentNullException.ThrowIfNull(stream); - return RecordWriter.WriteAsync(stream, model, Formatter, _options, cancellationToken); - } - - /// Writes every record, each followed by a line break. - public Task WriteManyAsync(Stream stream, IEnumerable models, CancellationToken cancellationToken = default) - { - ArgumentNullException.ThrowIfNull(stream); - ArgumentNullException.ThrowIfNull(models); - return RecordWriter.WriteManyAsync(stream, models, Formatter, _options, cancellationToken); - } - } -} diff --git a/tests/Benchmarks/Perf/AsyncReaderBenchmarks.cs b/tests/Benchmarks/Perf/AsyncReaderBenchmarks.cs index fdda07e..35ceee0 100644 --- a/tests/Benchmarks/Perf/AsyncReaderBenchmarks.cs +++ b/tests/Benchmarks/Perf/AsyncReaderBenchmarks.cs @@ -2,7 +2,6 @@ using BenchmarkDotNet.Attributes; using CommunityToolkit.HighPerformance.Buffers; using FixedWidthParser; -using FixedWidthParser.Parsers; using FixedWidthParser.Readers; namespace Benchmarks.Perf @@ -14,9 +13,8 @@ namespace Benchmarks.Perf [Config(typeof(RegressionConfig))] public class AsyncReaderBenchmarks { - private readonly FixedWidthParser _parser = new(); - private readonly FixedWidthReader _reader = new(); - private readonly FixedWidthReader _pooledReader = new(new FixedWidthReaderOptions { StringPool = new StringPool() }); + private static readonly FixedWidthLayout _layout = FixedWidthLayout.FromAttributes(); + private readonly FixedWidthReaderOptions _pooledReaderOptions = new FixedWidthReaderOptions { StringPool = new StringPool() }; private readonly FixedWidthReaderOptions _generatedPooled = new() { StringPool = new StringPool() }; private byte[] _bytes = []; @@ -37,7 +35,7 @@ public async Task Naive_ReadLineAsync() string? line; while ((line = await reader.ReadLineAsync().ConfigureAwait(false)) is not null) { - if (_parser.TryParse(Encoding.UTF8.GetBytes(line), default, out var model)) + if (FixedWidth.TryParse(Encoding.UTF8.GetBytes(line), _layout, default, out var model)) { sum += model.Age; } @@ -49,7 +47,7 @@ public async Task Naive_ReadLineAsync() public async Task Reader_ReadAsync() { int sum = 0; - await foreach (var model in _reader.ReadAsync(new MemoryStream(_bytes, writable: false)).ConfigureAwait(false)) + await foreach (var model in FixedWidth.ReadAsync(new MemoryStream(_bytes, writable: false), _layout).ConfigureAwait(false)) { sum += model.Age; } @@ -60,7 +58,7 @@ public async Task Reader_ReadAsync() public async Task Reader_ReadAsync_Pooled() { int sum = 0; - await foreach (var model in _pooledReader.ReadAsync(new MemoryStream(_bytes, writable: false)).ConfigureAwait(false)) + await foreach (var model in FixedWidth.ReadAsync(new MemoryStream(_bytes, writable: false), _layout, options: _pooledReaderOptions).ConfigureAwait(false)) { sum += model.Age; } diff --git a/tests/Benchmarks/Perf/ComparisonBenchmarks.cs b/tests/Benchmarks/Perf/ComparisonBenchmarks.cs index 96f3460..f33ec83 100644 --- a/tests/Benchmarks/Perf/ComparisonBenchmarks.cs +++ b/tests/Benchmarks/Perf/ComparisonBenchmarks.cs @@ -33,7 +33,7 @@ public sealed class FileHelpersRecord [Config(typeof(RegressionConfig))] public class ComparisonBenchmarks { - private readonly FixedWidthReader _reader = new(); + private static readonly FixedWidthLayout _layout = FixedWidthLayout.FromAttributes(); private readonly FileHelperEngine _engine = new(); private readonly FixedLengthSchema _schema = CreateSchema(); private readonly FixedLengthReaderOptions _options = CreateReader(); @@ -112,7 +112,7 @@ public int RecordParser_Read() public int FixedWidthParser_Read() { int sum = 0; - foreach (var model in _reader.Read(new MemoryStream(_bytes, writable: false))) + foreach (var model in FixedWidth.Read(new MemoryStream(_bytes, writable: false), _layout)) { sum += model.Age; } diff --git a/tests/Benchmarks/Perf/ComparisonWriterBenchmarks.cs b/tests/Benchmarks/Perf/ComparisonWriterBenchmarks.cs index b1de550..5905fe3 100644 --- a/tests/Benchmarks/Perf/ComparisonWriterBenchmarks.cs +++ b/tests/Benchmarks/Perf/ComparisonWriterBenchmarks.cs @@ -1,8 +1,8 @@ +using FixedWidthParser; using BenchmarkDotNet.Attributes; using FileHelpers; using FlatFiles; using FlatFiles.TypeMapping; -using FixedWidthParser.Writers; using RecordParser.Parsers; using RecordParser.Builders.Writer; using RecordParser.Extensions; @@ -25,7 +25,7 @@ public sealed class FlatFilesWriteModel public class ComparisonWriterBenchmarks : IDisposable { private static readonly FixedLengthOptions FlatFilesOptions = new(); - private readonly FixedWidthWriter _writer = new(); + private static readonly FixedWidthLayout _layout = FixedWidthLayout.FromAttributes(); private readonly FileHelperEngine _engine = new(); private readonly IFixedLengthWriter _fixedWidthWriter = CreateWriter(); private StreamWriter _sink = null!; @@ -100,7 +100,7 @@ public void RecordParser_Write() [Benchmark] public void FixedWidthParser_Write() { - _writer.WriteMany(Stream.Null, _models.AsSpan()); + FixedWidth.WriteMany(Stream.Null, _models.AsSpan(), _layout); } public virtual void Dispose() diff --git a/tests/Benchmarks/Perf/ParserBenchmarks.cs b/tests/Benchmarks/Perf/ParserBenchmarks.cs index c80a285..e967fbe 100644 --- a/tests/Benchmarks/Perf/ParserBenchmarks.cs +++ b/tests/Benchmarks/Perf/ParserBenchmarks.cs @@ -2,7 +2,6 @@ using BenchmarkDotNet.Attributes; using CommunityToolkit.HighPerformance.Buffers; using FixedWidthParser; -using FixedWidthParser.Parsers; namespace Benchmarks.Perf { @@ -13,7 +12,7 @@ namespace Benchmarks.Perf [Config(typeof(RegressionConfig))] public class ParserBenchmarks { - private readonly FixedWidthParser _parser = new(); + private static readonly FixedWidthLayout _layout = FixedWidthLayout.FromAttributes(); private FixedWidthContext _pooled; private byte[] _line = []; @@ -32,14 +31,14 @@ public void Setup() [Benchmark(Baseline = true)] public SampleModel Parse_NoPool() { - _parser.TryParse(_line, default, out var model); + FixedWidth.TryParse(_line, _layout, default, out var model); return model; } [Benchmark] public SampleModel Parse_WithStringPool() { - _parser.TryParse(_line, _pooled, out var model); + FixedWidth.TryParse(_line, _layout, _pooled, out var model); return model; } } diff --git a/tests/Benchmarks/Perf/ReaderBenchmarks.cs b/tests/Benchmarks/Perf/ReaderBenchmarks.cs index dca3b25..d0cd6e7 100644 --- a/tests/Benchmarks/Perf/ReaderBenchmarks.cs +++ b/tests/Benchmarks/Perf/ReaderBenchmarks.cs @@ -2,7 +2,6 @@ using BenchmarkDotNet.Attributes; using CommunityToolkit.HighPerformance.Buffers; using FixedWidthParser; -using FixedWidthParser.Parsers; using FixedWidthParser.Readers; namespace Benchmarks.Perf @@ -15,9 +14,8 @@ namespace Benchmarks.Perf [Config(typeof(RegressionConfig))] public class ReaderBenchmarks { - private readonly FixedWidthParser _parser = new(); - private readonly FixedWidthReader _reader = new(); - private readonly FixedWidthReader _pooledReader = new(new FixedWidthReaderOptions { StringPool = new StringPool() }); + private static readonly FixedWidthLayout _layout = FixedWidthLayout.FromAttributes(); + private readonly FixedWidthReaderOptions _pooledReaderOptions = new FixedWidthReaderOptions { StringPool = new StringPool() }; private readonly FixedWidthReaderOptions _generatedPooled = new() { StringPool = new StringPool() }; private byte[] _bytes = []; @@ -38,7 +36,7 @@ public int Naive_ReadLine() string? line; while ((line = reader.ReadLine()) is not null) { - if (_parser.TryParse(Encoding.UTF8.GetBytes(line), default, out var model)) + if (FixedWidth.TryParse(Encoding.UTF8.GetBytes(line), _layout, default, out var model)) { sum += model.Age; } @@ -50,7 +48,7 @@ public int Naive_ReadLine() public int Reader_Read() { int sum = 0; - foreach (var model in _reader.Read(new MemoryStream(_bytes, writable: false))) + foreach (var model in FixedWidth.Read(new MemoryStream(_bytes, writable: false), _layout)) { sum += model.Age; } @@ -61,7 +59,7 @@ public int Reader_Read() public int Reader_Read_Pooled() { int sum = 0; - foreach (var model in _pooledReader.Read(new MemoryStream(_bytes, writable: false))) + foreach (var model in FixedWidth.Read(new MemoryStream(_bytes, writable: false), _layout, options: _pooledReaderOptions)) { sum += model.Age; } diff --git a/tests/Benchmarks/Perf/ReaderPoolingBenchmarks.cs b/tests/Benchmarks/Perf/ReaderPoolingBenchmarks.cs index d3086c9..76ecd3b 100644 --- a/tests/Benchmarks/Perf/ReaderPoolingBenchmarks.cs +++ b/tests/Benchmarks/Perf/ReaderPoolingBenchmarks.cs @@ -2,7 +2,6 @@ using CommunityToolkit.HighPerformance.Buffers; using FixedWidthParser; using FixedWidthParser.Readers; -using FixedWidthParser.Writers; namespace Benchmarks.Perf { @@ -14,9 +13,9 @@ namespace Benchmarks.Perf [Config(typeof(RegressionConfig))] public class ReaderPoolingBenchmarks { - private readonly FixedWidthReader _reader = new(); - private readonly FixedWidthReader _pooledReader = new(new FixedWidthReaderOptions { StringPool = new StringPool() }); - private readonly FixedWidthReader _numericReader = new(); + private static readonly FixedWidthLayout _layout = FixedWidthLayout.FromAttributes(); + private readonly FixedWidthReaderOptions _pooledReaderOptions = new FixedWidthReaderOptions { StringPool = new StringPool() }; + private static readonly FixedWidthLayout _numericLayout = FixedWidthLayout.FromAttributes(); private byte[] _bytes = []; private byte[] _numericBytes = []; @@ -29,7 +28,7 @@ public void Setup() _bytes = SampleData.Bytes(Count); var numeric = SampleData.Models(Count).Select(m => new NumericModel { Age = m.Age, Salary = m.Salary }).ToArray(); using var ms = new MemoryStream(); - new FixedWidthWriter().WriteMany(ms, numeric.AsSpan()); + FixedWidth.WriteMany(ms, numeric.AsSpan(), FixedWidthLayout.FromAttributes()); _numericBytes = ms.ToArray(); } @@ -38,7 +37,7 @@ public void Setup() public int WithString_NoPool() { int sum = 0; - foreach (var model in _reader.Read(new MemoryStream(_bytes, writable: false))) + foreach (var model in FixedWidth.Read(new MemoryStream(_bytes, writable: false), _layout)) { sum += model.Age; } @@ -50,7 +49,7 @@ public int WithString_NoPool() public int WithString_Pooled() { int sum = 0; - foreach (var model in _pooledReader.Read(new MemoryStream(_bytes, writable: false))) + foreach (var model in FixedWidth.Read(new MemoryStream(_bytes, writable: false), _layout, options: _pooledReaderOptions)) { sum += model.Age; } @@ -62,7 +61,7 @@ public int WithString_Pooled() public int NoString() { int sum = 0; - foreach (var model in _numericReader.Read(new MemoryStream(_numericBytes, writable: false))) + foreach (var model in FixedWidth.Read(new MemoryStream(_numericBytes, writable: false), _numericLayout)) { sum += model.Age; } diff --git a/tests/Benchmarks/Perf/ReaderStartupBenchmarks.cs b/tests/Benchmarks/Perf/ReaderStartupBenchmarks.cs index a71f18d..48c3bae 100644 --- a/tests/Benchmarks/Perf/ReaderStartupBenchmarks.cs +++ b/tests/Benchmarks/Perf/ReaderStartupBenchmarks.cs @@ -18,7 +18,7 @@ public class ReaderStartupBenchmarks public int ReflectionReader_ConstructAndReadOne() { int sum = 0; - foreach (var model in new FixedWidthReader().Read(new MemoryStream(Text, writable: false))) + foreach (var model in FixedWidth.Read(new MemoryStream(Text, writable: false), FixedWidthLayout.FromAttributes())) { sum += model.Age; } diff --git a/tests/Benchmarks/Perf/SourceGenParserBenchmarks.cs b/tests/Benchmarks/Perf/SourceGenParserBenchmarks.cs index dfbe5ae..0918408 100644 --- a/tests/Benchmarks/Perf/SourceGenParserBenchmarks.cs +++ b/tests/Benchmarks/Perf/SourceGenParserBenchmarks.cs @@ -2,7 +2,6 @@ using BenchmarkDotNet.Attributes; using FixedWidthParser; using FixedWidthParser.Attributes; -using FixedWidthParser.Parsers; namespace Benchmarks.Perf { @@ -21,7 +20,7 @@ namespace Benchmarks.Perf [Config(typeof(RegressionConfig))] public class SourceGenParserBenchmarks { - private readonly FixedWidthParser _reflection = new(); + private static readonly FixedWidthLayout _layout = FixedWidthLayout.FromAttributes(); private byte[] _line = []; [Params("John Doe 30 6000000.00")] @@ -36,7 +35,7 @@ public void Setup() [Benchmark(Baseline = true)] public int Reflection() { - _reflection.TryParse(_line, default, out var model); + FixedWidth.TryParse(_line, _layout, default, out var model); return model.Age; } diff --git a/tests/Benchmarks/Perf/WriterBenchmarks.cs b/tests/Benchmarks/Perf/WriterBenchmarks.cs index bbd38ff..a84e834 100644 --- a/tests/Benchmarks/Perf/WriterBenchmarks.cs +++ b/tests/Benchmarks/Perf/WriterBenchmarks.cs @@ -1,5 +1,5 @@ +using FixedWidthParser; using BenchmarkDotNet.Attributes; -using FixedWidthParser.Writers; namespace Benchmarks.Perf { @@ -10,7 +10,7 @@ namespace Benchmarks.Perf [Config(typeof(RegressionConfig))] public class WriterBenchmarks { - private readonly FixedWidthWriter _writer = new(); + private static readonly FixedWidthLayout _layout = FixedWidthLayout.FromAttributes(); private SampleModel[] _models = []; [Params(1, 100, 1000)] @@ -25,21 +25,21 @@ public void Setup() [Benchmark(Baseline = true)] public void WriteMany_Enumerable() { - _writer.WriteMany(Stream.Null, (IEnumerable)_models); + FixedWidth.WriteMany(Stream.Null, (IEnumerable)_models, _layout); } /// ReadOnlySpan: no enumerator, each record read by reference. Should be zero-alloc. [Benchmark] public void WriteMany_Span() { - _writer.WriteMany(Stream.Null, _models.AsSpan()); + FixedWidth.WriteMany(Stream.Null, _models.AsSpan(), _layout); } /// The async overload: the difference from WriteMany_Enumerable is the async state machine. [Benchmark] public Task WriteManyAsync() { - return _writer.WriteManyAsync(Stream.Null, _models); + return FixedWidth.WriteManyAsync(Stream.Null, _models, _layout); } } } diff --git a/tests/Benchmarks/SampleData.cs b/tests/Benchmarks/SampleData.cs index 983956b..61a1fc2 100644 --- a/tests/Benchmarks/SampleData.cs +++ b/tests/Benchmarks/SampleData.cs @@ -1,4 +1,4 @@ -using FixedWidthParser.Writers; +using FixedWidthParser; namespace Benchmarks { @@ -18,7 +18,7 @@ public static SampleModel[] Models(int count) public static byte[] Bytes(int count) { using var ms = new MemoryStream(); - new FixedWidthWriter().WriteMany(ms, Models(count).AsSpan()); + FixedWidth.WriteMany(ms, Models(count).AsSpan(), FixedWidthLayout.FromAttributes()); return ms.ToArray(); } } diff --git a/tests/FixedWidthParser.Tests/AllocationGuardTests.cs b/tests/FixedWidthParser.Tests/AllocationGuardTests.cs index ae7e200..3c37847 100644 --- a/tests/FixedWidthParser.Tests/AllocationGuardTests.cs +++ b/tests/FixedWidthParser.Tests/AllocationGuardTests.cs @@ -1,6 +1,5 @@ using System.Text; using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Readers; namespace FixedWidthParser.Tests { @@ -23,24 +22,25 @@ public class AllocationGuardTests [Fact] public void Read_AllocatesEnumerableOnly_NoClosure() { - var reader = new FixedWidthReader(); + var layout = FixedWidthLayout.FromAttributes(); using var stream = new MemoryStream(); - _ = reader.Read(stream); + _ = FixedWidth.Read(stream, layout); _ = FixedWidth.Read(stream); - Assert.True(PerCall(() => reader.Read(stream)) <= 72, "Reflection Read allocated more than the enumerable."); + Assert.True(PerCall(() => FixedWidth.Read(stream, layout)) <= 72, "Reflection Read allocated more than the enumerable."); Assert.True(PerCall(() => FixedWidth.Read(stream)) <= 72, "Generated Read allocated more than the enumerable."); } [Fact] public void ReflectionPooledReader_AllocatesNothingPerLine() { - var reader = new FixedWidthReader(new FixedWidthReaderOptions { StringPool = new StringPool() }); + var options = new FixedWidthReaderOptions { StringPool = new StringPool() }; + var layout = FixedWidthLayout.FromAttributes(); byte[] small = Repeat(Small), large = Repeat(Large); - SumReflection(reader, small); - SumReflection(reader, large); + SumReflection(layout, options, small); + SumReflection(layout, options, large); - double perLine = MarginalPerLine(() => SumReflection(reader, small), () => SumReflection(reader, large)); + double perLine = MarginalPerLine(() => SumReflection(layout, options, small), () => SumReflection(layout, options, large)); Assert.True(perLine <= 1, $"Reflection pooled reader regressed to {perLine} B/line (expected ~0)."); } @@ -79,10 +79,10 @@ private static byte[] Repeat(int lines) return Encoding.UTF8.GetBytes(sb.ToString()); } - private static int SumReflection(FixedWidthReader reader, byte[] bytes) + private static int SumReflection(FixedWidthLayout layout, FixedWidthReaderOptions options, byte[] bytes) { int total = 0; - foreach (var m in reader.Read(new MemoryStream(bytes, writable: false))) + foreach (var m in FixedWidth.Read(new MemoryStream(bytes, writable: false), layout, options: options)) { total += m.Code.Length; } diff --git a/tests/FixedWidthParser.Tests/ColumnValidationTests.cs b/tests/FixedWidthParser.Tests/ColumnValidationTests.cs index 5d2d5d8..1620f2a 100644 --- a/tests/FixedWidthParser.Tests/ColumnValidationTests.cs +++ b/tests/FixedWidthParser.Tests/ColumnValidationTests.cs @@ -1,5 +1,4 @@ -using FixedWidthParser.Parsers; -using FixedWidthParser.Writers; +using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { @@ -12,63 +11,63 @@ public class ColumnValidationTests [Fact] public void Parser_OverlappingColumns_ThrowsOnConstruction() { - var ex = Assert.Throws(() => new FixedWidthParser()); + var ex = Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); Assert.Contains("Overlapping", ex.Message, StringComparison.Ordinal); } [Fact] public void Writer_OverlappingColumns_ThrowsOnConstruction() { - var ex = Assert.Throws(() => new FixedWidthWriter()); + var ex = Assert.Throws(() => FixedWidth.WriteMany(Stream.Null, Array.Empty(), FixedWidthLayout.FromAttributes())); Assert.Contains("Overlapping", ex.Message, StringComparison.Ordinal); } [Fact] public void Parser_NegativeStart_Throws() { - var ex = Assert.Throws(() => new FixedWidthParser()); + var ex = Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); Assert.Contains("Start", ex.Message, StringComparison.Ordinal); } [Fact] public void Writer_ZeroLength_Throws() { - var ex = Assert.Throws(() => new FixedWidthWriter()); + var ex = Assert.Throws(() => FixedWidth.WriteMany(Stream.Null, Array.Empty(), FixedWidthLayout.FromAttributes())); Assert.Contains("Length", ex.Message, StringComparison.Ordinal); } [Fact] public void Parser_NoColumns_ThrowsOnConstruction() { - var ex = Assert.Throws(() => new FixedWidthParser()); + var ex = Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); Assert.Contains("no [FixedColumn]", ex.Message, StringComparison.Ordinal); } [Fact] public void Parser_NonAsciiTrimChar_ThrowsOnConstruction() { - var ex = Assert.Throws(() => new FixedWidthParser()); + var ex = Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); Assert.Contains("TrimChar", ex.Message, StringComparison.Ordinal); } [Fact] public void Writer_NonAsciiPadding_ThrowsOnConstruction() { - var ex = Assert.Throws(() => new FixedWidthWriter()); + var ex = Assert.Throws(() => FixedWidth.WriteMany(Stream.Null, Array.Empty(), FixedWidthLayout.FromAttributes())); Assert.Contains("Padding", ex.Message, StringComparison.Ordinal); } [Fact] public void Parser_UnsupportedColumnType_ThrowsOnConstruction() { - var ex = Assert.Throws(() => new FixedWidthParser()); + var ex = Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); Assert.Contains("Converter", ex.Message, StringComparison.Ordinal); } [Fact] public void Writer_NoColumns_ThrowsOnConstruction() { - var ex = Assert.Throws(() => new FixedWidthWriter()); + var ex = Assert.Throws(() => FixedWidth.WriteMany(Stream.Null, Array.Empty(), FixedWidthLayout.FromAttributes())); Assert.Contains("no [FixedColumn]", ex.Message, StringComparison.Ordinal); } @@ -76,8 +75,8 @@ public void Writer_NoColumns_ThrowsOnConstruction() public void AdjacentColumns_AreValid() { // Touching columns (end of one == start of the next) do not overlap. - var parser = new FixedWidthParser(); - var writer = new FixedWidthWriter(); + var parser = FixedWidthLayout.FromAttributes(); + var writer = FixedWidthLayout.FromAttributes(); Assert.NotNull(parser); Assert.NotNull(writer); diff --git a/tests/FixedWidthParser.Tests/ConverterAttributeTests.cs b/tests/FixedWidthParser.Tests/ConverterAttributeTests.cs index 4352e4e..288be68 100644 --- a/tests/FixedWidthParser.Tests/ConverterAttributeTests.cs +++ b/tests/FixedWidthParser.Tests/ConverterAttributeTests.cs @@ -1,5 +1,3 @@ -using FixedWidthParser.Parsers; -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -16,16 +14,16 @@ public void Reflection_RoundTrips() { var original = new CentsConverterModel { Amount = new CentsValue(12345) }; - string line = WriteOne(new FixedWidthWriter(), original); + string line = WriteOne(FixedWidthLayout.FromAttributes(), original); - Assert.True(new FixedWidthParser().TryParse(Bytes(line), Ctx, out var parsed)); + Assert.True(FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var parsed)); Assert.Equal(original.Amount, parsed.Amount); } [Fact] public void Generated_MatchesReflection() { - bool okR = new FixedWidthParser().TryParse("12345 "u8, Ctx, out var r); + bool okR = FixedWidth.TryParse("12345 "u8, FixedWidthLayout.FromAttributes(), Ctx, out var r); bool okG = FixedWidth.TryParse("12345 "u8, Ctx, out var g); Assert.True(okR); @@ -36,20 +34,20 @@ public void Generated_MatchesReflection() [Fact] public void Converter_RejectingInput_ReturnsFalse() { - Assert.False(new FixedWidthParser().TryParse("notanum "u8, Ctx, out _)); + Assert.False(FixedWidth.TryParse("notanum "u8, FixedWidthLayout.FromAttributes(), Ctx, out _)); Assert.False(FixedWidth.TryParse("notanum "u8, Ctx, out _)); } [Fact] public void MismatchedConverter_ParserConstruction_Throws() { - Assert.Throws(() => new FixedWidthParser()); + Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); } [Fact] public void MismatchedConverter_WriterConstruction_Throws() { - Assert.Throws(() => new FixedWidthWriter()); + Assert.Throws(() => FixedWidth.WriteMany(Stream.Null, Array.Empty(), FixedWidthLayout.FromAttributes())); } } } diff --git a/tests/FixedWidthParser.Tests/CorrectnessTests.cs b/tests/FixedWidthParser.Tests/CorrectnessTests.cs index b33b0c9..e06ad70 100644 --- a/tests/FixedWidthParser.Tests/CorrectnessTests.cs +++ b/tests/FixedWidthParser.Tests/CorrectnessTests.cs @@ -1,5 +1,3 @@ -using FixedWidthParser.Parsers; -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -13,11 +11,11 @@ public void DateTimeExact_RoundTrip_BothPaths() var dateOnly = new DateOnly(2026, 7, 16); const string input = "2026071620260716"; - var parser = new FixedWidthParser(); - Assert.True(parser.TryParse(Bytes(input), Ctx, out var reflected)); + var layout = FixedWidthLayout.FromAttributes(); + Assert.True(FixedWidth.TryParse(Bytes(input), layout, Ctx, out var reflected)); Assert.Equal(date, reflected.Date); Assert.Equal(dateOnly, reflected.DateOnlyVal); - Assert.Equal(input, WriteOne(new FixedWidthWriter(), reflected)); + Assert.Equal(input, WriteOne(FixedWidthLayout.FromAttributes(), reflected)); Assert.True(FixedWidth.TryParse(Bytes(input), Ctx, out var generated)); Assert.Equal(date, generated.Date); @@ -28,8 +26,8 @@ public void DateTimeExact_RoundTrip_BothPaths() [Fact] public void ZeroPadded_Leading_TrimMode() { - var parser = new FixedWidthParser(); - Assert.True(parser.TryParse("0004203.14"u8, Ctx, out var reflected)); + var layout = FixedWidthLayout.FromAttributes(); + Assert.True(FixedWidth.TryParse("0004203.14"u8, layout, Ctx, out var reflected)); Assert.Equal(42, reflected.Value); Assert.Equal(3.14, reflected.DoubleValue); @@ -41,7 +39,7 @@ public void ZeroPadded_Leading_TrimMode() [Fact] public void ZeroPadded_Both_TrimMode() { - Assert.True(new FixedWidthParser().TryParse("00420"u8, Ctx, out var reflected)); + Assert.True(FixedWidth.TryParse("00420"u8, FixedWidthLayout.FromAttributes(), Ctx, out var reflected)); Assert.Equal(42, reflected.Value); Assert.True(FixedWidth.TryParse("00420"u8, Ctx, out var generated)); @@ -54,14 +52,14 @@ public void Nullable_BlankDetection_WithCustomTrimChar() const string spaces = " "; const string padded = "00000*****"; - var parser = new FixedWidthParser(); - Assert.True(parser.TryParse(Bytes(spaces), Ctx, out var fromSpaces)); + var layout = FixedWidthLayout.FromAttributes(); + Assert.True(FixedWidth.TryParse(Bytes(spaces), layout, Ctx, out var fromSpaces)); Assert.Null(fromSpaces.Value); Assert.Null(fromSpaces.DoubleValue); - Assert.True(parser.TryParse(Bytes(padded), Ctx, out var fromPadding)); + Assert.True(FixedWidth.TryParse(Bytes(padded), layout, Ctx, out var fromPadding)); Assert.Null(fromPadding.Value); Assert.Null(fromPadding.DoubleValue); - Assert.Equal(padded, WriteOne(new FixedWidthWriter(), fromPadding)); + Assert.Equal(padded, WriteOne(FixedWidthLayout.FromAttributes(), fromPadding)); Assert.True(FixedWidth.TryParse(Bytes(spaces), Ctx, out var generatedSpaces)); Assert.Null(generatedSpaces.Value); @@ -74,16 +72,16 @@ public void Nullable_BlankDetection_WithCustomTrimChar() [Fact] public void SignAware_ZeroPadding() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - Assert.Equal("-00042", WriteOne(writer, new SignPaddedModel { Value = -42 })); - Assert.Equal("000042", WriteOne(writer, new SignPaddedModel { Value = 42 })); + Assert.Equal("-00042", WriteOne(layout, new SignPaddedModel { Value = -42 })); + Assert.Equal("000042", WriteOne(layout, new SignPaddedModel { Value = 42 })); } [Fact] public void Writer_RejectsTruncateOnNonString() { - Assert.Throws(() => new FixedWidthWriter()); + Assert.Throws(() => FixedWidth.WriteMany(Stream.Null, Array.Empty(), FixedWidthLayout.FromAttributes())); } } } diff --git a/tests/FixedWidthParser.Tests/CoverageTests.cs b/tests/FixedWidthParser.Tests/CoverageTests.cs index 82b93a7..c22acb8 100644 --- a/tests/FixedWidthParser.Tests/CoverageTests.cs +++ b/tests/FixedWidthParser.Tests/CoverageTests.cs @@ -2,7 +2,6 @@ using System.Text; using FixedWidthParser.Attributes; using FixedWidthParser.Formatters; -using FixedWidthParser.Parsers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -20,7 +19,7 @@ public void TimeExact_RoundTrip_BothPaths() var timestamp = new DateTimeOffset(2026, 7, 16, 13, 45, 30, TimeSpan.FromHours(3)); byte[] input = Bytes("13453020260716134530+03:00"); - Assert.True(new FixedWidthParser().TryParse(input, Ctx, out var reflected)); + Assert.True(FixedWidth.TryParse(input, FixedWidthLayout.FromAttributes(), Ctx, out var reflected)); Assert.Equal(time, reflected.Time); Assert.Equal(timestamp, reflected.Timestamp); @@ -34,7 +33,7 @@ public void TimeExact_InvalidField_ReturnsFalse() { byte[] bad = Bytes("xxxxxx20260716134530+03:00"); - Assert.False(new FixedWidthParser().TryParse(bad, Ctx, out _)); + Assert.False(FixedWidth.TryParse(bad, FixedWidthLayout.FromAttributes(), Ctx, out _)); Assert.False(FixedWidth.TryParse(bad, Ctx, out _)); } @@ -44,7 +43,7 @@ public void WideNumericField_CommaCulture_DecodesThroughArrayPool() // Wider than the 128-char stack buffer, with a ',' separator so the BCL fallback runs. string field = ("0," + new string('1', 150)).PadRight(200); - bool ok = new FixedWidthParser().TryParse(Bytes(field + field), CtxWith(CultureInfo.GetCultureInfo("de-DE")), out var m); + bool ok = FixedWidth.TryParse(Bytes(field + field), FixedWidthLayout.FromAttributes(), CtxWith(CultureInfo.GetCultureInfo("de-DE")), out var m); Assert.True(ok); Assert.InRange(m.D, 0.11, 0.12); diff --git a/tests/FixedWidthParser.Tests/CultureTests.cs b/tests/FixedWidthParser.Tests/CultureTests.cs index f6feded..82d9a31 100644 --- a/tests/FixedWidthParser.Tests/CultureTests.cs +++ b/tests/FixedWidthParser.Tests/CultureTests.cs @@ -1,6 +1,4 @@ using System.Globalization; -using FixedWidthParser.Parsers; -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -17,35 +15,35 @@ public class CultureTests [Fact] public void Parse_DecimalColumn_HonorsCommaCulture() { - Assert.True(new FixedWidthParser().TryParse("1234,56 "u8, German, out var model)); + Assert.True(FixedWidth.TryParse("1234,56 "u8, FixedWidthLayout.FromAttributes(), German, out var model)); Assert.Equal(1234.56m, model.Amount); } [Fact] public void Parse_DecimalColumn_DefaultContext_IsInvariant() { - Assert.True(new FixedWidthParser().TryParse("1234.56 "u8, Ctx, out var model)); + Assert.True(FixedWidth.TryParse("1234.56 "u8, FixedWidthLayout.FromAttributes(), Ctx, out var model)); Assert.Equal(1234.56m, model.Amount); } [Fact] public void Parse_DoubleColumn_HonorsCommaCulture() { - Assert.True(new FixedWidthParser().TryParse("Bob 30 1234,50 "u8, German, out var model)); + Assert.True(FixedWidth.TryParse("Bob 30 1234,50 "u8, FixedWidthLayout.FromAttributes(), German, out var model)); Assert.Equal(1234.50, model.Salary, 2); } [Fact] public void Parse_DoubleColumn_DefaultContext_UsesDot() { - Assert.True(new FixedWidthParser().TryParse("Bob 30 1234.50 "u8, Ctx, out var model)); + Assert.True(FixedWidth.TryParse("Bob 30 1234.50 "u8, FixedWidthLayout.FromAttributes(), Ctx, out var model)); Assert.Equal(1234.50, model.Salary, 2); } [Fact] public void Parse_FloatColumn_HonorsCommaCulture() { - Assert.True(new FixedWidthParser().TryParse("3,14 "u8, German, out var model)); + Assert.True(FixedWidth.TryParse("3,14 "u8, FixedWidthLayout.FromAttributes(), German, out var model)); Assert.Equal(3.14f, model.Value, 2); } @@ -54,7 +52,7 @@ public void Parse_FloatColumn_HonorsCommaCulture() [Fact] public void Parse_DoubleColumn_DotCultureThousandsSeparator_ParsesFullValue() { - Assert.True(new FixedWidthParser().TryParse("Bob 30 1,234.50 "u8, Ctx, out var model)); + Assert.True(FixedWidth.TryParse("Bob 30 1,234.50 "u8, FixedWidthLayout.FromAttributes(), Ctx, out var model)); Assert.Equal(1234.50, model.Salary, 2); } @@ -62,20 +60,20 @@ public void Parse_DoubleColumn_DotCultureThousandsSeparator_ParsesFullValue() [Fact] public void Parse_DoubleColumn_DeDeThousandsSeparator_ParsesFullValue() { - Assert.True(new FixedWidthParser().TryParse("Bob 30 1.234,50 "u8, German, out var model)); + Assert.True(FixedWidth.TryParse("Bob 30 1.234,50 "u8, FixedWidthLayout.FromAttributes(), German, out var model)); Assert.Equal(1234.50, model.Salary, 2); } [Fact] public void Parse_DoubleColumn_TrailingGarbageAfterNumber_Fails() { - Assert.False(new FixedWidthParser().TryParse("Bob 30 12x "u8, Ctx, out _)); + Assert.False(FixedWidth.TryParse("Bob 30 12x "u8, FixedWidthLayout.FromAttributes(), Ctx, out _)); } [Fact] public void Parse_FloatColumn_TrailingGarbageAfterNumber_Fails() { - Assert.False(new FixedWidthParser().TryParse("12x "u8, Ctx, out _)); + Assert.False(FixedWidth.TryParse("12x "u8, FixedWidthLayout.FromAttributes(), Ctx, out _)); } // csFastFloat's characters_consumed does not consistently include skipped leading whitespace, @@ -83,28 +81,28 @@ public void Parse_FloatColumn_TrailingGarbageAfterNumber_Fails() [Fact] public void Parse_DoubleColumn_RightAligned_DotCulture_Parses() { - Assert.True(new FixedWidthParser().TryParse("Bob 30 123.45"u8, Ctx, out var model)); + Assert.True(FixedWidth.TryParse("Bob 30 123.45"u8, FixedWidthLayout.FromAttributes(), Ctx, out var model)); Assert.Equal(123.45, model.Salary, 2); } [Fact] public void Parse_FloatColumn_RightAligned_DotCulture_Parses() { - Assert.True(new FixedWidthParser().TryParse(" 3.14"u8, Ctx, out var model)); + Assert.True(FixedWidth.TryParse(" 3.14"u8, FixedWidthLayout.FromAttributes(), Ctx, out var model)); Assert.Equal(3.14f, model.Value, 2); } [Fact] public void Parse_DoubleColumn_RightAligned_CommaCulture_Parses() { - Assert.True(new FixedWidthParser().TryParse("Bob 30 123,45"u8, German, out var model)); + Assert.True(FixedWidth.TryParse("Bob 30 123,45"u8, FixedWidthLayout.FromAttributes(), German, out var model)); Assert.Equal(123.45, model.Salary, 2); } [Fact] public void Parse_FloatColumn_RightAligned_CommaCulture_Parses() { - Assert.True(new FixedWidthParser().TryParse(" 3,14"u8, German, out var model)); + Assert.True(FixedWidth.TryParse(" 3,14"u8, FixedWidthLayout.FromAttributes(), German, out var model)); Assert.Equal(3.14f, model.Value, 2); } @@ -116,7 +114,7 @@ public void Parse_DoubleColumn_NonAsciiSeparatorCulture_Parses() var culture = (CultureInfo)CultureInfo.InvariantCulture.Clone(); culture.NumberFormat.NumberDecimalSeparator = "٫"; - Assert.True(new FixedWidthParser().TryParse(Bytes("Bob 30 1234٫50 "), CtxWith(culture), out var model)); + Assert.True(FixedWidth.TryParse(Bytes("Bob 30 1234٫50 "), FixedWidthLayout.FromAttributes(), CtxWith(culture), out var model)); Assert.Equal(1234.50, model.Salary, 2); } @@ -124,14 +122,14 @@ public void Parse_DoubleColumn_NonAsciiSeparatorCulture_Parses() [Fact] public void Parse_DoubleColumn_AlternatingProviders_BothStayCorrect() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); for (int i = 0; i < 5; i++) { - Assert.True(parser.TryParse("Bob 30 1234.50 "u8, Ctx, out var invariant)); + Assert.True(FixedWidth.TryParse("Bob 30 1234.50 "u8, layout, Ctx, out var invariant)); Assert.Equal(1234.50, invariant.Salary, 2); - Assert.True(parser.TryParse("Bob 30 1234,50 "u8, German, out var german)); + Assert.True(FixedWidth.TryParse("Bob 30 1234,50 "u8, layout, German, out var german)); Assert.Equal(1234.50, german.Salary, 2); } } @@ -139,9 +137,10 @@ public void Parse_DoubleColumn_AlternatingProviders_BothStayCorrect() [Fact] public void Write_DoubleColumn_UsesCultureDecimalSeparator() { - var writer = new FixedWidthWriter(new FixedWidthWriterOptions { FormatProvider = DeDe }); + var writerOptions = new FixedWidthWriterOptions { FormatProvider = DeDe }; + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }); + string line = WriteOne(layout, new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }, writerOptions); Assert.Equal("1234,5", line[15..21]); } @@ -151,21 +150,22 @@ public void RoundTrip_DoubleColumn_DefaultOptions_PreservesValue() { var original = new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }; - string line = WriteOne(new FixedWidthWriter(), original); + string line = WriteOne(FixedWidthLayout.FromAttributes(), original); - Assert.True(new FixedWidthParser().TryParse(Bytes(line), Ctx, out var parsed)); + Assert.True(FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var parsed)); Assert.Equal(original.Salary, parsed.Salary, 2); } [Fact] public void RoundTrip_DoubleColumn_UnderCommaCulture_PreservesValue() { - var writer = new FixedWidthWriter(new FixedWidthWriterOptions { FormatProvider = DeDe }); + var writerOptions = new FixedWidthWriterOptions { FormatProvider = DeDe }; + var layout = FixedWidthLayout.FromAttributes(); var original = new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }; - string line = WriteOne(writer, original); + string line = WriteOne(layout, original, writerOptions); - Assert.True(new FixedWidthParser().TryParse(Bytes(line), German, out var parsed)); + Assert.True(FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), German, out var parsed)); Assert.Equal(original.Salary, parsed.Salary, 2); } } diff --git a/tests/FixedWidthParser.Tests/GeneratedModels.cs b/tests/FixedWidthParser.Tests/GeneratedModels.cs index a9fd364..d5d4432 100644 --- a/tests/FixedWidthParser.Tests/GeneratedModels.cs +++ b/tests/FixedWidthParser.Tests/GeneratedModels.cs @@ -4,7 +4,7 @@ namespace FixedWidthParser.Tests { // Models that opt into the source-generated parser and writer. Each mirrors a reflection-based // counterpart in TestModels.cs so the parity tests can compare the generated TryParse/TryFormat - // against the runtime FixedWidthParser/FixedWidthWriter over the same lines. + // against the same type's attribute layout (FixedWidthLayout.FromAttributes) over the same lines. /// Property-based model (string + int + double) — mirrors . public readonly partial record struct GenPersonModel : IFixedWidthModel diff --git a/tests/FixedWidthParser.Tests/GeneratedReaderTests.cs b/tests/FixedWidthParser.Tests/GeneratedReaderTests.cs index 7ae9549..7523550 100644 --- a/tests/FixedWidthParser.Tests/GeneratedReaderTests.cs +++ b/tests/FixedWidthParser.Tests/GeneratedReaderTests.cs @@ -7,7 +7,7 @@ namespace FixedWidthParser.Tests { /// /// The read methods for generated models, sync and async, and their - /// parity with the reflection . + /// parity with reading through . /// public class GeneratedReaderTests { @@ -79,7 +79,7 @@ public void Read_LineLongerThanBuffer_MatchesReflectionReader() { var options = new FixedWidthReaderOptions { BufferSize = 4 }; - var expected = new FixedWidthReader(options).Read(StreamOf(TwoPeople)).ToList(); + var expected = FixedWidth.Read(StreamOf(TwoPeople), FixedWidthLayout.FromAttributes(), options: options).ToList(); var actual = FixedWidth.Read(StreamOf(TwoPeople), options: options).ToList(); Assert.Equal(expected[0].Name, actual[0].Name); @@ -92,7 +92,7 @@ public void Read_InvalidLine_ThrowsSameErrorAsReflectionReader() { const string text = "John Doe 30 60000.00 \nJane XX 55000.00 "; - var expected = Assert.Throws(() => new FixedWidthReader().Read(StreamOf(text)).ToList()); + var expected = Assert.Throws(() => FixedWidth.Read(StreamOf(text), FixedWidthLayout.FromAttributes()).ToList()); var actual = Assert.Throws(() => FixedWidth.Read(StreamOf(text)).ToList()); Assert.Equal(expected.LineNumber, actual.LineNumber); @@ -160,7 +160,7 @@ public void ReadFile_IsReEnumerable() [Fact] public async Task ReadAsync_MatchesReflectionReader() { - var expected = await new FixedWidthReader().ReadAsync(StreamOf(TwoPeople)).ToListAsync(TestContext.Current.CancellationToken); + var expected = await FixedWidth.ReadAsync(StreamOf(TwoPeople), FixedWidthLayout.FromAttributes()).ToListAsync(TestContext.Current.CancellationToken); var actual = await FixedWidth.ReadAsync(StreamOf(TwoPeople)).ToListAsync(TestContext.Current.CancellationToken); Assert.Equal(expected.Count, actual.Count); diff --git a/tests/FixedWidthParser.Tests/GeneratedWriterTests.cs b/tests/FixedWidthParser.Tests/GeneratedWriterTests.cs index 4d587a8..9d2b826 100644 --- a/tests/FixedWidthParser.Tests/GeneratedWriterTests.cs +++ b/tests/FixedWidthParser.Tests/GeneratedWriterTests.cs @@ -1,12 +1,11 @@ using System.Globalization; -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { /// /// The source-generated TryFormat (via ): parity - /// with , round-tripping through the generated TryParse, + /// with the attribute layout (), round-tripping through the generated TryParse, /// formatting options, nullable columns, converters, culture, and a destination that is too small. /// public class GeneratedWriterTests @@ -23,10 +22,10 @@ private static string WriteGenerated(in TModel model, IFormatProvider? f [InlineData("Max", -5, -1234.56)] public void Person_MatchesReflection(string name, int age, double salary) { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); Assert.Equal( - WriteOne(writer, new PersonModel { Name = name, Age = age, Salary = salary }), + WriteOne(layout, new PersonModel { Name = name, Age = age, Salary = salary }), WriteGenerated(new GenPersonModel { Name = name, Age = age, Salary = salary })); } @@ -47,7 +46,7 @@ public void Person_RoundTripsThroughGeneratedParse() public void FieldModel_MatchesReflection() { Assert.Equal( - WriteOne(new FixedWidthWriter(), new ProductModel { Code = "ABCDE", Quantity = 1234 }), + WriteOne(FixedWidthLayout.FromAttributes(), new ProductModel { Code = "ABCDE", Quantity = 1234 }), WriteGenerated(new GenProductModel { Code = "ABCDE", Quantity = 1234 })); } @@ -55,7 +54,7 @@ public void FieldModel_MatchesReflection() public void FloatColumn_MatchesReflection() { Assert.Equal( - WriteOne(new FixedWidthWriter(), new MeasurementModel { Value = 3.14f }), + WriteOne(FixedWidthLayout.FromAttributes(), new MeasurementModel { Value = 3.14f }), WriteGenerated(new GenMeasurementModel { Value = 3.14f })); } @@ -63,7 +62,7 @@ public void FloatColumn_MatchesReflection() public void DecimalColumn_MatchesReflection() { Assert.Equal( - WriteOne(new FixedWidthWriter(), new DecimalModel { Amount = 1234.56m }), + WriteOne(FixedWidthLayout.FromAttributes(), new DecimalModel { Amount = 1234.56m }), WriteGenerated(new GenDecimalModel { Amount = 1234.56m })); } @@ -71,7 +70,7 @@ public void DecimalColumn_MatchesReflection() public void RightAligned_MatchesReflection() { Assert.Equal( - WriteOne(new FixedWidthWriter(), new RightAlignedModel { Value = 30 }), + WriteOne(FixedWidthLayout.FromAttributes(), new RightAlignedModel { Value = 30 }), WriteGenerated(new GenRightAlignedModel { Value = 30 })); } @@ -79,7 +78,7 @@ public void RightAligned_MatchesReflection() public void ZeroPadded_MatchesReflection() { Assert.Equal( - WriteOne(new FixedWidthWriter(), new ZeroPaddedModel { Value = 30 }), + WriteOne(FixedWidthLayout.FromAttributes(), new ZeroPaddedModel { Value = 30 }), WriteGenerated(new GenZeroPaddedModel { Value = 30 })); } @@ -87,14 +86,14 @@ public void ZeroPadded_MatchesReflection() public void FormatString_MatchesReflection() { Assert.Equal( - WriteOne(new FixedWidthWriter(), new FormattedModel { Amount = 1234.5 }), + WriteOne(FixedWidthLayout.FromAttributes(), new FormattedModel { Amount = 1234.5 }), WriteGenerated(new GenFormattedModel { Amount = 1234.5 })); } [Fact] public void Overflow_Throws_MatchesReflection() { - Assert.Throws(() => WriteOne(new FixedWidthWriter(), new NarrowModel { Value = 12345 })); + Assert.Throws(() => WriteOne(FixedWidthLayout.FromAttributes(), new NarrowModel { Value = 12345 })); var buffer = new byte[16]; var generatedModel = new GenNarrowModel { Value = 12345 }; @@ -117,7 +116,7 @@ public void DestinationTooSmall_ReturnsFalseWithZeroBytesWritten() public void Nullable_Null_MatchesReflection() { Assert.Equal( - WriteOne(new FixedWidthWriter(), new NullableModel { Age = null, Amount = null }), + WriteOne(FixedWidthLayout.FromAttributes(), new NullableModel { Age = null, Amount = null }), WriteGenerated(new GenNullableModel { Age = null, Amount = null })); } @@ -125,7 +124,7 @@ public void Nullable_Null_MatchesReflection() public void Nullable_Value_MatchesReflection() { Assert.Equal( - WriteOne(new FixedWidthWriter(), new NullableModel { Age = 42, Amount = 123.45m }), + WriteOne(FixedWidthLayout.FromAttributes(), new NullableModel { Age = 42, Amount = 123.45m }), WriteGenerated(new GenNullableModel { Age = 42, Amount = 123.45m })); } @@ -133,20 +132,20 @@ public void Nullable_Value_MatchesReflection() public void Converter_MatchesReflection() { Assert.Equal( - WriteOne(new FixedWidthWriter(), new CentsConverterModel { Amount = new CentsValue(12345) }), + WriteOne(FixedWidthLayout.FromAttributes(), new CentsConverterModel { Amount = new CentsValue(12345) }), WriteGenerated(new GenCentsConverterModel { Amount = new CentsValue(12345) })); } [Fact] public void NullableConverter_MatchesReflection() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); Assert.Equal( - WriteOne(writer, new NullableConverterModel { Amount = null }), + WriteOne(layout, new NullableConverterModel { Amount = null }), WriteGenerated(new GenNullableConverterModel { Amount = null })); Assert.Equal( - WriteOne(writer, new NullableConverterModel { Amount = new CentsValue(999) }), + WriteOne(layout, new NullableConverterModel { Amount = new CentsValue(999) }), WriteGenerated(new GenNullableConverterModel { Amount = new CentsValue(999) })); } @@ -154,7 +153,7 @@ public void NullableConverter_MatchesReflection() public void DecodedParsable_MatchesReflection() { Assert.Equal( - WriteOne(new FixedWidthWriter(), new ShoutModel { Word = new Shout("HEY") }), + WriteOne(FixedWidthLayout.FromAttributes(), new ShoutModel { Word = new Shout("HEY") }), WriteGenerated(new GenShoutModel { Word = new Shout("HEY") })); } @@ -162,10 +161,11 @@ public void DecodedParsable_MatchesReflection() public void Culture_HonorsDecimalSeparator_MatchesReflection() { var deDe = CultureInfo.GetCultureInfo("de-DE"); - var writer = new FixedWidthWriter(new FixedWidthWriterOptions { FormatProvider = deDe }); + var writerOptions = new FixedWidthWriterOptions { FormatProvider = deDe }; + var layout = FixedWidthLayout.FromAttributes(); Assert.Equal( - WriteOne(writer, new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }), + WriteOne(layout, new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }, writerOptions), WriteGenerated(new GenPersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }, deDe)); } } diff --git a/tests/FixedWidthParser.Tests/LayoutWriteTests.cs b/tests/FixedWidthParser.Tests/LayoutWriteTests.cs index ef3e278..1ac66ce 100644 --- a/tests/FixedWidthParser.Tests/LayoutWriteTests.cs +++ b/tests/FixedWidthParser.Tests/LayoutWriteTests.cs @@ -1,5 +1,4 @@ using System.Text; -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -78,7 +77,7 @@ public void Layout_WriteMany_MatchesWriter() FixedWidth.WriteMany(stream, People, FixedWidthLayout.FromAttributes(), Lf); - Assert.Equal(WriteMany(new FixedWidthWriter(Lf), (IEnumerable)People), Decode(stream)); + Assert.Equal(WriteMany(FixedWidthLayout.FromAttributes(), (IEnumerable)People, Lf), Decode(stream)); } [Fact] diff --git a/tests/FixedWidthParser.Tests/NullableColumnTests.cs b/tests/FixedWidthParser.Tests/NullableColumnTests.cs index 969fc68..24fda02 100644 --- a/tests/FixedWidthParser.Tests/NullableColumnTests.cs +++ b/tests/FixedWidthParser.Tests/NullableColumnTests.cs @@ -1,5 +1,3 @@ -using FixedWidthParser.Parsers; -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -19,7 +17,7 @@ private static byte[] Line(string age, string amount) [Fact] public void BlankColumn_ParsesToNull() { - Assert.True(new FixedWidthParser().TryParse(Line("", ""), Ctx, out var model)); + Assert.True(FixedWidth.TryParse(Line("", ""), FixedWidthLayout.FromAttributes(), Ctx, out var model)); Assert.Null(model.Age); Assert.Null(model.Amount); } @@ -27,7 +25,7 @@ public void BlankColumn_ParsesToNull() [Fact] public void PresentValue_Parses() { - Assert.True(new FixedWidthParser().TryParse(Line("30", "123.45"), Ctx, out var model)); + Assert.True(FixedWidth.TryParse(Line("30", "123.45"), FixedWidthLayout.FromAttributes(), Ctx, out var model)); Assert.Equal(30, model.Age); Assert.Equal(123.45m, model.Amount); } @@ -35,13 +33,13 @@ public void PresentValue_Parses() [Fact] public void InvalidValue_ReturnsFalse() { - Assert.False(new FixedWidthParser().TryParse(Line("XX", ""), Ctx, out _)); + Assert.False(FixedWidth.TryParse(Line("XX", ""), FixedWidthLayout.FromAttributes(), Ctx, out _)); } [Fact] public void Write_Null_WritesBlank() { - string line = WriteOne(new FixedWidthWriter(), new NullableModel { Age = null, Amount = null }); + string line = WriteOne(FixedWidthLayout.FromAttributes(), new NullableModel { Age = null, Amount = null }); Assert.Equal(new string(' ', 15), line); } @@ -51,9 +49,9 @@ public void RoundTrip_PreservesNullAndValue() { var original = new NullableModel { Age = 42, Amount = null }; - string line = WriteOne(new FixedWidthWriter(), original); + string line = WriteOne(FixedWidthLayout.FromAttributes(), original); - Assert.True(new FixedWidthParser().TryParse(Bytes(line), Ctx, out var parsed)); + Assert.True(FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var parsed)); Assert.Equal(original.Age, parsed.Age); Assert.Null(parsed.Amount); } @@ -64,7 +62,7 @@ public void RoundTrip_PreservesNullAndValue() [InlineData("XX ")] public void Generated_MatchesReflection(string line) { - bool okR = new FixedWidthParser().TryParse(Bytes(line), Ctx, out var r); + bool okR = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var r); bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); Assert.Equal(okR, okG); @@ -77,7 +75,7 @@ public void Generated_MatchesReflection(string line) [InlineData("12345 ")] public void NullableConverter_Generated_MatchesReflection(string line) { - bool okR = new FixedWidthParser().TryParse(Bytes(line), Ctx, out var r); + bool okR = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var r); bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); Assert.True(okR); @@ -88,14 +86,14 @@ public void NullableConverter_Generated_MatchesReflection(string line) [Fact] public void NullableConverter_PresentValue_Parses() { - Assert.True(new FixedWidthParser().TryParse("12345 "u8, Ctx, out var model)); + Assert.True(FixedWidth.TryParse("12345 "u8, FixedWidthLayout.FromAttributes(), Ctx, out var model)); Assert.Equal(new CentsValue(12345), model.Amount); } [Fact] public void NullableConverter_Write_Null_WritesBlank() { - string line = WriteOne(new FixedWidthWriter(), new NullableConverterModel { Amount = null }); + string line = WriteOne(FixedWidthLayout.FromAttributes(), new NullableConverterModel { Amount = null }); Assert.Equal(new string(' ', 8), line); } diff --git a/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs b/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs index 786f813..631e80f 100644 --- a/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs +++ b/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs @@ -91,17 +91,17 @@ private static string Content(int lines, int badLine = 0) private List Sequential(FixedWidthReaderOptions? options = null) { - return [.. new FixedWidthReader(options).ReadFile(_path).Select(m => m.Age)]; + return [.. FixedWidth.ReadFile(_path, FixedWidthLayout.FromAttributes(), options).Select(m => m.Age)]; } private Task Aggregate(Source source, FixedWidthParallelOptions parallel, FixedWidthReaderOptions? options = null) { - var reader = new FixedWidthReader(options); + var layout = FixedWidthLayout.FromAttributes(); var ct = TestContext.Current.CancellationToken; return source switch { - Source.File => reader.AggregateFileAsync(_path, parallel, ct), - _ => reader.AggregateAsync(File.OpenRead(_path), leaveOpen: false, parallel, ct), + Source.File => FixedWidth.AggregateFileAsync(_path, layout, options, parallel, ct), + _ => FixedWidth.AggregateAsync(File.OpenRead(_path), layout, leaveOpen: false, options, parallel, ct), }; } @@ -206,8 +206,7 @@ public async Task Stream_IsDisposedUnlessLeftOpen(bool leaveOpen) File.WriteAllText(_path, Content(10)); await using var stream = File.OpenRead(_path); - await new FixedWidthReader().AggregateAsync( - stream, leaveOpen, new FixedWidthParallelOptions { DegreeOfParallelism = 2, ChunkSize = 16 }, TestContext.Current.CancellationToken); + await FixedWidth.AggregateAsync(stream, FixedWidthLayout.FromAttributes(), leaveOpen, null, new FixedWidthParallelOptions { DegreeOfParallelism = 2, ChunkSize = 16 }, TestContext.Current.CancellationToken); Assert.Equal(leaveOpen, stream.CanRead); } @@ -219,13 +218,13 @@ public async Task CancelledToken_Throws(Source source) File.WriteAllText(_path, Content(10)); using var cts = new CancellationTokenSource(); await cts.CancelAsync(); - var reader = new FixedWidthReader(); + var layout = FixedWidthLayout.FromAttributes(); var parallel = new FixedWidthParallelOptions { DegreeOfParallelism = 2 }; await Assert.ThrowsAnyAsync(() => source switch { - Source.File => reader.AggregateFileAsync(_path, parallel, cts.Token), - _ => reader.AggregateAsync(File.OpenRead(_path), leaveOpen: false, parallel, cts.Token), + Source.File => FixedWidth.AggregateFileAsync(_path, layout, null, parallel, cts.Token), + _ => FixedWidth.AggregateAsync(File.OpenRead(_path), layout, leaveOpen: false, null, parallel, cts.Token), }); } @@ -234,13 +233,13 @@ public async Task CancelledToken_Throws(Source source) public async Task NegativeDegreeOfParallelism_Throws() { File.WriteAllText(_path, Content(10)); - var reader = new FixedWidthReader(); + var layout = FixedWidthLayout.FromAttributes(); var parallel = new FixedWidthParallelOptions { DegreeOfParallelism = -1 }; var ct = TestContext.Current.CancellationToken; await using var stream = File.OpenRead(_path); - await Assert.ThrowsAsync(() => reader.AggregateFileAsync(_path, parallel, ct)); - await Assert.ThrowsAsync(() => reader.AggregateAsync(stream, leaveOpen: false, parallel, ct)); + await Assert.ThrowsAsync(() => FixedWidth.AggregateFileAsync(_path, layout, null, parallel, ct)); + await Assert.ThrowsAsync(() => FixedWidth.AggregateAsync(stream, layout, leaveOpen: false, null, parallel, ct)); Assert.True(stream.CanRead); } } diff --git a/tests/FixedWidthParser.Tests/ParserTests.cs b/tests/FixedWidthParser.Tests/ParserTests.cs index e780bce..10d1a6b 100644 --- a/tests/FixedWidthParser.Tests/ParserTests.cs +++ b/tests/FixedWidthParser.Tests/ParserTests.cs @@ -1,5 +1,4 @@ using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Parsers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -9,10 +8,10 @@ public class ParserTests [Fact] public void TryParse_WellFormedLine_ParsesAllColumns() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); // Name Age Salary - bool ok = parser.TryParse("John Doe 30 60000.00 "u8, Ctx, out var model); + bool ok = FixedWidth.TryParse("John Doe 30 60000.00 "u8, layout, Ctx, out var model); Assert.True(ok); Assert.Equal("John Doe", model.Name); @@ -23,9 +22,9 @@ public void TryParse_WellFormedLine_ParsesAllColumns() [Fact] public void TryParse_TrimsTrailingSpacesOnStringColumns() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); - parser.TryParse("Al 25 100.00 "u8, Ctx, out var model); + FixedWidth.TryParse("Al 25 100.00 "u8, layout, Ctx, out var model); Assert.Equal("Al", model.Name); } @@ -33,9 +32,9 @@ public void TryParse_TrimsTrailingSpacesOnStringColumns() [Fact] public void TryParse_InvalidNumber_ReturnsFalse() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); - bool ok = parser.TryParse("John Doe XX 60000.00 "u8, Ctx, out _); + bool ok = FixedWidth.TryParse("John Doe XX 60000.00 "u8, layout, Ctx, out _); Assert.False(ok); } @@ -43,9 +42,9 @@ public void TryParse_InvalidNumber_ReturnsFalse() [Fact] public void TryParse_ShortLine_ReturnsFalse() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); - bool ok = parser.TryParse("John"u8, Ctx, out _); + bool ok = FixedWidth.TryParse("John"u8, layout, Ctx, out _); Assert.False(ok); } @@ -53,9 +52,9 @@ public void TryParse_ShortLine_ReturnsFalse() [Fact] public void TryParse_ShortLine_StringColumnMissing_ReturnsFalse() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); - bool ok = parser.TryParse("42"u8, Ctx, out var model); + bool ok = FixedWidth.TryParse("42"u8, layout, Ctx, out var model); Assert.False(ok); Assert.Equal(default, model); @@ -64,11 +63,11 @@ public void TryParse_ShortLine_StringColumnMissing_ReturnsFalse() [Fact] public void TryParse_WithStringPool_ReturnsInternedInstances() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); var context = CtxWith(pool: new StringPool()); - parser.TryParse("Alice 25 100.00 "u8, context, out var first); - parser.TryParse("Alice 40 200.00 "u8, context, out var second); + FixedWidth.TryParse("Alice 25 100.00 "u8, layout, context, out var first); + FixedWidth.TryParse("Alice 40 200.00 "u8, layout, context, out var second); Assert.Equal("Alice", first.Name); Assert.Same(first.Name, second.Name); @@ -77,10 +76,10 @@ public void TryParse_WithStringPool_ReturnsInternedInstances() [Fact] public void TryParse_WithoutPool_ReturnsDistinctInstances() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); - parser.TryParse("Alice 25 100.00 "u8, Ctx, out var first); - parser.TryParse("Alice 25 100.00 "u8, Ctx, out var second); + FixedWidth.TryParse("Alice 25 100.00 "u8, layout, Ctx, out var first); + FixedWidth.TryParse("Alice 25 100.00 "u8, layout, Ctx, out var second); Assert.Equal(first.Name, second.Name); Assert.NotSame(first.Name, second.Name); @@ -89,9 +88,9 @@ public void TryParse_WithoutPool_ReturnsDistinctInstances() [Fact] public void TryParse_FieldBasedModel_ParsesFields() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); - bool ok = parser.TryParse("ABC 12 "u8, Ctx, out var product); + bool ok = FixedWidth.TryParse("ABC 12 "u8, layout, Ctx, out var product); Assert.True(ok); Assert.Equal("ABC", product.Code); @@ -101,9 +100,9 @@ public void TryParse_FieldBasedModel_ParsesFields() [Fact] public void TryParse_FloatColumn_Parses() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); - bool ok = parser.TryParse("3.14 "u8, Ctx, out var measurement); + bool ok = FixedWidth.TryParse("3.14 "u8, layout, Ctx, out var measurement); Assert.True(ok); Assert.Equal(3.14f, measurement.Value, 2); @@ -112,9 +111,9 @@ public void TryParse_FloatColumn_Parses() [Fact] public void TryParse_StructWithoutDeclaredConstructor_Parses() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); - Assert.True(parser.TryParse("ABC"u8, Ctx, out var model)); + Assert.True(FixedWidth.TryParse("ABC"u8, layout, Ctx, out var model)); Assert.Equal("ABC", model.Code); } @@ -122,16 +121,16 @@ public void TryParse_StructWithoutDeclaredConstructor_Parses() public void Constructor_BuildError_IsRethrownOnEveryConstruction() { // The error is captured once in the static constructor and rethrown on each construction. - Assert.Throws(() => new FixedWidthParser()); - Assert.Throws(() => new FixedWidthParser()); + Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); + Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); } [Fact] public void TryParse_RefStructModel_Parses() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); - bool ok = parser.TryParse("John Doe 30 60000.00 "u8, Ctx, out var model); + bool ok = FixedWidth.TryParse("John Doe 30 60000.00 "u8, layout, Ctx, out var model); Assert.True(ok); Assert.Equal("John Doe", model.Name); diff --git a/tests/FixedWidthParser.Tests/PropertyRoundTripTests.cs b/tests/FixedWidthParser.Tests/PropertyRoundTripTests.cs index 06ab7d3..8d73154 100644 --- a/tests/FixedWidthParser.Tests/PropertyRoundTripTests.cs +++ b/tests/FixedWidthParser.Tests/PropertyRoundTripTests.cs @@ -1,7 +1,5 @@ using CsCheck; using FixedWidthParser.Attributes; -using FixedWidthParser.Parsers; -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -30,13 +28,13 @@ public class PropertyRoundTripTests [Fact] public void RoundTrip_TextAndNumber_IsLossless() { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); + var writer = FixedWidthLayout.FromAttributes(); + var parser = FixedWidthLayout.FromAttributes(); GenTextNumber.Sample(original => { string line = WriteOne(writer, original); - bool ok = parser.TryParse(Bytes(line), Ctx, out var parsed); + bool ok = FixedWidth.TryParse(Bytes(line), parser, Ctx, out var parsed); Assert.True(ok); Assert.Equal(original.Text, parsed.Text); @@ -47,14 +45,14 @@ public void RoundTrip_TextAndNumber_IsLossless() [Fact] public void RoundTrip_Decimal_IsExact() { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); + var writer = FixedWidthLayout.FromAttributes(); + var parser = FixedWidthLayout.FromAttributes(); GenMoney.Sample(amount => { var original = new MoneyModel { Amount = amount }; string line = WriteOne(writer, original); - bool ok = parser.TryParse(Bytes(line), Ctx, out var parsed); + bool ok = FixedWidth.TryParse(Bytes(line), parser, Ctx, out var parsed); Assert.True(ok); Assert.Equal(original.Amount, parsed.Amount); @@ -64,15 +62,15 @@ public void RoundTrip_Decimal_IsExact() [Fact] public void RoundTrip_Double_F2_PreservesToTwoDecimals() { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); + var writer = FixedWidthLayout.FromAttributes(); + var parser = FixedWidthLayout.FromAttributes(); // Values with at most two fractional digits, formatted "F2": survives to two decimals. Gen.Long[-9_999_999, 9_999_999].Select(cents => cents / 100.0).Sample(value => { var original = new DoubleF2Model { Value = value }; string line = WriteOne(writer, original); - bool ok = parser.TryParse(Bytes(line), Ctx, out var parsed); + bool ok = FixedWidth.TryParse(Bytes(line), parser, Ctx, out var parsed); Assert.True(ok); Assert.Equal(original.Value, parsed.Value, 2); diff --git a/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs b/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs index d26ef34..24a0e0c 100644 --- a/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs +++ b/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs @@ -19,7 +19,7 @@ public class ReaderOptionsTests [Fact] public void ParseError_NamesColumnAndRawValue_OnBothPaths() { - var reflected = Assert.Throws(() => new FixedWidthReader().Read(StreamOf(BadAge)).ToList()); + var reflected = Assert.Throws(() => FixedWidth.Read(StreamOf(BadAge), FixedWidthLayout.FromAttributes()).ToList()); var generated = Assert.Throws(() => FixedWidth.Read(StreamOf(BadAge)).ToList()); foreach (var ex in new[] { reflected, generated }) @@ -36,7 +36,7 @@ public void ParseError_NamesColumnAndRawValue_OnBothPaths() [Fact] public void ParseError_ShortLine_SaysHowLongTheLayoutIs() { - var reflected = Assert.Throws(() => new FixedWidthReader().Read(StreamOf("John\n")).ToList()); + var reflected = Assert.Throws(() => FixedWidth.Read(StreamOf("John\n"), FixedWidthLayout.FromAttributes()).ToList()); var generated = Assert.Throws(() => FixedWidth.Read(StreamOf("John\n")).ToList()); foreach (var ex in new[] { reflected, generated }) @@ -50,7 +50,7 @@ public void ParseError_ShortLine_SaysHowLongTheLayoutIs() [Fact] public void ParseError_IsAFormatException() { - Assert.ThrowsAny(() => new FixedWidthReader().Read(StreamOf(BadAge)).ToList()); + Assert.ThrowsAny(() => FixedWidth.Read(StreamOf(BadAge), FixedWidthLayout.FromAttributes()).ToList()); } [Fact] @@ -59,7 +59,7 @@ public void OnParseError_ReturningTrue_SkipsTheLine() var errors = new List(); var options = new FixedWidthReaderOptions { OnParseError = ex => { errors.Add(ex); return true; } }; - var reflected = new FixedWidthReader(options).Read(StreamOf(BadAge)).Select(p => p.Name).ToList(); + var reflected = FixedWidth.Read(StreamOf(BadAge), FixedWidthLayout.FromAttributes(), options: options).Select(p => p.Name).ToList(); var generated = FixedWidth.Read(StreamOf(BadAge), options: options).Select(p => p.Name).ToList(); Assert.Equal(["John Doe", "Max"], reflected); @@ -73,7 +73,7 @@ public void OnParseError_ReturningFalse_Throws() { var options = new FixedWidthReaderOptions { OnParseError = _ => false }; - Assert.Throws(() => new FixedWidthReader(options).Read(StreamOf(BadAge)).ToList()); + Assert.Throws(() => FixedWidth.Read(StreamOf(BadAge), FixedWidthLayout.FromAttributes(), options: options).ToList()); } [Fact] @@ -81,7 +81,7 @@ public void OnParseError_SkippingTheLastLine_EndsCleanly() { var options = new FixedWidthReaderOptions { OnParseError = _ => true }; - var codes = new FixedWidthReader(options).Read(StreamOf("ABC\nA")).Select(m => m.Code).ToList(); + var codes = FixedWidth.Read(StreamOf("ABC\nA"), FixedWidthLayout.FromAttributes(), options: options).Select(m => m.Code).ToList(); Assert.Equal(["ABC"], codes); } @@ -90,9 +90,9 @@ public void OnParseError_SkippingTheLastLine_EndsCleanly() public async Task OnParseError_SkipsOnAsyncReads() { var options = new FixedWidthReaderOptions { OnParseError = _ => true }; - var reader = new FixedWidthReader(options); + var layout = FixedWidthLayout.FromAttributes(); - var fromStream = await reader.ReadAsync(StreamOf(BadAge)).Select(p => p.Name).ToListAsync(TestContext.Current.CancellationToken); + var fromStream = await FixedWidth.ReadAsync(StreamOf(BadAge), layout, options: options).Select(p => p.Name).ToListAsync(TestContext.Current.CancellationToken); Assert.Equal(["John Doe", "Max"], fromStream); } @@ -104,7 +104,7 @@ public void MaxLineLength_UnterminatedLine_Throws() var options = new FixedWidthReaderOptions { BufferSize = 16, MaxLineLength = 100 }; var source = new MemoryStream(new byte[10_000]); - var ex = Assert.Throws(() => new FixedWidthReader(options).Read(source).ToList()); + var ex = Assert.Throws(() => FixedWidth.Read(source, FixedWidthLayout.FromAttributes(), options: options).ToList()); Assert.Contains("MaxLineLength", ex.Message, StringComparison.Ordinal); Assert.Contains("Line 1", ex.Message, StringComparison.Ordinal); @@ -115,7 +115,7 @@ public void MaxLineLength_TerminatedLongLine_Throws() { var options = new FixedWidthReaderOptions { MaxLineLength = 5 }; - var ex = Assert.Throws(() => new FixedWidthReader(options).Read(StreamOf("ABC\nABCDEFGH\n")).ToList()); + var ex = Assert.Throws(() => FixedWidth.Read(StreamOf("ABC\nABCDEFGH\n"), FixedWidthLayout.FromAttributes(), options: options).ToList()); Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); } @@ -125,7 +125,7 @@ public void MaxLineLength_LineAtTheLimit_IsRead() { var options = new FixedWidthReaderOptions { BufferSize = 2, MaxLineLength = 3 }; - var codes = new FixedWidthReader(options).Read(StreamOf("ABC\r\nDEF")).Select(m => m.Code).ToList(); + var codes = FixedWidth.Read(StreamOf("ABC\r\nDEF"), FixedWidthLayout.FromAttributes(), options: options).Select(m => m.Code).ToList(); Assert.Equal(["ABC", "DEF"], codes); } @@ -137,7 +137,7 @@ public async Task MaxLineLength_UnterminatedLine_ThrowsOnAsyncRead() await Assert.ThrowsAsync(async () => { - await foreach (var _ in new FixedWidthReader(options).ReadAsync(new MemoryStream(new byte[10_000]))) + await foreach (var _ in FixedWidth.ReadAsync(new MemoryStream(new byte[10_000]), FixedWidthLayout.FromAttributes(), options: options)) { } }); @@ -150,7 +150,7 @@ public void Options_OutOfRange_Throw(int bufferSize, int maxLineLength) { var options = new FixedWidthReaderOptions { BufferSize = bufferSize, MaxLineLength = maxLineLength }; - Assert.Throws(() => new FixedWidthReader(options)); + Assert.Throws(() => FixedWidth.Read(StreamOf("ABC"), FixedWidthLayout.FromAttributes(), options: options)); Assert.Throws(() => FixedWidth.Read(StreamOf("ABC"), options: options)); } @@ -161,7 +161,7 @@ public void Latin1Encoding_DoesNotSkipWhatLooksLikeAUtf8Bom() var options = new FixedWidthReaderOptions { Encoding = Encoding.Latin1 }; byte[] bytes = [0xEF, 0xBB, 0xBF, .. "\nABC\n"u8]; - var codes = new FixedWidthReader(options).Read(new MemoryStream(bytes)).Select(m => m.Code).ToList(); + var codes = FixedWidth.Read(new MemoryStream(bytes), FixedWidthLayout.FromAttributes(), options: options).Select(m => m.Code).ToList(); Assert.Equal(["", "ABC"], codes); } @@ -171,7 +171,7 @@ public void Utf8Text_WithMultiByteCharacters_KeepsBytePositions() { // "Zoë" is 4 bytes in UTF-8; the layout counts bytes, so the name column must be 6 wide here // and the age still starts at byte 6. - var person = new FixedWidthReader().Read(StreamOf("Zoë 42 \n")).Single(); + var person = FixedWidth.Read(StreamOf("Zoë 42 \n"), FixedWidthLayout.FromAttributes()).Single(); Assert.Equal("Zoë", person.Name); Assert.Equal(42, person.Age); diff --git a/tests/FixedWidthParser.Tests/ReaderTests.cs b/tests/FixedWidthParser.Tests/ReaderTests.cs index 1a070a8..bf94d2a 100644 --- a/tests/FixedWidthParser.Tests/ReaderTests.cs +++ b/tests/FixedWidthParser.Tests/ReaderTests.cs @@ -1,12 +1,11 @@ using CommunityToolkit.HighPerformance.Buffers; using FixedWidthParser.Readers; -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { /// - /// over streams and files, sync and async: line endings, BOM, + /// Reading through an attribute layout over streams and files, sync and async: line endings, BOM, /// buffer growth, error reporting, leaveOpen and string pooling. /// public class ReaderTests @@ -19,7 +18,7 @@ public class ReaderTests [Fact] public void Read_MultipleLines_ParsesAll() { - var people = new FixedWidthReader().Read(StreamOf(TwoPeople)).ToList(); + var people = FixedWidth.Read(StreamOf(TwoPeople), FixedWidthLayout.FromAttributes()).ToList(); Assert.Equal(2, people.Count); Assert.Equal("John Doe", people[0].Name); @@ -35,7 +34,7 @@ public void Read_MultipleLines_ParsesAll() [InlineData("ABC\nDEF")] public void Read_HandlesLineEndingsAndTrailingNewline(string text) { - var codes = new FixedWidthReader().Read(StreamOf(text)).Select(m => m.Code).ToList(); + var codes = FixedWidth.Read(StreamOf(text), FixedWidthLayout.FromAttributes()).Select(m => m.Code).ToList(); Assert.Equal(["ABC", "DEF"], codes); } @@ -43,7 +42,7 @@ public void Read_HandlesLineEndingsAndTrailingNewline(string text) [Fact] public void Read_SkipsEmptyLines() { - var codes = new FixedWidthReader().Read(StreamOf("ABC\n\n\nDEF\n")).Select(m => m.Code).ToList(); + var codes = FixedWidth.Read(StreamOf("ABC\n\n\nDEF\n"), FixedWidthLayout.FromAttributes()).Select(m => m.Code).ToList(); Assert.Equal(["ABC", "DEF"], codes); } @@ -53,7 +52,7 @@ public void Read_SkipsLeadingUtf8Bom() { byte[] bytes = [0xEF, 0xBB, 0xBF, .. "ABC\nDEF\n"u8]; - var codes = new FixedWidthReader().Read(new MemoryStream(bytes)).Select(m => m.Code).ToList(); + var codes = FixedWidth.Read(new MemoryStream(bytes), FixedWidthLayout.FromAttributes()).Select(m => m.Code).ToList(); Assert.Equal(["ABC", "DEF"], codes); } @@ -62,9 +61,10 @@ public void Read_SkipsLeadingUtf8Bom() public void Read_LineLongerThanBuffer_StillParses() { // A tiny buffer forces compaction and growth (25-byte lines). - var reader = new FixedWidthReader(new FixedWidthReaderOptions { BufferSize = 4 }); + var readerOptions = new FixedWidthReaderOptions { BufferSize = 4 }; + var layout = FixedWidthLayout.FromAttributes(); - var people = reader.Read(StreamOf(TwoPeople)).ToList(); + var people = FixedWidth.Read(StreamOf(TwoPeople), layout, options: readerOptions).ToList(); Assert.Equal(2, people.Count); Assert.Equal("John Doe", people[0].Name); @@ -74,10 +74,10 @@ public void Read_LineLongerThanBuffer_StillParses() [Fact] public void Read_InvalidLine_ThrowsWithLineNumber() { - var reader = new FixedWidthReader(); + var layout = FixedWidthLayout.FromAttributes(); var ex = Assert.Throws( - () => reader.Read(StreamOf("John Doe 30 60000.00 \nJane XX 55000.00 ")).ToList()); + () => FixedWidth.Read(StreamOf("John Doe 30 60000.00 \nJane XX 55000.00 "), layout).ToList()); Assert.Equal(2, ex.LineNumber); Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); @@ -88,7 +88,7 @@ public void Read_LeaveOpenTrue_KeepsStreamOpen() { using var stream = StreamOf("ABC\nDEF\n"); - _ = new FixedWidthReader().Read(stream, leaveOpen: true).ToList(); + _ = FixedWidth.Read(stream, FixedWidthLayout.FromAttributes(), leaveOpen: true).ToList(); Assert.True(stream.CanRead); } @@ -98,7 +98,7 @@ public void Read_LeaveOpenFalse_DisposesStream() { var stream = StreamOf("ABC\nDEF\n"); - _ = new FixedWidthReader().Read(stream).ToList(); + _ = FixedWidth.Read(stream, FixedWidthLayout.FromAttributes()).ToList(); Assert.False(stream.CanRead); } @@ -106,15 +106,16 @@ public void Read_LeaveOpenFalse_DisposesStream() [Fact] public void Read_NullStream_Throws() { - Assert.Throws(() => new FixedWidthReader().Read(null!)); + Assert.Throws(() => FixedWidth.Read(null!, FixedWidthLayout.FromAttributes())); } [Fact] public void Read_WithStringPool_InternsRepeatedValues() { - var reader = new FixedWidthReader(new FixedWidthReaderOptions { StringPool = new StringPool() }); + var readerOptions = new FixedWidthReaderOptions { StringPool = new StringPool() }; + var layout = FixedWidthLayout.FromAttributes(); - var codes = reader.Read(StreamOf("ABC\nABC\n")).ToList(); + var codes = FixedWidth.Read(StreamOf("ABC\nABC\n"), layout, options: readerOptions).ToList(); Assert.Equal("ABC", codes[0].Code); Assert.Same(codes[0].Code, codes[1].Code); @@ -126,7 +127,7 @@ public void GetEnumerator_IsStruct_ForAllocationFreeForeach() Assert.True(typeof(RecordEnumerator>).IsValueType); int count = 0; - foreach (var _ in new FixedWidthReader().Read(StreamOf("ABC\nDEF\n"))) + foreach (var _ in FixedWidth.Read(StreamOf("ABC\nDEF\n"), FixedWidthLayout.FromAttributes())) { count++; } @@ -137,11 +138,12 @@ public void GetEnumerator_IsStruct_ForAllocationFreeForeach() [Fact] public void TryParse_UsesTheReaderOptions() { - var reader = new FixedWidthReader(new FixedWidthReaderOptions { FormatProvider = System.Globalization.CultureInfo.GetCultureInfo("de-DE") }); + var readerOptions = new FixedWidthReaderOptions { FormatProvider = System.Globalization.CultureInfo.GetCultureInfo("de-DE") }; + var layout = FixedWidthLayout.FromAttributes(); - Assert.True(reader.TryParse("1234,56 "u8, out var model)); + Assert.True(FixedWidth.TryParse("1234,56 "u8, layout, readerOptions.Context, out var model)); Assert.Equal(1234.56m, model.Amount); - Assert.False(reader.TryParse("short"u8, out _)); + Assert.False(FixedWidth.TryParse("short"u8, layout, readerOptions.Context, out _)); } [Fact] @@ -151,10 +153,10 @@ public void ReadFile_IsReEnumerable() try { File.WriteAllText(path, "ABC\nDEF\n"); - var reader = new FixedWidthReader(); + var layout = FixedWidthLayout.FromAttributes(); - var first = reader.ReadFile(path).Select(m => m.Code).ToList(); - var second = reader.ReadFile(path).Select(m => m.Code).ToList(); + var first = FixedWidth.ReadFile(path, layout).Select(m => m.Code).ToList(); + var second = FixedWidth.ReadFile(path, layout).Select(m => m.Code).ToList(); Assert.Equal(["ABC", "DEF"], first); Assert.Equal(first, second); @@ -178,10 +180,10 @@ public void Writer_To_File_To_Reader_RoundTrip() ]; using (var fs = File.Create(path)) { - new FixedWidthWriter().WriteMany(fs, models.AsSpan()); + FixedWidth.WriteMany(fs, models.AsSpan(), FixedWidthLayout.FromAttributes()); } - var parsed = new FixedWidthReader().ReadFile(path).ToList(); + var parsed = FixedWidth.ReadFile(path, FixedWidthLayout.FromAttributes()).ToList(); Assert.Equal(2, parsed.Count); Assert.Equal("John Doe", parsed[0].Name); @@ -199,7 +201,7 @@ public void Writer_To_File_To_Reader_RoundTrip() [Fact] public async Task ReadAsync_Stream_ParsesAll() { - var people = await new FixedWidthReader().ReadAsync(StreamOf(TwoPeople)).ToListAsync(TestContext.Current.CancellationToken); + var people = await FixedWidth.ReadAsync(StreamOf(TwoPeople), FixedWidthLayout.FromAttributes()).ToListAsync(TestContext.Current.CancellationToken); Assert.Equal(2, people.Count); Assert.Equal("John Doe", people[0].Name); @@ -210,9 +212,10 @@ public async Task ReadAsync_Stream_ParsesAll() [Fact] public async Task ReadAsync_LineLongerThanBuffer_StillParses() { - var reader = new FixedWidthReader(new FixedWidthReaderOptions { BufferSize = 4 }); + var readerOptions = new FixedWidthReaderOptions { BufferSize = 4 }; + var layout = FixedWidthLayout.FromAttributes(); - var people = await reader.ReadAsync(StreamOf(TwoPeople)).ToListAsync(TestContext.Current.CancellationToken); + var people = await FixedWidth.ReadAsync(StreamOf(TwoPeople), layout, options: readerOptions).ToListAsync(TestContext.Current.CancellationToken); Assert.Equal(2, people.Count); Assert.Equal(55000.00, people[1].Salary, 2); @@ -223,7 +226,7 @@ public async Task ReadAsync_SkipsLeadingUtf8Bom() { byte[] bytes = [0xEF, 0xBB, 0xBF, .. "ABC\nDEF\n"u8]; - var codes = await new FixedWidthReader().ReadAsync(new MemoryStream(bytes)).Select(m => m.Code).ToListAsync(TestContext.Current.CancellationToken); + var codes = await FixedWidth.ReadAsync(new MemoryStream(bytes), FixedWidthLayout.FromAttributes()).Select(m => m.Code).ToListAsync(TestContext.Current.CancellationToken); Assert.Equal(["ABC", "DEF"], codes); } @@ -231,11 +234,11 @@ public async Task ReadAsync_SkipsLeadingUtf8Bom() [Fact] public async Task ReadAsync_InvalidLine_ThrowsWithLineNumber() { - var reader = new FixedWidthReader(); + var layout = FixedWidthLayout.FromAttributes(); var ex = await Assert.ThrowsAsync(async () => { - await foreach (var _ in reader.ReadAsync(StreamOf("John Doe 30 60000.00 \nJane XX 55000.00 "))) + await foreach (var _ in FixedWidth.ReadAsync(StreamOf("John Doe 30 60000.00 \nJane XX 55000.00 "), layout)) { } }); @@ -251,7 +254,7 @@ public async Task ReadAsync_CancelledToken_Throws() await Assert.ThrowsAnyAsync(async () => { - await foreach (var _ in new FixedWidthReader().ReadAsync(StreamOf("ABC\nDEF\n")).WithCancellation(cts.Token)) + await foreach (var _ in FixedWidth.ReadAsync(StreamOf("ABC\nDEF\n"), FixedWidthLayout.FromAttributes()).WithCancellation(cts.Token)) { } }); @@ -260,12 +263,12 @@ await Assert.ThrowsAnyAsync(async () => [Fact] public async Task ReadAsync_LeaveOpen_ControlsDisposal() { - var reader = new FixedWidthReader(); + var layout = FixedWidthLayout.FromAttributes(); using var kept = StreamOf("ABC\n"); var owned = StreamOf("ABC\n"); - _ = await reader.ReadAsync(kept, leaveOpen: true).ToListAsync(TestContext.Current.CancellationToken); - _ = await reader.ReadAsync(owned).ToListAsync(TestContext.Current.CancellationToken); + _ = await FixedWidth.ReadAsync(kept, layout, leaveOpen: true).ToListAsync(TestContext.Current.CancellationToken); + _ = await FixedWidth.ReadAsync(owned, layout).ToListAsync(TestContext.Current.CancellationToken); Assert.True(kept.CanRead); Assert.False(owned.CanRead); @@ -274,7 +277,7 @@ public async Task ReadAsync_LeaveOpen_ControlsDisposal() [Fact] public void ReadAsync_NullStream_Throws() { - Assert.Throws(() => new FixedWidthReader().ReadAsync((Stream)null!)); + Assert.Throws(() => FixedWidth.ReadAsync((Stream)null!, FixedWidthLayout.FromAttributes())); } [Fact] @@ -284,10 +287,10 @@ public async Task ReadFileAsync_MatchesSyncReadFile() try { await File.WriteAllTextAsync(path, "ABC\nDEF\nGHI\n", TestContext.Current.CancellationToken); - var reader = new FixedWidthReader(); + var layout = FixedWidthLayout.FromAttributes(); - var asyncCodes = await reader.ReadFileAsync(path).Select(m => m.Code).ToListAsync(TestContext.Current.CancellationToken); - var syncCodes = reader.ReadFile(path).Select(m => m.Code).ToList(); + var asyncCodes = await FixedWidth.ReadFileAsync(path, layout).Select(m => m.Code).ToListAsync(TestContext.Current.CancellationToken); + var syncCodes = FixedWidth.ReadFile(path, layout).Select(m => m.Code).ToList(); Assert.Equal(["ABC", "DEF", "GHI"], asyncCodes); Assert.Equal(syncCodes, asyncCodes); @@ -301,9 +304,10 @@ public async Task ReadFileAsync_MatchesSyncReadFile() [Fact] public async Task ReadAsync_WithStringPool_InternsRepeatedValues() { - var reader = new FixedWidthReader(new FixedWidthReaderOptions { StringPool = new StringPool() }); + var readerOptions = new FixedWidthReaderOptions { StringPool = new StringPool() }; + var layout = FixedWidthLayout.FromAttributes(); - var codes = await reader.ReadAsync(StreamOf("ABC\nABC\n")).ToListAsync(TestContext.Current.CancellationToken); + var codes = await FixedWidth.ReadAsync(StreamOf("ABC\nABC\n"), layout, options: readerOptions).ToListAsync(TestContext.Current.CancellationToken); Assert.Same(codes[0].Code, codes[1].Code); } diff --git a/tests/FixedWidthParser.Tests/RoundTripTests.cs b/tests/FixedWidthParser.Tests/RoundTripTests.cs index 27a5155..0df97dc 100644 --- a/tests/FixedWidthParser.Tests/RoundTripTests.cs +++ b/tests/FixedWidthParser.Tests/RoundTripTests.cs @@ -1,5 +1,3 @@ -using FixedWidthParser.Parsers; -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -21,12 +19,12 @@ public static IEnumerable PersonCases() [MemberData(nameof(PersonCases))] public void RoundTrip_Person_PreservesValues(string name, int age, double salary) { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); + var writer = FixedWidthLayout.FromAttributes(); + var parser = FixedWidthLayout.FromAttributes(); var original = new PersonModel { Name = name, Age = age, Salary = salary }; string line = WriteOne(writer, original); - bool ok = parser.TryParse(Bytes(line), Ctx, out var parsed); + bool ok = FixedWidth.TryParse(Bytes(line), parser, Ctx, out var parsed); Assert.True(ok); Assert.Equal(original.Name, parsed.Name); @@ -37,12 +35,12 @@ public void RoundTrip_Person_PreservesValues(string name, int age, double salary [Fact] public void RoundTrip_FieldModel_PreservesValues() { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); + var writer = FixedWidthLayout.FromAttributes(); + var parser = FixedWidthLayout.FromAttributes(); var original = new ProductModel { Code = "ABCDE", Quantity = 1234 }; string line = WriteOne(writer, original); - bool ok = parser.TryParse(Bytes(line), Ctx, out var parsed); + bool ok = FixedWidth.TryParse(Bytes(line), parser, Ctx, out var parsed); Assert.True(ok); Assert.Equal(original.Code, parsed.Code); diff --git a/tests/FixedWidthParser.Tests/SourceGenParityTests.cs b/tests/FixedWidthParser.Tests/SourceGenParityTests.cs index 5c4d3b9..f86b4b9 100644 --- a/tests/FixedWidthParser.Tests/SourceGenParityTests.cs +++ b/tests/FixedWidthParser.Tests/SourceGenParityTests.cs @@ -1,13 +1,12 @@ using System.Text; using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Parsers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { /// /// The source-generated TryParse (via ) produces the - /// same results as the runtime , including what each reports + /// same results as the type's layout, including what each reports /// about a rejected line. /// public class SourceGenParityTests @@ -20,7 +19,7 @@ public class SourceGenParityTests [InlineData("Ann abc 10.5 ")] public void Person_GeneratedMatchesReflection(string line) { - bool okR = new FixedWidthParser().TryParse(Bytes(line), Ctx, out var r); + bool okR = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var r); bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); Assert.Equal(okR, okG); @@ -47,7 +46,7 @@ public void Person_ParsesExpectedValues() [InlineData("AB")] public void Product_GeneratedMatchesReflection(string line) { - bool okR = new FixedWidthParser().TryParse(Bytes(line), Ctx, out var r); + bool okR = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var r); bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); Assert.Equal(okR, okG); @@ -61,7 +60,7 @@ public void Product_GeneratedMatchesReflection(string line) [InlineData("notanum ")] public void Measurement_GeneratedMatchesReflection(string line) { - bool okR = new FixedWidthParser().TryParse(Bytes(line), Ctx, out var r); + bool okR = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var r); bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); Assert.Equal(okR, okG); @@ -73,7 +72,7 @@ public void Measurement_GeneratedMatchesReflection(string line) [InlineData("42 short ")] public void TrailingString_GeneratedMatchesReflection(string line) { - bool okR = new FixedWidthParser().TryParse(Bytes(line), Ctx, out var r); + bool okR = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var r); bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); Assert.Equal(okR, okG); @@ -87,7 +86,7 @@ public void TrailingString_GeneratedMatchesReflection(string line) [InlineData("bad ")] public void Decimal_GeneratedMatchesReflection(string line) { - bool okR = new FixedWidthParser().TryParse(Bytes(line), Ctx, out var r); + bool okR = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var r); bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); Assert.Equal(okR, okG); @@ -97,7 +96,7 @@ public void Decimal_GeneratedMatchesReflection(string line) [Fact] public void DecodedParsable_GeneratedMatchesReflection() { - bool okR = new FixedWidthParser().TryParse("hey "u8, Ctx, out var r); + bool okR = FixedWidth.TryParse("hey "u8, FixedWidthLayout.FromAttributes(), Ctx, out var r); bool okG = FixedWidth.TryParse("hey "u8, Ctx, out var g); Assert.True(okR); @@ -113,7 +112,7 @@ public void Latin1_GeneratedMatchesReflection() var context = CtxWith(encoding: Encoding.Latin1); byte[] line = Encoding.Latin1.GetBytes("Joséé 42 "); - Assert.True(new FixedWidthParser().TryParse(line, context, out var r)); + Assert.True(FixedWidth.TryParse(line, FixedWidthLayout.FromAttributes(), context, out var r)); Assert.True(FixedWidth.TryParse(line, context, out var g)); Assert.Equal("Joséé", r.Name); @@ -125,7 +124,7 @@ public void Latin1_GeneratedMatchesReflection() [Fact] public void RefStruct_GeneratedMatchesReflection() { - Assert.True(new FixedWidthParser().TryParse("John Doe 30 6000000.00"u8, Ctx, out var r)); + Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, FixedWidthLayout.FromAttributes(), Ctx, out var r)); Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, Ctx, out var g)); Assert.Equal(r.Name, g.Name); @@ -139,7 +138,7 @@ public void StringPool_InternsAndMatchesReflection() var pool = new StringPool(); var context = CtxWith(pool: pool); - Assert.True(new FixedWidthParser().TryParse("John Doe 30 6000000.00"u8, context, out var r)); + Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, FixedWidthLayout.FromAttributes(), context, out var r)); Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, context, out var g)); Assert.Same(r.Name, g.Name); diff --git a/tests/FixedWidthParser.Tests/TestHelpers.cs b/tests/FixedWidthParser.Tests/TestHelpers.cs index b236964..9723673 100644 --- a/tests/FixedWidthParser.Tests/TestHelpers.cs +++ b/tests/FixedWidthParser.Tests/TestHelpers.cs @@ -1,7 +1,6 @@ using System.Globalization; using System.Text; using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Writers; namespace FixedWidthParser.Tests { @@ -35,32 +34,32 @@ public static string Format(in T model, in FixedWidthContext context = defaul return Encoding.UTF8.GetString(destination[..written]); } - /// Writes a model and returns the line without the trailing newline. - public static string WriteOne(FixedWidthWriter writer, in T model) where T : new() + /// Writes a model through an attribute or fluent layout and returns the line without the trailing newline. + public static string WriteOne(FixedWidthLayout layout, in T model, FixedWidthWriterOptions? options = null) { using var ms = new MemoryStream(); - writer.Write(ms, in model); + FixedWidth.Write(ms, in model, layout, options); return Decode(ms).TrimEnd('\r', '\n'); } - public static async Task WriteOneAsync(FixedWidthWriter writer, T model) where T : new() + public static async Task WriteOneAsync(FixedWidthLayout layout, T model, FixedWidthWriterOptions? options = null) { await using var ms = new MemoryStream(); - await writer.WriteAsync(ms, model, TestContext.Current.CancellationToken).ConfigureAwait(false); + await FixedWidth.WriteAsync(ms, model, layout, options, TestContext.Current.CancellationToken).ConfigureAwait(false); return Decode(ms).TrimEnd('\r', '\n'); } - public static string WriteMany(FixedWidthWriter writer, ReadOnlySpan models) where T : new() + public static string WriteMany(FixedWidthLayout layout, ReadOnlySpan models, FixedWidthWriterOptions? options = null) { using var ms = new MemoryStream(); - writer.WriteMany(ms, models); + FixedWidth.WriteMany(ms, models, layout, options); return Decode(ms); } - public static string WriteMany(FixedWidthWriter writer, IEnumerable models) where T : new() + public static string WriteMany(FixedWidthLayout layout, IEnumerable models, FixedWidthWriterOptions? options = null) { using var ms = new MemoryStream(); - writer.WriteMany(ms, models); + FixedWidth.WriteMany(ms, models, layout, options); return Decode(ms); } diff --git a/tests/FixedWidthParser.Tests/TrimCharTests.cs b/tests/FixedWidthParser.Tests/TrimCharTests.cs index a7ec6ce..4bb90b0 100644 --- a/tests/FixedWidthParser.Tests/TrimCharTests.cs +++ b/tests/FixedWidthParser.Tests/TrimCharTests.cs @@ -1,4 +1,3 @@ -using FixedWidthParser.Parsers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -11,7 +10,7 @@ public class TrimCharTests [InlineData("1****", 1)] public void AsteriskTrim_ParsesUnderlyingValue_OnBothPaths(string line, int expected) { - Assert.True(new FixedWidthParser().TryParse(Bytes(line), Ctx, out var reflected)); + Assert.True(FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var reflected)); Assert.True(FixedWidth.TryParse(Bytes(line), Ctx, out var generated)); Assert.Equal(expected, reflected.Value); @@ -21,7 +20,7 @@ public void AsteriskTrim_ParsesUnderlyingValue_OnBothPaths(string line, int expe [Fact] public void HashTrim_StringColumn_OnBothPaths() { - Assert.True(new FixedWidthParser().TryParse("AB######"u8, Ctx, out var reflected)); + Assert.True(FixedWidth.TryParse("AB######"u8, FixedWidthLayout.FromAttributes(), Ctx, out var reflected)); Assert.True(FixedWidth.TryParse("AB######"u8, Ctx, out var generated)); Assert.Equal("AB", reflected.Code); @@ -31,7 +30,7 @@ public void HashTrim_StringColumn_OnBothPaths() [Fact] public void NonAsciiTrimChar_Throws() { - Assert.Throws(() => new FixedWidthParser()); + Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); } } } diff --git a/tests/FixedWidthParser.Tests/WriterAsyncTests.cs b/tests/FixedWidthParser.Tests/WriterAsyncTests.cs index 9c3cac0..8121612 100644 --- a/tests/FixedWidthParser.Tests/WriterAsyncTests.cs +++ b/tests/FixedWidthParser.Tests/WriterAsyncTests.cs @@ -1,6 +1,5 @@ using System.Text; using FixedWidthParser.Readers; -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -20,13 +19,13 @@ private static PersonModel[] SampleModels() [Fact] public async Task WriteManyAsync_MatchesSyncOutput() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); var models = SampleModels(); - string sync = WriteMany(writer, (IEnumerable)models); + string sync = WriteMany(layout, (IEnumerable)models); await using var ms = new MemoryStream(); - await writer.WriteManyAsync(ms, models, TestContext.Current.CancellationToken); + await FixedWidth.WriteManyAsync(ms, models, layout, null, TestContext.Current.CancellationToken); Assert.Equal(sync, Encoding.UTF8.GetString(ms.ToArray())); } @@ -35,11 +34,11 @@ public async Task WriteManyAsync_MatchesSyncOutput() public async Task WriteManyAsync_LeavesStreamOpen_AndRoundTripsThroughReader() { await using var ms = new MemoryStream(); - await new FixedWidthWriter().WriteManyAsync(ms, SampleModels(), TestContext.Current.CancellationToken); + await FixedWidth.WriteManyAsync(ms, SampleModels(), FixedWidthLayout.FromAttributes(), null, TestContext.Current.CancellationToken); Assert.True(ms.CanRead); ms.Position = 0; - var parsed = new FixedWidthReader().Read(ms, leaveOpen: true).ToList(); + var parsed = FixedWidth.Read(ms, FixedWidthLayout.FromAttributes(), leaveOpen: true).ToList(); Assert.Equal(3, parsed.Count); Assert.Equal("John Doe", parsed[0].Name); @@ -52,20 +51,21 @@ public async Task WriteManyAsync_ManyRecords_SpansSeveralBatches() { // 5,000 records of 27 bytes exceed the 64 KB batch, so the buffer is flushed mid-way. var models = Enumerable.Range(0, 5000).Select(i => new PersonModel { Name = "P" + i, Age = i % 100, Salary = i }).ToList(); - var writer = new FixedWidthWriter(new FixedWidthWriterOptions { NewLine = "\r\n" }); + var writerOptions = new FixedWidthWriterOptions { NewLine = "\r\n" }; + var layout = FixedWidthLayout.FromAttributes(); await using var ms = new MemoryStream(); - await writer.WriteManyAsync(ms, models, TestContext.Current.CancellationToken); + await FixedWidth.WriteManyAsync(ms, models, layout, writerOptions, TestContext.Current.CancellationToken); Assert.Equal(5000 * 27, ms.Length); - Assert.Equal(WriteMany(writer, models), Encoding.UTF8.GetString(ms.ToArray())); + Assert.Equal(WriteMany(layout, models, writerOptions), Encoding.UTF8.GetString(ms.ToArray())); } [Fact] public async Task WriteManyAsync_EmptyCollection_WritesNothing() { await using var ms = new MemoryStream(); - await new FixedWidthWriter().WriteManyAsync(ms, [], TestContext.Current.CancellationToken); + await FixedWidth.WriteManyAsync(ms, [], FixedWidthLayout.FromAttributes(), null, TestContext.Current.CancellationToken); Assert.Equal(0, ms.Length); } diff --git a/tests/FixedWidthParser.Tests/WriterEncodingTests.cs b/tests/FixedWidthParser.Tests/WriterEncodingTests.cs index a703c19..be03828 100644 --- a/tests/FixedWidthParser.Tests/WriterEncodingTests.cs +++ b/tests/FixedWidthParser.Tests/WriterEncodingTests.cs @@ -1,5 +1,4 @@ using System.Text; -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -10,10 +9,11 @@ public class WriterEncodingTests [Fact] public void Latin1_WritesOneBytePerCharacter() { - var writer = new FixedWidthWriter(new FixedWidthWriterOptions { Encoding = Encoding.Latin1, NewLine = "\n" }); + var writerOptions = new FixedWidthWriterOptions { Encoding = Encoding.Latin1, NewLine = "\n" }; + var layout = FixedWidthLayout.FromAttributes(); using var ms = new MemoryStream(); - writer.Write(ms, new NameAgeModel { Name = "José", Age = 7 }); + FixedWidth.Write(ms, new NameAgeModel { Name = "José", Age = 7 }, layout, writerOptions); Assert.Equal(Encoding.Latin1.GetBytes("José 7 \n"), ms.ToArray()); } @@ -22,7 +22,7 @@ public void Latin1_WritesOneBytePerCharacter() public void Utf8_MultiByteCharacters_CountAsBytes() { // "José" is 5 bytes in UTF-8, so one pad byte is left in the 6-byte column. - string line = WriteOne(new FixedWidthWriter(), new NameAgeModel { Name = "José", Age = 7 }); + string line = WriteOne(FixedWidthLayout.FromAttributes(), new NameAgeModel { Name = "José", Age = 7 }); Assert.Equal("José 7 ", line); } @@ -31,7 +31,7 @@ public void Utf8_MultiByteCharacters_CountAsBytes() public void Utf8_Truncation_NeverSplitsACharacter() { // "ABCDEé" needs 7 bytes; the 6-byte column keeps "ABCDE" and pads, instead of half of 'é'. - string line = WriteOne(new FixedWidthWriter(), new NameAgeModel { Name = "ABCDEé", Age = 1 }); + string line = WriteOne(FixedWidthLayout.FromAttributes(), new NameAgeModel { Name = "ABCDEé", Age = 1 }); Assert.Equal("ABCDE 1 ", line); } @@ -40,7 +40,7 @@ public void Utf8_Truncation_NeverSplitsACharacter() public void Utf8_RightAlignedTruncation_KeepsWholeTrailingCharacters() { // Right-aligned width 4 keeps the rightmost whole characters: "éF" is 3 bytes, "DéF" would be 4. - string line = WriteOne(new FixedWidthWriter(), new RightTruncateModel { Code = "ABCDéF" }); + string line = WriteOne(FixedWidthLayout.FromAttributes(), new RightTruncateModel { Code = "ABCDéF" }); Assert.Equal("DéF", line); } @@ -49,12 +49,13 @@ public void Utf8_RightAlignedTruncation_KeepsWholeTrailingCharacters() public void Generated_MatchesReflection_ForLatin1() { var context = CtxWith(encoding: Encoding.Latin1); - var writer = new FixedWidthWriter(new FixedWidthWriterOptions { Encoding = Encoding.Latin1 }); + var writerOptions = new FixedWidthWriterOptions { Encoding = Encoding.Latin1 }; + var layout = FixedWidthLayout.FromAttributes(); Span generated = stackalloc byte[9]; Span reflected = stackalloc byte[9]; Assert.True(FixedWidth.TryFormat(new GenNameAgeModel { Name = "Joséé", Age = 42 }, generated, context, out int g)); - Assert.True(writer.TryFormat(new NameAgeModel { Name = "Joséé", Age = 42 }, reflected, out int r)); + Assert.True(FixedWidth.TryFormat(new NameAgeModel { Name = "Joséé", Age = 42 }, layout, reflected, writerOptions.Context, out int r)); Assert.Equal(9, g); Assert.Equal(9, r); @@ -71,12 +72,13 @@ public void NonAsciiFormattedValue_UsesTheConfiguredEncoding(string encodingName var encoding = Encoding.GetEncoding(encodingName); var culture = (System.Globalization.CultureInfo)Inv.Clone(); culture.NumberFormat.CurrencySymbol = "é"; - var writer = new FixedWidthWriter(new FixedWidthWriterOptions { Encoding = encoding, FormatProvider = culture }); + var writerOptions = new FixedWidthWriterOptions { Encoding = encoding, FormatProvider = culture }; + var layout = FixedWidthLayout.FromAttributes(); Span generated = stackalloc byte[8]; Span reflected = stackalloc byte[8]; Assert.True(FixedWidth.TryFormat(new GenPriceModel { Price = 5m }, generated, CtxWith(culture, encoding: encoding), out _)); - Assert.True(writer.TryFormat(new PriceModel { Price = 5m }, reflected, out _)); + Assert.True(FixedWidth.TryFormat(new PriceModel { Price = 5m }, layout, reflected, writerOptions.Context, out _)); Assert.Equal(encoding.GetBytes(expected), generated.ToArray()); Assert.Equal(encoding.GetBytes(expected), reflected.ToArray()); @@ -85,17 +87,18 @@ public void NonAsciiFormattedValue_UsesTheConfiguredEncoding(string encodingName [Fact] public void NewLine_IsConfigurable() { - var writer = new FixedWidthWriter(new FixedWidthWriterOptions { NewLine = "\r\n" }); + var writerOptions = new FixedWidthWriterOptions { NewLine = "\r\n" }; + var layout = FixedWidthLayout.FromAttributes(); - Assert.Equal("ABC\r\nDEF\r\n", WriteMany(writer, new[] { new CodeModel { Code = "ABC" }, new CodeModel { Code = "DEF" } })); + Assert.Equal("ABC\r\nDEF\r\n", WriteMany(layout, new[] { new CodeModel { Code = "ABC" }, new CodeModel { Code = "DEF" } }, writerOptions)); } [Fact] public void TryFormat_DestinationTooSmall_ReturnsFalse() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - Assert.False(writer.TryFormat(new CodeModel { Code = "ABC" }, new byte[2], out int written)); + Assert.False(FixedWidth.TryFormat(new CodeModel { Code = "ABC" }, layout, new byte[2], Ctx, out int written)); Assert.Equal(0, written); } @@ -103,10 +106,11 @@ public void TryFormat_DestinationTooSmall_ReturnsFalse() public void WriteMany_ManyRecords_SpansSeveralBatches() { var models = Enumerable.Range(0, 20_000).Select(i => new CodeModel { Code = (i % 1000).ToString("D3", Inv) }).ToArray(); - var writer = new FixedWidthWriter(new FixedWidthWriterOptions { NewLine = "\n" }); + var writerOptions = new FixedWidthWriterOptions { NewLine = "\n" }; + var layout = FixedWidthLayout.FromAttributes(); - string fromSpan = WriteMany(writer, models.AsSpan()); - string fromEnumerable = WriteMany(writer, (IEnumerable)models); + string fromSpan = WriteMany(layout, models.AsSpan(), writerOptions); + string fromEnumerable = WriteMany(layout, (IEnumerable)models, writerOptions); Assert.Equal(20_000 * 4, fromSpan.Length); Assert.Equal(fromSpan, fromEnumerable); diff --git a/tests/FixedWidthParser.Tests/WriterFormattingTests.cs b/tests/FixedWidthParser.Tests/WriterFormattingTests.cs index f8ac549..1651cee 100644 --- a/tests/FixedWidthParser.Tests/WriterFormattingTests.cs +++ b/tests/FixedWidthParser.Tests/WriterFormattingTests.cs @@ -1,4 +1,3 @@ -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -11,9 +10,9 @@ public class WriterFormattingTests [Fact] public void Write_RightAligned_PadsOnLeftWithSpaces() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new RightAlignedModel { Value = 30 }); + string line = WriteOne(layout, new RightAlignedModel { Value = 30 }); Assert.Equal(" 30", line); } @@ -21,9 +20,9 @@ public void Write_RightAligned_PadsOnLeftWithSpaces() [Fact] public void Write_ZeroPaddedRightAligned_FillsWithZeros() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new ZeroPaddedModel { Value = 30 }); + string line = WriteOne(layout, new ZeroPaddedModel { Value = 30 }); Assert.Equal("00030", line); } @@ -31,9 +30,9 @@ public void Write_ZeroPaddedRightAligned_FillsWithZeros() [Fact] public void Write_FormatString_IsApplied() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new FormattedModel { Amount = 1234.5 }); + string line = WriteOne(layout, new FormattedModel { Amount = 1234.5 }); // "F2" → "1234.50" (7 chars), left-aligned in 8 → one padding char. Assert.Equal("1234.50 ", line); @@ -42,9 +41,9 @@ public void Write_FormatString_IsApplied() [Fact] public void Write_RightAlignedString_PadsOnLeft() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new RightStringModel { Code = "AB" }); + string line = WriteOne(layout, new RightStringModel { Code = "AB" }); Assert.Equal(" AB", line); } @@ -52,10 +51,10 @@ public void Write_RightAlignedString_PadsOnLeft() [Fact] public void Write_NumericOverflow_ThrowsByDefault() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); var ex = Assert.Throws( - () => WriteOne(writer, new NarrowModel { Value = 12345 })); + () => WriteOne(layout, new NarrowModel { Value = 12345 })); Assert.Contains("Value", ex.Message, StringComparison.Ordinal); } @@ -63,17 +62,17 @@ public void Write_NumericOverflow_ThrowsByDefault() [Fact] public void Write_NumericOverflow_TruncatesWhenOptedIn() { - Assert.Throws(() => new FixedWidthWriter()); + Assert.Throws(() => FixedWidth.WriteMany(Stream.Null, Array.Empty(), FixedWidthLayout.FromAttributes())); } [Fact] public void Write_FormattedValueLargerThanStackBuffer_UsesArrayPoolFallback() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); // 700 chars > 256 (stack buffer) and > 512 (first rented buffer): exercises the // ArrayPool fallback and its grow (size *= 2) path. - string line = WriteOne(writer, new WideValueModel { Value = new RepeatedChar(700) }); + string line = WriteOne(layout, new WideValueModel { Value = new RepeatedChar(700) }); Assert.Equal(new string('X', 700) + new string(' ', 100), line); } @@ -81,10 +80,10 @@ public void Write_FormattedValueLargerThanStackBuffer_UsesArrayPoolFallback() [Fact] public void Write_FormattedValueJustOverStackBuffer_FitsFirstRentedBuffer() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); // 300 chars > 256 (stack) but <= 512: fallback succeeds on the first rented buffer. - string line = WriteOne(writer, new WideValueModel { Value = new RepeatedChar(300) }); + string line = WriteOne(layout, new WideValueModel { Value = new RepeatedChar(300) }); Assert.Equal(new string('X', 300) + new string(' ', 500), line); } @@ -92,13 +91,13 @@ public void Write_FormattedValueJustOverStackBuffer_FitsFirstRentedBuffer() [Fact] public void Write_ValueThatNeverFits_ThrowsClearError() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); // 1<<21 chars exceeds the formatter's grow ceiling (1<<20), so TryFormat never succeeds. // The grow loop must fail fast with a clear, named error rather than overflowing its size // counter into a negative/huge ArrayPool rent. var ex = Assert.Throws( - () => WriteOne(writer, new WideValueModel { Value = new RepeatedChar(1 << 21) })); + () => WriteOne(layout, new WideValueModel { Value = new RepeatedChar(1 << 21) })); Assert.Contains(nameof(WideValueModel.Value), ex.Message, StringComparison.Ordinal); Assert.Contains("could not be formatted", ex.Message, StringComparison.Ordinal); @@ -107,10 +106,10 @@ public void Write_ValueThatNeverFits_ThrowsClearError() [Fact] public void Write_RightAlignedOverflow_TruncatesKeepingRightmostChars() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); // "ABCDEF" (6) in a right-aligned width-4 column → keeps the last 4 chars. - string line = WriteOne(writer, new RightTruncateModel { Code = "ABCDEF" }); + string line = WriteOne(layout, new RightTruncateModel { Code = "ABCDEF" }); Assert.Equal("CDEF", line); } @@ -119,9 +118,9 @@ public void Write_RightAlignedOverflow_TruncatesKeepingRightmostChars() public void Write_StringOverflow_TruncatesByDefault() { // Default string behavior preserved (PersonModel.Name has width 10). - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new PersonModel { Name = "VeryLongName", Age = 1, Salary = 0 }); + string line = WriteOne(layout, new PersonModel { Name = "VeryLongName", Age = 1, Salary = 0 }); Assert.Equal("VeryLongNa", line[..10]); } diff --git a/tests/FixedWidthParser.Tests/WriterTests.cs b/tests/FixedWidthParser.Tests/WriterTests.cs index 5de4c73..feaf3f7 100644 --- a/tests/FixedWidthParser.Tests/WriterTests.cs +++ b/tests/FixedWidthParser.Tests/WriterTests.cs @@ -1,4 +1,3 @@ -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -8,9 +7,9 @@ public class WriterTests [Fact] public void Write_ProducesFixedWidthLayout() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new PersonModel { Name = "John Doe", Age = 30, Salary = 60000 }); + string line = WriteOne(layout, new PersonModel { Name = "John Doe", Age = 30, Salary = 60000 }); Assert.Equal(25, line.Length); Assert.Equal("John Doe ", line[..10]); @@ -21,9 +20,9 @@ public void Write_ProducesFixedWidthLayout() [Fact] public void Write_StringLongerThanColumn_Truncates() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new PersonModel { Name = "VeryLongName", Age = 1, Salary = 0 }); + string line = WriteOne(layout, new PersonModel { Name = "VeryLongName", Age = 1, Salary = 0 }); Assert.Equal("VeryLongNa", line[..10]); } @@ -31,9 +30,9 @@ public void Write_StringLongerThanColumn_Truncates() [Fact] public void Write_EmptyString_PadsWithSpaces() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new PersonModel { Name = string.Empty, Age = 1, Salary = 0 }); + string line = WriteOne(layout, new PersonModel { Name = string.Empty, Age = 1, Salary = 0 }); Assert.Equal(" ", line[..10]); } @@ -41,9 +40,9 @@ public void Write_EmptyString_PadsWithSpaces() [Fact] public void Write_FieldBasedModel_Writes() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new ProductModel { Code = "AB", Quantity = 7 }); + string line = WriteOne(layout, new ProductModel { Code = "AB", Quantity = 7 }); Assert.Equal("AB ", line[..5]); Assert.Equal("7 ", line[5..9]); @@ -52,15 +51,15 @@ public void Write_FieldBasedModel_Writes() [Fact] public void WriteMany_SpanAndEnumerable_ProduceSameOutput() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); var models = new[] { new PersonModel { Name = "Alice", Age = 25, Salary = 100 }, new PersonModel { Name = "Bob", Age = 40, Salary = 200 }, }; - string spanOutput = WriteMany(writer, models.AsSpan()); - string enumerableOutput = WriteMany(writer, (IEnumerable)models); + string spanOutput = WriteMany(layout, models.AsSpan()); + string enumerableOutput = WriteMany(layout, (IEnumerable)models); Assert.Equal(enumerableOutput, spanOutput); } @@ -68,13 +67,13 @@ public void WriteMany_SpanAndEnumerable_ProduceSameOutput() [Fact] public async Task WriteAsync_MatchesSyncOutput() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); var model = new PersonModel { Name = "Alice", Age = 25, Salary = 100 }; #pragma warning disable S6966 // Awaitable method should be used - string sync = WriteOne(writer, model); + string sync = WriteOne(layout, model); #pragma warning restore S6966 // Awaitable method should be used - string asyncResult = await WriteOneAsync(writer, model).ConfigureAwait(true); + string asyncResult = await WriteOneAsync(layout, model).ConfigureAwait(true); Assert.Equal(sync, asyncResult); } @@ -84,9 +83,9 @@ public async Task WriteAsync_MatchesSyncOutput() [Fact] public void Write_LineLongerThan1024_UsesArrayPoolPath() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new WideLineModel { Name = "Wide", Number = 42 }); + string line = WriteOne(layout, new WideLineModel { Name = "Wide", Number = 42 }); Assert.Equal(WideLineModel.LineLength, line.Length); Assert.Equal("Wide", line[..1200].TrimEnd()); @@ -96,14 +95,14 @@ public void Write_LineLongerThan1024_UsesArrayPoolPath() [Fact] public void WriteMany_Span_LineLongerThan1024_UsesArrayPoolPath() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); var models = new[] { new WideLineModel { Name = "Alice", Number = 1 }, new WideLineModel { Name = "Bob", Number = 2 }, }; - string output = WriteMany(writer, models.AsSpan()); + string output = WriteMany(layout, models.AsSpan()); var lines = output.Split('\n', StringSplitOptions.RemoveEmptyEntries) .Select(l => l.TrimEnd('\r')) @@ -119,15 +118,15 @@ public void WriteMany_Span_LineLongerThan1024_UsesArrayPoolPath() [Fact] public void WriteMany_SpanAndEnumerable_LineLongerThan1024_ProduceSameOutput() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); var models = new[] { new WideLineModel { Name = "Alice", Number = 1 }, new WideLineModel { Name = "Bob", Number = 2 }, }; - string spanOutput = WriteMany(writer, models.AsSpan()); - string enumerableOutput = WriteMany(writer, (IEnumerable)models); + string spanOutput = WriteMany(layout, models.AsSpan()); + string enumerableOutput = WriteMany(layout, (IEnumerable)models); Assert.Equal(enumerableOutput, spanOutput); } From 02da8afe0a2ed94ede309e7ddce1b4e678f66335 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 23 Oct 2026 22:44:55 -0300 Subject: [PATCH 22/85] test: publish a fluent-layout app with NativeAOT and record the read/write timings tests/AotSmoke builds a layout with FixedWidthLayout.Build and round-trips 1,000 records through FixedWidth; publishing it with NativeAOT must report no trimming or AOT warning. docs/fluent-layout.md compares reading and writing through layouts with the removed classes: every figure is within noise. --- FixedWidthParser.slnx | 1 + docs/README.md | 2 ++ docs/fluent-layout.md | 36 ++++++++++++++++++++++++++++++++++ tests/AotSmoke/AotSmoke.csproj | 24 +++++++++++++++++++++++ tests/AotSmoke/Order.cs | 10 ++++++++++ tests/AotSmoke/Program.cs | 29 +++++++++++++++++++++++++++ 6 files changed, 102 insertions(+) create mode 100644 docs/fluent-layout.md create mode 100644 tests/AotSmoke/AotSmoke.csproj create mode 100644 tests/AotSmoke/Order.cs create mode 100644 tests/AotSmoke/Program.cs diff --git a/FixedWidthParser.slnx b/FixedWidthParser.slnx index 3d9b6e9..ed74dba 100644 --- a/FixedWidthParser.slnx +++ b/FixedWidthParser.slnx @@ -4,6 +4,7 @@ + diff --git a/docs/README.md b/docs/README.md index b393c73..f8b95f4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,8 @@ a constraint; the numbers that justified it live here, so they can be re-run and scale with the number of workers. - [pipe-reader.md](pipe-reader.md) — why a `PipeReader` is read through `AsStream()` rather than its own enumerator. +- [fluent-layout.md](fluent-layout.md) — reading and writing through a `FixedWidthLayout` against + the classes it replaced. Usage documentation is the [README](../README.md); the shape of the code is [ARCHITECTURE.md](../ARCHITECTURE.md). diff --git a/docs/fluent-layout.md b/docs/fluent-layout.md new file mode 100644 index 0000000..9f85b70 --- /dev/null +++ b/docs/fluent-layout.md @@ -0,0 +1,36 @@ +# Moving the runtime path onto `FixedWidthLayout` + +2.0 replaced `FixedWidthParser`, `FixedWidthReader` and `FixedWidthWriter` with one layout +object passed to the `FixedWidth` methods. The parsers and formatters moved from `static readonly` +fields of a generic class, built in a static constructor, to fields of a layout instance, and writing +moved into `RecordWriter`, generic over a format strategy. This page checks that neither move cost +anything measurable, and records what generated models gained. + +## Machine and method + +AMD Ryzen 7 5700X (8C/16T), 32 GB, Windows 10 Pro 19045, .NET SDK 11.0.100-rc.2, `net10.0`, Release. +A Stopwatch console app (not BenchmarkDotNet) over a model of three columns (`string` 10, `int` 5, +`double` 10, lines of 25 bytes plus a line break): + +- **read**: `ReadFile` over a 1,000,000-line file, summing one column; +- **write**: `WriteMany` of 1,000,000 records to `Stream.Null`. + +Each figure is the best of five runs after one warm-up run. The baseline is `d36b958` (the old +classes); the current build calls `FixedWidth.*(…, FixedWidthLayout.FromAttributes(), …)`. The two +programs ran alternately, twice each; both rounds are shown. + +## Result + +| Operation | `d36b958` | Layout API | +|---|--:|--:| +| read, attribute model | 66.9 / 64.4 ms | 64.6 / 67.3 ms | +| read, generated model | 66.8 / 66.6 ms | 66.6 / 69.7 ms | +| write, attribute model | 60.5 / 58.0 ms | 60.8 / 60.5 ms | +| write, generated model | — (no stream writer) | 47.1 / 45.9 ms | + +Every pair sits within the run-to-run spread of the same build (up to 4.7 %, on the generated read, +whose code did not change). The layout instance costs nothing measurable over the static fields, and +batching through `RecordWriter` matches the old writer. + +Generated models can now write to a stream. They do so about a quarter faster than attribute models, +since the generated `TryFormat` has no per-column delegate or interface call. diff --git a/tests/AotSmoke/AotSmoke.csproj b/tests/AotSmoke/AotSmoke.csproj new file mode 100644 index 0000000..9322536 --- /dev/null +++ b/tests/AotSmoke/AotSmoke.csproj @@ -0,0 +1,24 @@ + + + + + Exe + net10.0 + enable + enable + true + true + false + true + false + + $(NoWarn);CA2007;MA0004 + + + + + + + diff --git a/tests/AotSmoke/Order.cs b/tests/AotSmoke/Order.cs new file mode 100644 index 0000000..7f6f719 --- /dev/null +++ b/tests/AotSmoke/Order.cs @@ -0,0 +1,10 @@ +namespace AotSmoke +{ + internal sealed record class Order + { + public int Id { get; set; } + public string Name { get; set; } = ""; + public decimal Amount { get; set; } + public DateOnly Date { get; set; } + } +} diff --git a/tests/AotSmoke/Program.cs b/tests/AotSmoke/Program.cs new file mode 100644 index 0000000..f947ca6 --- /dev/null +++ b/tests/AotSmoke/Program.cs @@ -0,0 +1,29 @@ +using AotSmoke; +using FixedWidthParser; +using FixedWidthParser.Attributes; + +var layout = FixedWidthLayout.Build(b => b + .Factory(() => new Order()) + .Column("Id", 0, 6, (in Order o) => o.Id, (ref Order o, int v) => o.Id = v, + new FixedWidthColumnOptions { Alignment = Alignment.Right, Padding = '0' }) + .Column("Name", 6, 20, (in Order o) => o.Name, (ref Order o, string v) => o.Name = v) + .Column("Amount", 26, 12, (in Order o) => o.Amount, (ref Order o, decimal v) => o.Amount = v) + .DecodedColumn("Date", 38, 8, (in Order o) => o.Date, (ref Order o, DateOnly v) => o.Date = v, + new FixedWidthColumnOptions { Format = "yyyyMMdd" })); + +var orders = Enumerable.Range(1, 1000) + .Select(i => new Order { Id = i, Name = "Order " + i, Amount = i * 1.25m, Date = new DateOnly(2026, 1, 1).AddDays(i) }) + .ToList(); + +await using var stream = new MemoryStream(); +await FixedWidth.WriteManyAsync(stream, orders, layout); +stream.Position = 0; +var read = await FixedWidth.ReadAsync(stream, layout, leaveOpen: true).ToListAsync(); + +if (!orders.SequenceEqual(read)) +{ + await Console.Error.WriteLineAsync("Round trip mismatch."); + return 1; +} +await Console.Out.WriteLineAsync($"Round-tripped {read.Count} orders."); +return 0; From 9b017fc107fdfd5df164a0c3baaa9ff07285e5f0 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 23 Oct 2026 22:48:48 -0300 Subject: [PATCH 23/85] docs: document FixedWidthLayout and the single FixedWidth API README shows reading and writing through FixedWidth with a layout and adds a section on fluent layouts. ARCHITECTURE describes the layout object, its two sources and RecordWriter. STYLEGUIDE and CHANGELOG follow the removal of the runtime classes. --- ARCHITECTURE.md | 84 +++++++++++++++++++++++++++------------------- CHANGELOG.md | 35 +++++++++++++------- README.md | 88 ++++++++++++++++++++++++++++++------------------- STYLEGUIDE.md | 26 ++++++++------- 4 files changed, 142 insertions(+), 91 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index cc2c126..3dfacbb 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -10,14 +10,14 @@ and [`STYLEGUIDE.md`](STYLEGUIDE.md#comments) says why. ## Two paths over one set of helpers -Every model is parsed and written by one of two paths: +Every model is parsed and written by one of two paths, both behind the static `FixedWidth` class +(`src/FixedWidthParser/FixedWidth.cs`), where each method has one overload per path: -| | Runtime (reflection-built) | Generated | +| | Layout | Generated | |---|---|---| -| Model | any class or struct with `[FixedColumn]` members | a `partial` type implementing `IFixedWidthModel` | -| One line | `FixedWidthParser.TryParse`, `FixedWidthWriter.TryFormat` | `FixedWidth.TryParse`/`TryFormat`, calling the generated statics | -| Many lines | `FixedWidthReader`, `FixedWidthWriter` | `FixedWidth.Read*`; writing goes through `TryFormat` | -| Built | once per type, in a static constructor, from compiled expression trees | at compile time by `FixedWidthModelGenerator` | +| Model | any type, through a `FixedWidthLayout` from `[FixedColumn]` attributes or from `FixedWidthLayout.Build` | a `partial` type implementing `IFixedWidthModel` | +| Call | `FixedWidth.X(…, layout, …)` | `FixedWidth.X(…)`, calling the generated statics | +| Built | when the layout is built: by reflection and compiled expression trees for attributes, from delegates for `Build` | at compile time by `FixedWidthModelGenerator` | Both paths call the same helpers in `FixedWidthRuntime` (`src/FixedWidthParser/FixedWidthRuntime.cs`) for every column: trim, decode a string, parse a number, format a value, pad and align. That class is @@ -30,7 +30,7 @@ characters alone, is decoded with the configured encoding. ## Settings: options and the context -Readers take a `FixedWidthReaderOptions` and the writer a `FixedWidthWriterOptions`. Both reduce to a +Reads take a `FixedWidthReaderOptions` and writes a `FixedWidthWriterOptions`, validated on each call. Both reduce to a `FixedWidthContext` (`src/FixedWidthParser/FixedWidthContext.cs`), a `readonly record struct` passed by `in` to every parse and format call, generated or not, and to every `IFixedWidthConverter`. @@ -39,28 +39,41 @@ implemented by users, so a minor version cannot add a member to them; a new sett property on the context instead. The context also caches what the hot path would otherwise recompute per column, such as whether the encoding is UTF-8 and the provider's decimal separator. -## Building a runtime layout - -`ModelColumns.ForEachColumn` (`src/FixedWidthParser/Attributes/`) walks the `[FixedColumn]` properties, -then fields, and hands each to a visitor; `ColumnLayoutValidator` then rejects a negative start, a -non-positive length and overlapping columns. The parser and the writer share this walk. - -- **Parser.** `ParserBuilder` compiles a model factory and, per column, a setter delegate. - `ColumnParserFactory` wraps each setter with the parse it needs into one `ColumnParser` - closure. Its resolution order — converter, date/time with a `Format`, `string`, `double`/`float`, - `IUtf8SpanParsable`, then `ISpanParsable` over decoded text — is the order the generator's - `Classify` mirrors. -- **Writer.** `FixedWidthWriter` compiles a getter per column and wraps it in an - `IColumnFormatter` (`Formatters/ColumnFormatters.cs`): string, `IUtf8SpanFormattable`, - `ISpanFormattable`, converter, and a nullable wrapper over any of them. `ClassifyWrite` in the - generator mirrors that choice. - -The build runs once per closed generic type in a static constructor. A layout error is captured there -and rethrown from the instance constructor, so the caller sees an `InvalidOperationException` rather -than a `TypeInitializationException` on first use. - -`FixedWidthParser` accepts `ref struct` models (`allows ref struct`); the readers do not, since an -enumerator has to hold its current model in a field. +## The layout object + +A `FixedWidthLayout` (`src/FixedWidthParser/Layouts/`) holds, per column, a `ColumnParserInfo` +(start, length, name and a `ColumnParser` closure that parses and assigns) and an +`IColumnFormatter` (`Formatters/ColumnFormatters.cs`), plus the line length and the model factory. +It is immutable, so one instance serves every read, write and worker thread. A layout that lacks the +parsers or the formatters reads or writes nothing; `EnsureCanRead`/`EnsureCanWrite` throw before the +first line. + +Two sources fill it, and both reach the same parse and format closures: + +- **`FromAttributes()`.** `ModelColumns.ForEachColumn` (`Attributes/`) walks the `[FixedColumn]` + properties, then fields; `ColumnLayoutValidator` rejects a negative start, a non-positive length and + overlapping columns. `ParserBuilder` compiles a model factory and a setter per column, and + `ColumnParserFactory` picks the parse for the member's runtime `Type` through `MakeGenericMethod`. + `FormatterBuilder` compiles a getter per column and picks its formatter the same way. The result is + cached once per `T` in a static constructor. The read side and the write side build independently: + a model only one side can handle (a column type that formats but does not parse, say) still gets a + layout, and the other side's error is rethrown on its first use. Only an error both sides hit, such + as an overlap, throws from `FromAttributes`. +- **`Build(configure)`.** `FixedWidthLayoutBuilder` receives a `ColumnGetter`/`ColumnSetter` + pair per column, already typed, so it selects the parse and the formatter by generic constraints + (`Column` for UTF-8 parsable types, `DecodedColumn`, `NullableColumn`, `ConvertedColumn`) + and `typeof(T) ==` checks the JIT folds. No reflection: this is the trimming- and AOT-safe source, + which `tests/AotSmoke` publishes with NativeAOT to prove. + +`ColumnParserFactory`'s resolution order — converter, date/time with a `Format`, `string`, +`double`/`float`, `IUtf8SpanParsable`, then `ISpanParsable` over decoded text — is shared by both +sources (`TryBuildSpecial`) and mirrored by the generator's `Classify`; the formatter choice (converter, +string, `IUtf8SpanFormattable`, `ISpanFormattable`, a nullable wrapper) is mirrored by `ClassifyWrite`. +Column options are validated by `FormatterBuilder.ResolveOptions` for both. + +A layout accepts `ref struct` models (`allows ref struct`) for `TryParse` and `TryFormat`; the readers +do not, since an enumerator has to hold its current model in a field, and an attribute layout over a +`ref struct` does not write. ## Reading @@ -71,7 +84,7 @@ source -> line splitter -> IRecordLineParser -> model ``` Two enumerators share one shape, each specialized by a `struct` parse strategy -(`IRecordLineParser`: `ReflectionLineParser` or `GeneratedLineParser`, in +(`IRecordLineParser`: `LayoutLineParser` or `GeneratedLineParser`, in `Readers/ILineParser.cs`). Because the strategy is a `struct` type argument, the JIT compiles one enumerator per `(TModel, TParser)` pair and the parse call devirtualizes. @@ -142,10 +155,13 @@ result is a `FixedWidthParseException` naming the line, column, position and raw ## Writing -`FixedWidthWriter` formats each record into a fixed-length slot: the line, then the encoded -`NewLine`. `WriteMany` and `WriteManyAsync` fill one pooled 64 KB batch with whole records and write it -to the stream when the next record would not fit, so a write allocates nothing per record. Each line is -filled with spaces first, then every column formats into its own slice. +`RecordWriter` (`Writers/RecordWriter.cs`) writes for both paths, specialized like the readers by a +`struct` strategy (`IRecordFormatter`: `LayoutRecordFormatter` or `GeneratedRecordFormatter`, +in `Writers/RecordFormatters.cs`). Each record is formatted, followed by the encoded `NewLine`, into one +pooled 64 KB batch. A generated model does not declare its line length, so the batch does not reserve +a slot: a record that does not fit the rest of the batch flushes it, and one that does not fit an empty +batch grows it. A write allocates nothing per record. A layout line is filled with spaces first, then +every column formats into its own slice. Per column, `FixedWidthRuntime` picks the cheapest route that keeps the bytes the encoding would produce: diff --git a/CHANGELOG.md b/CHANGELOG.md index d20a235..6b6ca46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Not released yet. -2.0 reads and writes **bytes only**: one parser, one reader, one writer and one generated -`TryParse`/`TryFormat`, where 1.x had a `char` and a UTF-8 version of each. Column positions are -bytes; text columns are decoded with an ASCII-compatible encoding, UTF-8 by default. Settings travel +2.0 reads and writes **bytes only**, where 1.x had a `char` and a UTF-8 version of each reader, +parser and writer, and goes through **one entry point**: every `FixedWidth` method comes in an +overload for generated models and one that takes a `FixedWidthLayout`, built from the attributes or +fluently in code. The 1.x runtime classes are gone. Column positions are bytes; text columns are decoded with an ASCII-compatible encoding, UTF-8 by default. Settings travel in options objects and a `FixedWidthContext`, so later versions can add to them without another break. 2.0 targets `net10.0` only. @@ -19,16 +20,16 @@ in options objects and a `FixedWidthContext`, so later versions can add to them | 1.x | 2.0 | |---|---| -| `FixedWidthReader` over a `TextReader`, `FixedWidthByteReader` | `FixedWidthReader` over a `Stream` or a file | +| `FixedWidthReader` over a `TextReader`, `FixedWidthByteReader` | `FixedWidth.Read(stream, FixedWidthLayout.FromAttributes(), …)` or `FixedWidth.ReadFile(path, FixedWidthLayout.FromAttributes(), …)` | | `FixedWidthUtf8.Read*`, `FixedWidth.Read*` over a `TextReader` | `FixedWidth.Read*` over a `Stream` or a file | -| `new FixedWidthReader(provider, pool, bufferSize)` | `new FixedWidthReader(new FixedWidthReaderOptions { FormatProvider = …, StringPool = …, BufferSize = … })` | +| `new FixedWidthReader(provider, pool, bufferSize).Read(…)` | `FixedWidth.Read(…, FixedWidthLayout.FromAttributes(), options: new FixedWidthReaderOptions { FormatProvider = …, StringPool = …, BufferSize = … })` | | `FixedWidth.Read(stream, encoding, leaveOpen, provider, pool, bufferSize)` | `FixedWidth.Read(stream, leaveOpen, options)`; the encoding is `FixedWidthReaderOptions.Encoding` | -| `FixedWidthParser`, `Utf8FixedWidthParser` `.TryParse(line, provider, pool, out m)` | `FixedWidthParser.TryParse(ReadOnlySpan line, in FixedWidthContext context, out m)` | +| `FixedWidthParser`, `Utf8FixedWidthParser` `.TryParse(line, provider, pool, out m)` | `FixedWidth.TryParse(ReadOnlySpan line, FixedWidthLayout.FromAttributes(), in FixedWidthContext context, out m)` | | `FixedWidth.TryParse(chars, provider, pool, out m)`, `FixedWidthUtf8.TryParse(bytes, …)` | `FixedWidth.TryParse(bytes, in FixedWidthContext context, out m)` | | `FixedWidth.TryFormat(in m, Span, provider, out n)` | `FixedWidth.TryFormat(in m, Span, in FixedWidthContext context, out n)` | | `IFixedWidthModel` + `IUtf8FixedWidthModel` | `IFixedWidthModel`, whose generated members take bytes and a context | | `IFixedWidthConverter` + `IUtf8FixedWidthConverter` with `IFormatProvider?` | `IFixedWidthConverter` over bytes, with `in FixedWidthContext` | -| `FixedWidthWriter.Write*(StreamWriter or Stream, …, provider)` | `new FixedWidthWriter(new FixedWidthWriterOptions { FormatProvider = …, Encoding = …, NewLine = … })`, then `Write*(Stream, …)` | +| `FixedWidthWriter.Write*(StreamWriter or Stream, …, provider)` | `FixedWidth.Write*(Stream, …, FixedWidthLayout.FromAttributes(), new FixedWidthWriterOptions { FormatProvider = …, Encoding = …, NewLine = … })` | | `catch (FormatException)` on a bad line | still works; the exception is now `FixedWidthParseException` | A UTF-8 file with multi-byte text whose layout counts characters no longer lines up: positions are @@ -36,8 +37,15 @@ bytes. Read files written in a single-byte encoding with that encoding (`Encodin Windows-1252 after registering `CodePagesEncodingProvider`). ### Added -- Parallel aggregation: `FixedWidthReader.AggregateFileAsync` and `FixedWidth.AggregateFileAsync` - read a file in byte-range partitions on several threads and fold each into an +- `FixedWidthLayout`: how a model maps to the columns of a line, for reading and writing, passed to + the `FixedWidth` methods. `FixedWidthLayout.FromAttributes()` builds it from `[FixedColumn]` + attributes; `FixedWidthLayout.Build(b => …)` describes it in code, binding each column with a + `ColumnGetter`/`ColumnSetter` delegate and `FixedWidthColumnOptions`, with no reflection, so it works + under trimming and NativeAOT. A layout can read only or write only. +- `FixedWidth.Write`, `WriteMany`, `WriteAsync` and `WriteManyAsync` for generated models, which had no + stream writer. +- Parallel aggregation: `FixedWidth.AggregateFileAsync`, for generated models and layouts alike, + reads a file in byte-range partitions on several threads and fold each into an `IRecordAccumulator`, merged in file order. Every line is parsed exactly once, and errors carry the same line number as a sequential read. `FixedWidthParallelOptions.DegreeOfParallelism` sets the thread count. `AggregateAsync` does the same over a `Stream`: one thread @@ -58,19 +66,19 @@ Windows-1252 after registering `CodePagesEncodingProvider`). from characters. A UTF-8 byte order mark is skipped only when the encoding is UTF-8. - Columns of any `ISpanParsable` type without a UTF-8 parser, decoded first (they used to be refused on the byte path). -- `FixedWidthWriter.TryFormat(in model, Span, out int)`. +- `FixedWidth.TryFormat(in model, layout, Span, in FixedWidthContext, out int)`. - `FixedColumnAttribute.Converter`: a per-property `IFixedWidthConverter` that takes priority over the built-in parsing and formatting, with diagnostic `FWP008` when it targets the wrong type. - Nullable value-type columns (`int?`, `decimal?`, `DateTime?`, …): a blank column parses to `null` without invoking the parser or converter, and `null` writes as a blank column. - The source-generated writer: `IFixedWidthModel.TryFormat`, with the same alignment, padding, - format, overflow, nullable and converter handling as `FixedWidthWriter`, and diagnostic `FWP010` + format, overflow, nullable and converter handling as the model's attribute layout, and diagnostic `FWP010` when a column's type cannot be written. - `FixedColumnAttribute.TrimChar` and `TrimMode`: what to strip from a column when parsing, and from which side. - `FixedColumnAttribute.Format` parses `DateTime`, `DateOnly`, `TimeOnly` and `DateTimeOffset` with that exact format. -- Diagnostic `FWP012`: `Padding` or `TrimChar` is not ASCII. The runtime parser and writer throw +- Diagnostic `FWP012`: `Padding` or `TrimChar` is not ASCII. Layouts throw `InvalidOperationException` for the same models. - Public API tracking (`PublicAPI.*.txt`) and package validation against the last release, run by CI on every pull request. @@ -96,6 +104,9 @@ Windows-1252 after registering `CodePagesEncodingProvider`). - The library and generator build with warnings as errors. ### Removed +- **Breaking:** `FixedWidthParser`, `FixedWidthReader` and `FixedWidthWriter`. Pass + `FixedWidthLayout.FromAttributes()` to the matching `FixedWidth` method instead (see the table above); the layout is + built once per type and shared. - **Breaking:** reading from a `PipeReader` (`ReadAsync(PipeReader)` and its enumerator types). Pass `pipeReader.AsStream()` to the `Stream` overloads instead; it reads twice as fast as the dedicated path did ([docs/pipe-reader.md](docs/pipe-reader.md)). diff --git a/README.md b/README.md index 1265e1c..ffffe74 100644 --- a/README.md +++ b/README.md @@ -8,18 +8,18 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) [![Benchmarks](https://img.shields.io/badge/benchmarks-GitHub%20Pages-informational)](https://gabrielmarquezmatte.github.io/FixedWidthParser/dev/bench/) -A high-performance, low-allocation library for **parsing and writing fixed-width (flat) files** in .NET. Columns are declared with attributes, layouts are validated up front, and lines are parsed straight from the file's bytes, without a string per line. +A high-performance, low-allocation library for **parsing and writing fixed-width (flat) files** in .NET. Columns are declared with attributes or described in code, layouts are validated up front, and lines are parsed straight from the file's bytes, without a string per line. -The package includes both: +Everything goes through one static class, `FixedWidth`, whose every method comes in two overloads: -- a runtime parser, reader and writer for regular attribute-mapped models; and -- a bundled Roslyn source generator for reflection-free `TryParse` and `TryFormat` implementations. +- one for models the bundled Roslyn source generator implements (`IFixedWidthModel`): reflection-free and statically dispatched; and +- one that takes a `FixedWidthLayout`, built from `[FixedColumn]` attributes or fluently in code. Upgrading from 1.x? See [the 2.0 entry in the changelog](CHANGELOG.md#200). ## Features -- **Attribute-driven** column mapping (`[FixedColumn(start, length)]`) on properties and public fields. +- **Attribute-driven** column mapping (`[FixedColumn(start, length)]`) on properties and public fields, or a **fluent layout** bound with delegates, for types you cannot annotate. Fluent layouts are trimming- and Native AOT-safe. - **Byte positions.** Lines are read and written as bytes, with no transcoding to UTF-16. Text columns are decoded with the file's encoding: UTF-8 by default, or any ASCII-compatible encoding such as Latin-1 or Windows-1252. - **Lazy reading** from a `Stream` or a file, synchronous (`IEnumerable` with a struct enumerator) and asynchronous (`IAsyncEnumerable`). - **Source-generated parsing and writing** for `partial` models implementing `IFixedWidthModel`. @@ -28,8 +28,8 @@ Upgrading from 1.x? See [the 2.0 entry in the changelog](CHANGELOG.md#200). - **Writing** of single records and batches, synchronous and asynchronous, batched into one pooled buffer. - **Configurable formatting** per column: alignment, padding character, format string and explicit overflow policy. - **Culture-aware** numbers and dates, including `double`/`float` via csFastFloat and any `IUtf8SpanParsable`/`ISpanParsable` type. -- **Layout validation** at construction or generation time: negative `Start`, non-positive `Length`, overlapping columns and non-ASCII padding fail clearly. -- **`ref struct` model support** on single-line parsing. +- **Layout validation** when a layout is built or generated: negative `Start`, non-positive `Length`, overlapping columns and non-ASCII padding fail clearly. +- **`ref struct` model support** on single-line parsing and formatting. ## Requirements @@ -47,7 +47,7 @@ The package ships the Roslyn source generator bundled as an analyzer. Models tha ## Defining a Model -Runtime/reflection models only need a public parameterless constructor and mapped fields or properties: +An attribute model needs a public parameterless constructor and mapped fields or properties; `FixedWidthLayout.FromAttributes()` turns it into a layout, built once per type by reflection: ```csharp using FixedWidthParser.Attributes; @@ -81,38 +81,61 @@ public readonly partial record struct GeneratedPerson : IFixedWidthModel(); +```csharp +using FixedWidthParser; +using FixedWidthParser.Attributes; -foreach (var person in reader.ReadFile("people.txt")) -{ - // ... -} +var layout = FixedWidthLayout.Build(b => b + .Factory(() => new Order()) + .Column("Id", 0, 6, (in Order o) => o.Id, (ref Order o, int v) => o.Id = v, + new FixedWidthColumnOptions { Alignment = Alignment.Right, Padding = '0' }) + .Column("Name", 6, 20, (in Order o) => o.Name, (ref Order o, string v) => o.Name = v) + .NullableColumn("Discount", 26, 10, (in Order o) => o.Discount, (ref Order o, decimal? v) => o.Discount = v) + .DecodedColumn("Date", 36, 8, (in Order o) => o.Date, (ref Order o, DateOnly v) => o.Date = v, + new FixedWidthColumnOptions { Format = "yyyyMMdd" }) + .ConvertedColumn("Total", 44, 12, (in Order o) => o.Total, (ref Order o, Money v) => o.Total = v, new MoneyConverter())); ``` -`Read(Stream, leaveOpen)` reads a stream once; `ReadFile` reopens the file on each enumeration. Reading is lazy and reuses a pooled buffer; each line is parsed in place, without a string per line. +| Method | Column type | +|---|---| +| `Column` | `string` | +| `Column` | a type that parses from and formats to UTF-8: numbers, `Guid`, … | +| `DecodedColumn` | a type that parses only from characters (decoded first) | +| `NullableColumn` | `T?` of a `Column` type; a blank column is `null` | +| `ConvertedColumn` | any type, through an `IFixedWidthConverter` instance | -Empty lines (including a trailing newline at end of file) are skipped: they are counted toward the line number but not yielded. Lines end at `\n`, with a preceding `\r` dropped, and a leading UTF-8 byte order mark is skipped. +`FixedWidthColumnOptions` carries the same settings as `[FixedColumn]`, with the same defaults (see [Formatting Options](#formatting-options)). A reference-type model needs `Factory` to be read; a value type without one starts from `default`. Pass `null` as the getter or the setter to build a layout that only reads or only writes; using it the other way throws `InvalidOperationException` before any line is touched. The column name appears in error messages. -The source-generated facade has the same methods: +## Reading ```csharp using FixedWidthParser; +var layout = FixedWidthLayout.FromAttributes(); + +foreach (var person in FixedWidth.ReadFile("people.txt", layout)) +{ + // ... +} + foreach (var person in FixedWidth.ReadFile("people.txt")) { // generated TryParse for each line } ``` +`Read(Stream, …, leaveOpen)` reads a stream once; `ReadFile` reopens the file on each enumeration. Reading is lazy and reuses a pooled buffer; each line is parsed in place, without a string per line. A layout is immutable: build it once and share it across reads and threads. + +Empty lines (including a trailing newline at end of file) are skipped: they are counted toward the line number but not yielded. Lines end at `\n`, with a preceding `\r` dropped, and a leading UTF-8 byte order mark is skipped. + ### Async ```csharp -await foreach (var person in reader.ReadFileAsync("people.txt")) +await foreach (var person in FixedWidth.ReadFileAsync("people.txt", layout)) { // ... } @@ -140,7 +163,7 @@ public sealed class Payroll : IRecordAccumulator } } -var payroll = await reader.AggregateFileAsync("people.txt"); +var payroll = await FixedWidth.AggregateFileAsync("people.txt", layout); // GeneratedPayroll: the same accumulator over GeneratedPerson. var generated = await FixedWidth.AggregateFileAsync("people.txt"); ``` @@ -160,7 +183,7 @@ and the others parse. Every read takes an optional `FixedWidthReaderOptions`: ```csharp -var reader = new FixedWidthReader(new FixedWidthReaderOptions +var options = new FixedWidthReaderOptions { FormatProvider = CultureInfo.GetCultureInfo("pt-BR"), Encoding = Encoding.Latin1, @@ -168,7 +191,8 @@ var reader = new FixedWidthReader(new FixedWidthReaderOptions BufferSize = 64 * 1024, MaxLineLength = 4096, OnParseError = ex => { log.Warning(ex.Message); return true; }, // skip the line -}); +}; +var people = FixedWidth.ReadFile("people.txt", layout, options); ``` | Option | Default | Meaning | @@ -197,20 +221,16 @@ A reader buffers a line until it finds its line break. `MaxLineLength` bounds th ## Single Lines ```csharp -var parser = new FixedWidthParser(); -parser.TryParse("John Doe 30 60000.00 "u8, default, out var person); +FixedWidth.TryParse("John Doe 30 60000.00 "u8, layout, default, out Person person); FixedWidth.TryParse(lineBytes, new FixedWidthContext { FormatProvider = culture }, out var generated); ``` -`FixedWidthContext` carries the format provider, the encoding and the string pool for one line; `default` means invariant culture, UTF-8 and no pool. `FixedWidthReader.TryParse(line, out model)` parses one line with the reader's options. +`FixedWidthContext` carries the format provider, the encoding and the string pool for one line; `default` means invariant culture, UTF-8 and no pool. ## Writing ```csharp -using FixedWidthParser.Writers; - -var writer = new FixedWidthWriter(); var people = new[] { new Person { Name = "John Doe", Age = 30, Salary = 60000 }, @@ -218,12 +238,11 @@ var people = new[] }; using var stream = File.Create("out.txt"); -writer.WriteMany(stream, people.AsSpan()); +FixedWidth.WriteMany(stream, people.AsSpan(), layout); +FixedWidth.WriteMany(stream, generatedPeople); // generated models need no layout ``` -`Write`, `WriteMany` (span or enumerable) and their async versions write each record followed by `NewLine`, batched into one pooled buffer. `FixedWidthWriterOptions` sets the `FormatProvider`, the `Encoding` and the `NewLine` (default `Environment.NewLine`). `TryFormat(in model, Span, out int)` formats one line into your own buffer. - -Generated models format through `FixedWidth.TryFormat(in model, Span, in FixedWidthContext, out int)`. +`Write`, `WriteMany` (span or enumerable) and their async versions write each record followed by `NewLine`, batched into one pooled buffer. `FixedWidthWriterOptions` sets the `FormatProvider`, the `Encoding` and the `NewLine` (default `Environment.NewLine`). `FixedWidth.TryFormat(in model, layout, Span, in FixedWidthContext, out int)`, or its generated overload without the layout, formats one line into your own buffer. ## Formatting Options @@ -260,7 +279,7 @@ On .NET, Windows-1252 and other code pages need `Encoding.RegisterProvider(CodeP ## Validation -Invalid layouts fail fast with an `InvalidOperationException` on the runtime parser/writer, or a generator diagnostic on generated models: negative `Start`, `Length < 1`, overlapping columns, a non-ASCII `Padding` or `TrimChar` (FWP012), an unsupported column type and a converter for the wrong type. Adjacent columns are valid. +Invalid layouts fail fast with an `InvalidOperationException` from `FixedWidthLayout.Build` or `FromAttributes`, or a generator diagnostic on generated models: negative `Start`, `Length < 1`, overlapping columns, a non-ASCII `Padding` or `TrimChar` (FWP012), an unsupported column type and a converter for the wrong type. Adjacent columns are valid. An attribute model that can be read but not written (or the reverse), such as one with a column type that only formats, gets a layout anyway; the side it cannot do throws on first use. ## Versioning @@ -348,6 +367,7 @@ src/FixedWidthParser.Generator/ Roslyn source generator tests/FixedWidthParser.Tests/ Runtime, reader, writer and parity tests tests/FixedWidthParser.Generator.Tests/Source generator tests tests/Benchmarks/ BenchmarkDotNet benchmarks +tests/AotSmoke/ Fluent layout published with NativeAOT docs/ Measurements behind design decisions ``` diff --git a/STYLEGUIDE.md b/STYLEGUIDE.md index 1077816..9bf5c16 100644 --- a/STYLEGUIDE.md +++ b/STYLEGUIDE.md @@ -177,12 +177,11 @@ Common splits: ### The runtime and generated paths are peers -A model is parsed and written either by the runtime path (`FixedWidthParser`, `FixedWidthReader`, -`FixedWidthWriter`) or by the generated one (`FixedWidth.*` over `IFixedWidthModel`). A -capability on one gets the same name shape on the other: `FixedWidth.ReadFile` mirrors -`FixedWidthReader.ReadFile`, and the same options object configures both. A caller who learns one -path should be able to guess the other. When you add a member to one, grep the other and match, or add -the missing sibling in the same change. +A model is parsed and written either through a layout (`FixedWidth.X(…, FixedWidthLayout layout, …)`) +or by the generated path (`FixedWidth.X(…)` over `IFixedWidthModel`). Every `FixedWidth` +operation has both overloads, side by side, with the same parameters in the same order apart from the +layout, and the same options object configures both. A caller who learns one path should be able to +guess the other. When you add an overload for one, add its sibling in the same change. --- @@ -267,9 +266,11 @@ Never answer a malformed file with `ArgumentException`/`ArgumentOutOfRangeExcept caller they made a mistake when the input is what is broken. If file bytes can reach a .NET throw helper, you are missing a validation step upstream. -The runtime path builds its layout once per type in a static constructor. A layout error there is -captured and rethrown from the instance constructor, never left to surface as a -`TypeInitializationException`. +A layout error throws `InvalidOperationException` from `FixedWidthLayout.Build` or `FromAttributes`. +`FromAttributes` builds once per type in a static constructor; an error there is captured and rethrown +from every call, never left to surface as a `TypeInitializationException`. An error only one side of an +attribute layout hits (it can read but not write, or the reverse) is rethrown when that side is first +used, before any line is touched. --- @@ -312,7 +313,7 @@ not a performance nit. Three sets of code paths in this library must stay behaviorally identical, and nothing in the compiler keeps them honest: -- **Runtime and generated.** `FixedWidthParser`/`FixedWidthWriter` and the generator's +- **Layout and generated.** `FixedWidthLayout` (from attributes or `Build`) and the generator's `TryParse`/`TryFormat` call the same `FixedWidthRuntime` helpers. Put shared behavior there, not in either side. - **Sequential and parallel reads.** `AggregateFileAsync` and `AggregateAsync` must return what a @@ -365,4 +366,7 @@ formatting a column): - Optimize against a measurement, and record what it showed in [`docs/`](docs/README.md) — never in a comment. An optimization with no number behind it is a guess that costs readability. -Outside the hot path (building a layout, the static constructors), ordinary allocations and reflection are fine. +Outside the hot path (building a layout), ordinary allocations are fine, and reflection is fine in +`FromAttributes`. `FixedWidthLayout.Build` and the generated overloads must stay free of reflection, +`MakeGenericMethod` and `Expression.Compile`, so they keep working under trimming and NativeAOT; +`tests/AotSmoke` checks it. From c57d36a4837a37ceabf701d736f1781d7492576c Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 23 Oct 2026 22:56:22 -0300 Subject: [PATCH 24/85] test: take the best of three allocation measurements per guard Under the parallel test run, the per-line allocation guard sometimes saw a one-off ~8 KB allocation on its thread during one read (2.02 B/line over 4,000 extra lines); the baseline commit shows the same kind of failure. The guard now keeps the smallest of three measurements: a real per-line allocation still shows in all three, as a deliberately injected 24 B/line one does. --- .../FixedWidthParser.Tests/AllocationGuardTests.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/FixedWidthParser.Tests/AllocationGuardTests.cs b/tests/FixedWidthParser.Tests/AllocationGuardTests.cs index 3c37847..22edd22 100644 --- a/tests/FixedWidthParser.Tests/AllocationGuardTests.cs +++ b/tests/FixedWidthParser.Tests/AllocationGuardTests.cs @@ -99,8 +99,20 @@ private static int SumGenerated(FixedWidthReaderOptions options, byte[] bytes) return total; } - // Bytes allocated per extra line, isolating per-line cost from fixed per-call overhead. + // Bytes allocated per extra line, isolating per-line cost from fixed per-call overhead. The best of + // three attempts: while other test classes run, the runtime can make a one-off allocation on this + // thread during one read, which a per-line regression would not survive in every attempt. private static double MarginalPerLine(Action readSmall, Action readLarge) + { + double best = double.MaxValue; + for (int attempt = 0; attempt < 3; attempt++) + { + best = Math.Min(best, MarginalPerLineOnce(readSmall, readLarge)); + } + return best; + } + + private static double MarginalPerLineOnce(Action readSmall, Action readLarge) { long b1 = GC.GetAllocatedBytesForCurrentThread(); readSmall(); From cff0c3906d171a7e507af76bf53fe6b01db8604b Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 23 Oct 2026 23:03:53 -0300 Subject: [PATCH 25/85] feat(layout): add nullable decoded and nullable converted fluent columns NullableDecodedColumn covers DateOnly?, DateTime? and other types that parse only from characters; NullableConvertedColumn covers T? through an IFixedWidthConverter. Both were possible with attributes and not with the fluent builder. Also adds sync/async parity tests for records longer than a write batch, for generated and layout models. --- README.md | 2 + .../Layouts/FixedWidthLayoutBuilder.cs | 57 +++++++++++++++++++ src/FixedWidthParser/PublicAPI.Unshipped.txt | 2 + .../FluentLayoutTests.cs | 10 +++- .../LayoutWriteTests.cs | 28 +++++++++ tests/FixedWidthParser.Tests/TestModels.cs | 2 + 6 files changed, 99 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ffffe74..086433c 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,9 @@ var layout = FixedWidthLayout.Build(b => b | `Column` | a type that parses from and formats to UTF-8: numbers, `Guid`, … | | `DecodedColumn` | a type that parses only from characters (decoded first) | | `NullableColumn` | `T?` of a `Column` type; a blank column is `null` | +| `NullableDecodedColumn` | `T?` of a `DecodedColumn` type, such as `DateOnly?` | | `ConvertedColumn` | any type, through an `IFixedWidthConverter` instance | +| `NullableConvertedColumn` | `T?` of a value type, through an `IFixedWidthConverter` instance | `FixedWidthColumnOptions` carries the same settings as `[FixedColumn]`, with the same defaults (see [Formatting Options](#formatting-options)). A reference-type model needs `Factory` to be read; a value type without one starts from `default`. Pass `null` as the getter or the setter to build a layout that only reads or only writes; using it the other way throws `InvalidOperationException` before any line is touched. The column name appears in error messages. diff --git a/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs b/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs index 7c4a858..a7b89c2 100644 --- a/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs +++ b/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs @@ -127,6 +127,63 @@ public FixedWidthLayoutBuilder NullableColumn( return Add(column, parser, formatter); } + /// + /// Adds a column of a type that parses only from characters, such as a + /// date or time: blank reads as , and writes the column + /// filled with its trim character. A parses a date or + /// time exactly. + /// + /// + public FixedWidthLayoutBuilder NullableDecodedColumn( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, FixedWidthColumnOptions? options = null) + where TValue : struct, ISpanParsable, ISpanFormattable + { + var column = Begin(name, start, length, typeof(TValue?), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForNullable(set, column.Trim, + adapted => ColumnParserFactory.ForDecodedParsable(adapted, column.Trim, column.TrimMode, column.Format)); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + var inner = new SpanFormattableColumnFormatter( + start, length, column.FormatOptions, name, (in TModel model) => get(in model).GetValueOrDefault()); + formatter = new NullableColumnFormatter(start, length, column.Trim, get, inner); + } + return Add(column, parser, formatter); + } + + /// + /// Adds a column parsed and formatted by : + /// blank reads as without calling the converter, and + /// writes the column filled with its trim character. + /// + /// + public FixedWidthLayoutBuilder NullableConvertedColumn( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, + IFixedWidthConverter converter, FixedWidthColumnOptions? options = null) + where TValue : struct + { + ArgumentNullException.ThrowIfNull(converter); + var column = Begin(name, start, length, typeof(TValue?), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForNullable(set, column.Trim, + adapted => ColumnParserFactory.ForConverter(adapted, converter, column.Trim, column.TrimMode)); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + var inner = new ConverterColumnFormatter>( + start, length, column.FormatOptions, name, (in TModel model) => get(in model).GetValueOrDefault(), converter); + formatter = new NullableColumnFormatter(start, length, column.Trim, get, inner); + } + return Add(column, parser, formatter); + } + /// Adds a column parsed and formatted by . /// The column's name in error messages. /// The 0-based offset of the column, in bytes. diff --git a/src/FixedWidthParser/PublicAPI.Unshipped.txt b/src/FixedWidthParser/PublicAPI.Unshipped.txt index 8581501..a6f5b1c 100644 --- a/src/FixedWidthParser/PublicAPI.Unshipped.txt +++ b/src/FixedWidthParser/PublicAPI.Unshipped.txt @@ -202,3 +202,5 @@ FixedWidthParser.FixedWidthLayoutBuilder.DecodedColumn(string! n FixedWidthParser.FixedWidthLayoutBuilder.Factory(System.Func! factory) -> FixedWidthParser.FixedWidthLayoutBuilder! FixedWidthParser.FixedWidthLayoutBuilder.NullableColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! static FixedWidthParser.FixedWidthLayout.Build(System.Action!>! configure) -> FixedWidthParser.FixedWidthLayout! +FixedWidthParser.FixedWidthLayoutBuilder.NullableConvertedColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.Processors.IFixedWidthConverter! converter, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.FixedWidthLayoutBuilder.NullableDecodedColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! diff --git a/tests/FixedWidthParser.Tests/FluentLayoutTests.cs b/tests/FixedWidthParser.Tests/FluentLayoutTests.cs index 8f8f2a6..2a19e63 100644 --- a/tests/FixedWidthParser.Tests/FluentLayoutTests.cs +++ b/tests/FixedWidthParser.Tests/FluentLayoutTests.cs @@ -41,7 +41,11 @@ private static FixedWidthLayout Options() .NullableColumn("NullableAmount", 27, 10, (in MutableOptionsModel m) => m.NullableAmount, (ref MutableOptionsModel m, decimal? v) => m.NullableAmount = v) .ConvertedColumn("Cents", 37, 8, (in MutableOptionsModel m) => m.Cents, (ref MutableOptionsModel m, CentsValue v) => m.Cents = v, new CentsConverter()) .DecodedColumn("Date", 45, 8, (in MutableOptionsModel m) => m.Date, (ref MutableOptionsModel m, DateOnly v) => m.Date = v, - new FixedWidthColumnOptions { Format = "yyyyMMdd" })); + new FixedWidthColumnOptions { Format = "yyyyMMdd" }) + .NullableDecodedColumn("NullableDate", 53, 8, (in MutableOptionsModel m) => m.NullableDate, (ref MutableOptionsModel m, DateOnly? v) => m.NullableDate = v, + new FixedWidthColumnOptions { Format = "yyyyMMdd" }) + .NullableConvertedColumn("NullableCents", 61, 8, (in MutableOptionsModel m) => m.NullableCents, (ref MutableOptionsModel m, CentsValue? v) => m.NullableCents = v, + new CentsConverter())); } [Theory] @@ -85,6 +89,8 @@ public void Fluent_Options_MatchAttributes(int zero, double fixed2, string trunc NullableAmount = amount is null ? null : decimal.Parse(amount, Inv), Cents = new CentsValue(1234), Date = new DateOnly(2026, 10, 23), + NullableDate = age is null ? null : new DateOnly(2027, 1, 2), + NullableCents = age is null ? null : new CentsValue(99), }; var attributes = FixedWidthLayout.FromAttributes(); @@ -100,7 +106,7 @@ public void Fluent_Options_MatchAttributes(int zero, double fixed2, string trunc [Fact] public void Fluent_StarTrim_ParsesLikeAttributes() { - byte[] line = Bytes("00042" + "3.14 " + "ABCD" + "42***" + new string(' ', 15) + "1234 " + "20261023"); + byte[] line = Bytes("00042" + "3.14 " + "ABCD" + "42***" + new string(' ', 15) + "1234 " + "20261023" + new string(' ', 16)); Assert.True(FixedWidth.TryParse(line, FixedWidthLayout.FromAttributes(), Ctx, out var a)); Assert.True(FixedWidth.TryParse(line, Options(), Ctx, out var f)); diff --git a/tests/FixedWidthParser.Tests/LayoutWriteTests.cs b/tests/FixedWidthParser.Tests/LayoutWriteTests.cs index 1ac66ce..73d85a3 100644 --- a/tests/FixedWidthParser.Tests/LayoutWriteTests.cs +++ b/tests/FixedWidthParser.Tests/LayoutWriteTests.cs @@ -70,6 +70,34 @@ public void Generated_WriteMany_RecordLongerThanBatch() Assert.Equal("A".PadRight(70_000) + "\n" + "B".PadRight(70_000) + "\n", Decode(stream)); } + [Fact] + public async Task LongRecords_AsyncTwinsMatchSync() + { + GenHugeNameModel[] huge = [new() { Name = "A" }, new() { Name = "B" }]; + var hugeLayout = FixedWidthLayout.Build(b => b.Column("Name", 0, 70_000, (in GenHugeNameModel m) => m.Name, null)); + var ct = TestContext.Current.CancellationToken; + using var genSync = new MemoryStream(); + await using var genAsync = new MemoryStream(); + using var layoutSync = new MemoryStream(); + await using var layoutAsync = new MemoryStream(); + using var oneSync = new MemoryStream(); + await using var oneAsync = new MemoryStream(); + + FixedWidth.WriteMany(genSync, huge, Lf); + await FixedWidth.WriteManyAsync(genAsync, huge, Lf, ct); + FixedWidth.WriteMany(layoutSync, huge, hugeLayout, Lf); + await FixedWidth.WriteManyAsync(layoutAsync, huge, hugeLayout, Lf, ct); + FixedWidth.Write(oneSync, new GenLongNameModel { Name = "Wide" }, Lf); + await FixedWidth.WriteAsync(oneAsync, new GenLongNameModel { Name = "Wide" }, Lf, ct); + + string expected = "A".PadRight(70_000) + "\n" + "B".PadRight(70_000) + "\n"; + Assert.Equal(expected, Decode(genSync)); + Assert.Equal(expected, Decode(genAsync)); + Assert.Equal(expected, Decode(layoutSync)); + Assert.Equal(expected, Decode(layoutAsync)); + Assert.Equal(Decode(oneSync), Decode(oneAsync)); + } + [Fact] public void Layout_WriteMany_MatchesWriter() { diff --git a/tests/FixedWidthParser.Tests/TestModels.cs b/tests/FixedWidthParser.Tests/TestModels.cs index fb04a55..c068350 100644 --- a/tests/FixedWidthParser.Tests/TestModels.cs +++ b/tests/FixedWidthParser.Tests/TestModels.cs @@ -459,5 +459,7 @@ public sealed record class MutableOptionsModel [FixedColumn(27, 10)] public decimal? NullableAmount { get; set; } [FixedColumn(37, 8, Converter = typeof(CentsConverter))] public CentsValue Cents { get; set; } [FixedColumn(45, 8, Format = "yyyyMMdd")] public DateOnly Date { get; set; } + [FixedColumn(53, 8, Format = "yyyyMMdd")] public DateOnly? NullableDate { get; set; } + [FixedColumn(61, 8, Converter = typeof(CentsConverter))] public CentsValue? NullableCents { get; set; } } } \ No newline at end of file From bcfc2008c1177f5711dd99a19c0b0f15da1d967c Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 23 Oct 2026 23:13:29 -0300 Subject: [PATCH 26/85] refactor!: move user-facing types to the root namespace and plumbing to Infrastructure Alignment, TrimMode, OverflowBehavior, FixedColumnAttribute and IFixedWidthConverter now live in FixedWidthParser, so a fluent layout or a converter needs no extra using. The types that are public only for generated code or allocation-free enumeration (FixedWidthRuntime, ColumnFormatOptions, the record enumerables and enumerators, IRecordLineParser and the line-parser structs) move to FixedWidthParser.Infrastructure. The generator emits the new names. BREAKING CHANGE: replace using FixedWidthParser.Attributes and using FixedWidthParser.Processors with using FixedWidthParser. Co-Authored-By: Claude Opus 5.5 --- .../FixedWidthModelGenerator.cs | 30 +-- .../{Attributes => }/FixedColumnAttribute.cs | 2 +- src/FixedWidthParser/FixedWidth.cs | 1 + src/FixedWidthParser/FixedWidthContext.cs | 2 +- .../Formatters/ColumnFormatters.cs | 2 +- .../Formatters/FormatterBuilder.cs | 2 +- .../{Processors => }/IFixedWidthConverter.cs | 2 +- .../AsyncRecordEnumerator.cs | 4 +- .../ColumnFormatOptions.cs | 3 +- .../FixedWidthAsyncRecordEnumerable.cs | 2 +- .../FixedWidthRecordEnumerable.cs | 2 +- .../{ => Infrastructure}/FixedWidthRuntime.cs | 8 +- .../RecordEnumerator.cs | 3 +- .../RecordLineParsers.cs} | 2 +- .../Layouts/FixedWidthColumnOptions.cs | 2 - .../Layouts/FixedWidthLayoutBuilder.cs | 1 + .../Processors/ColumnParserFactory.cs | 2 +- .../Processors/CultureHelpers.cs | 5 +- src/FixedWidthParser/PublicAPI.Unshipped.txt | 206 +++++++++--------- .../Readers/ParallelAggregator.cs | 1 + .../Readers/ParallelStreamAggregator.cs | 1 + src/FixedWidthParser/Readers/ParseFailure.cs | 1 + .../Writers/RecordFormatters.cs | 4 +- tests/AotSmoke/Program.cs | 1 - .../Benchmarks/Perf/AsyncReaderBenchmarks.cs | 1 - tests/Benchmarks/Perf/ComparisonBenchmarks.cs | 9 +- tests/Benchmarks/Perf/ReaderBenchmarks.cs | 1 - .../Perf/ReaderPoolingBenchmarks.cs | 1 - .../Perf/ReaderStartupBenchmarks.cs | 1 - .../Perf/SourceGenParserBenchmarks.cs | 1 - tests/Benchmarks/SampleModel.cs | 2 +- .../FixedWidthModelGeneratorTests.cs | 14 +- .../ConverterAttributeTests.cs | 2 +- tests/FixedWidthParser.Tests/CoverageTests.cs | 3 +- .../DoubleFormatTests.cs | 3 +- .../FluentLayoutTests.cs | 1 - .../FixedWidthParser.Tests/GeneratedModels.cs | 2 - .../GeneratedReaderTests.cs | 2 +- .../ParallelAggregateTests.cs | 1 - .../PropertyRoundTripTests.cs | 1 - .../ReaderOptionsTests.cs | 1 - tests/FixedWidthParser.Tests/ReaderTests.cs | 2 +- tests/FixedWidthParser.Tests/TestModels.cs | 3 +- .../WriterAsyncTests.cs | 1 - 44 files changed, 164 insertions(+), 177 deletions(-) rename src/FixedWidthParser/{Attributes => }/FixedColumnAttribute.cs (98%) rename src/FixedWidthParser/{Processors => }/IFixedWidthConverter.cs (97%) rename src/FixedWidthParser/{Readers => Infrastructure}/AsyncRecordEnumerator.cs (97%) rename src/FixedWidthParser/{Formatters => Infrastructure}/ColumnFormatOptions.cs (97%) rename src/FixedWidthParser/{Readers => Infrastructure}/FixedWidthAsyncRecordEnumerable.cs (97%) rename src/FixedWidthParser/{Readers => Infrastructure}/FixedWidthRecordEnumerable.cs (98%) rename src/FixedWidthParser/{ => Infrastructure}/FixedWidthRuntime.cs (99%) rename src/FixedWidthParser/{Readers => Infrastructure}/RecordEnumerator.cs (97%) rename src/FixedWidthParser/{Readers/ILineParser.cs => Infrastructure/RecordLineParsers.cs} (97%) diff --git a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs index f312e08..5e93d58 100644 --- a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs +++ b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs @@ -1,11 +1,11 @@ -using System; using System.Collections.Immutable; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Text; -using Microsoft.CodeAnalysis; -using Microsoft.CodeAnalysis.CSharp; +using System; using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis; namespace FixedWidthParser.Generator { @@ -18,10 +18,10 @@ namespace FixedWidthParser.Generator public sealed class FixedWidthModelGenerator : IIncrementalGenerator { private const string MarkerMetadataName = "FixedWidthParser.IFixedWidthModel`1"; - private const string ColumnAttributeMetadataName = "FixedWidthParser.Attributes.FixedColumnAttribute"; - private const string ConverterInterfaceMetadataName = "FixedWidthParser.Processors.IFixedWidthConverter`1"; + private const string ColumnAttributeMetadataName = "FixedWidthParser.FixedColumnAttribute"; + private const string ConverterInterfaceMetadataName = "FixedWidthParser.IFixedWidthConverter`1"; private const string SpanFormattableMetadataName = "System.ISpanFormattable"; - private const string RuntimeFqn = "global::FixedWidthParser.FixedWidthRuntime"; + private const string RuntimeFqn = "global::FixedWidthParser.Infrastructure.FixedWidthRuntime"; private static readonly DiagnosticDescriptor MustBePartial = new( "FWP001", "Fixed-width model must be partial", @@ -482,7 +482,7 @@ private static void AppendFields(StringBuilder sb, string body, ModelInfo model) sb.Append(body).Append("private static readonly ").Append(c.ConverterFqn) .Append(" __converter").Append(i).Append(" = new ").Append(c.ConverterFqn).AppendLine("();"); } - sb.Append(body).Append("private static readonly global::FixedWidthParser.Formatters.ColumnFormatOptions __options").Append(i) + sb.Append(body).Append("private static readonly global::FixedWidthParser.Infrastructure.ColumnFormatOptions __options").Append(i) .Append(" = new(").Append(AlignmentLiteral(c.Alignment)).Append(", ").Append(SymbolDisplay.FormatLiteral(c.Padding, true)) .Append(", ").Append(c.Format is null ? "null" : SymbolDisplay.FormatLiteral(c.Format, true)) .Append(", ").Append(OverflowLiteral(c.Overflow)).AppendLine(");"); @@ -681,17 +681,17 @@ private static string ByteLiteral(char value) private static string AlignmentLiteral(int value) { return value == 1 - ? "global::FixedWidthParser.Attributes.Alignment.Right" - : "global::FixedWidthParser.Attributes.Alignment.Left"; + ? "global::FixedWidthParser.Alignment.Right" + : "global::FixedWidthParser.Alignment.Left"; } private static string OverflowLiteral(int resolvedValue) { return resolvedValue switch { - 1 => "global::FixedWidthParser.Attributes.OverflowBehavior.Truncate", - 2 => "global::FixedWidthParser.Attributes.OverflowBehavior.Throw", - _ => "global::FixedWidthParser.Attributes.OverflowBehavior.Default", + 1 => "global::FixedWidthParser.OverflowBehavior.Truncate", + 2 => "global::FixedWidthParser.OverflowBehavior.Throw", + _ => "global::FixedWidthParser.OverflowBehavior.Default", }; } @@ -699,9 +699,9 @@ private static string TrimModeLiteral(int value) { return value switch { - 1 => "global::FixedWidthParser.Attributes.TrimMode.Leading", - 2 => "global::FixedWidthParser.Attributes.TrimMode.Both", - _ => "global::FixedWidthParser.Attributes.TrimMode.Trailing" + 1 => "global::FixedWidthParser.TrimMode.Leading", + 2 => "global::FixedWidthParser.TrimMode.Both", + _ => "global::FixedWidthParser.TrimMode.Trailing" }; } diff --git a/src/FixedWidthParser/Attributes/FixedColumnAttribute.cs b/src/FixedWidthParser/FixedColumnAttribute.cs similarity index 98% rename from src/FixedWidthParser/Attributes/FixedColumnAttribute.cs rename to src/FixedWidthParser/FixedColumnAttribute.cs index 0d663bd..a541366 100644 --- a/src/FixedWidthParser/Attributes/FixedColumnAttribute.cs +++ b/src/FixedWidthParser/FixedColumnAttribute.cs @@ -1,4 +1,4 @@ -namespace FixedWidthParser.Attributes +namespace FixedWidthParser { /// Alignment of the content within the column when writing. public enum Alignment diff --git a/src/FixedWidthParser/FixedWidth.cs b/src/FixedWidthParser/FixedWidth.cs index e32305e..03fc55e 100644 --- a/src/FixedWidthParser/FixedWidth.cs +++ b/src/FixedWidthParser/FixedWidth.cs @@ -1,3 +1,4 @@ +using FixedWidthParser.Infrastructure; using FixedWidthParser.Readers; using FixedWidthParser.Writers; diff --git a/src/FixedWidthParser/FixedWidthContext.cs b/src/FixedWidthParser/FixedWidthContext.cs index 783ba13..124ac1f 100644 --- a/src/FixedWidthParser/FixedWidthContext.cs +++ b/src/FixedWidthParser/FixedWidthContext.cs @@ -7,7 +7,7 @@ namespace FixedWidthParser { /// /// Settings a model parser, model writer or converter needs for one line, handed to every - /// generated TryParse/TryFormat and every call. + /// generated TryParse/TryFormat and every call. /// /// /// Passed by reference. New settings arrive as new properties, so a converter diff --git a/src/FixedWidthParser/Formatters/ColumnFormatters.cs b/src/FixedWidthParser/Formatters/ColumnFormatters.cs index b101f25..51d0300 100644 --- a/src/FixedWidthParser/Formatters/ColumnFormatters.cs +++ b/src/FixedWidthParser/Formatters/ColumnFormatters.cs @@ -1,4 +1,4 @@ -using FixedWidthParser.Processors; +using FixedWidthParser.Infrastructure; namespace FixedWidthParser.Formatters { diff --git a/src/FixedWidthParser/Formatters/FormatterBuilder.cs b/src/FixedWidthParser/Formatters/FormatterBuilder.cs index a4de1c1..27180e1 100644 --- a/src/FixedWidthParser/Formatters/FormatterBuilder.cs +++ b/src/FixedWidthParser/Formatters/FormatterBuilder.cs @@ -1,7 +1,7 @@ using System.Linq.Expressions; using System.Reflection; using FixedWidthParser.Attributes; -using FixedWidthParser.Processors; +using FixedWidthParser.Infrastructure; namespace FixedWidthParser.Formatters { diff --git a/src/FixedWidthParser/Processors/IFixedWidthConverter.cs b/src/FixedWidthParser/IFixedWidthConverter.cs similarity index 97% rename from src/FixedWidthParser/Processors/IFixedWidthConverter.cs rename to src/FixedWidthParser/IFixedWidthConverter.cs index 6816795..60c3f41 100644 --- a/src/FixedWidthParser/Processors/IFixedWidthConverter.cs +++ b/src/FixedWidthParser/IFixedWidthConverter.cs @@ -1,4 +1,4 @@ -namespace FixedWidthParser.Processors +namespace FixedWidthParser { /// /// Per-property custom conversion for a fixed-width column, wired via diff --git a/src/FixedWidthParser/Readers/AsyncRecordEnumerator.cs b/src/FixedWidthParser/Infrastructure/AsyncRecordEnumerator.cs similarity index 97% rename from src/FixedWidthParser/Readers/AsyncRecordEnumerator.cs rename to src/FixedWidthParser/Infrastructure/AsyncRecordEnumerator.cs index d1cbb69..abf8903 100644 --- a/src/FixedWidthParser/Readers/AsyncRecordEnumerator.cs +++ b/src/FixedWidthParser/Infrastructure/AsyncRecordEnumerator.cs @@ -1,4 +1,6 @@ -namespace FixedWidthParser.Readers +using FixedWidthParser.Readers; + +namespace FixedWidthParser.Infrastructure { /// /// Asynchronous record enumerator over a stream. The line scan stays synchronous; only the refill diff --git a/src/FixedWidthParser/Formatters/ColumnFormatOptions.cs b/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs similarity index 97% rename from src/FixedWidthParser/Formatters/ColumnFormatOptions.cs rename to src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs index f7e02b4..2596246 100644 --- a/src/FixedWidthParser/Formatters/ColumnFormatOptions.cs +++ b/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs @@ -1,7 +1,6 @@ using System.Text; -using FixedWidthParser.Attributes; -namespace FixedWidthParser.Formatters +namespace FixedWidthParser.Infrastructure { /// /// Resolved formatting options for a column (alignment, padding, format string and an overflow diff --git a/src/FixedWidthParser/Readers/FixedWidthAsyncRecordEnumerable.cs b/src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs similarity index 97% rename from src/FixedWidthParser/Readers/FixedWidthAsyncRecordEnumerable.cs rename to src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs index 1be8d89..10ff8dd 100644 --- a/src/FixedWidthParser/Readers/FixedWidthAsyncRecordEnumerable.cs +++ b/src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs @@ -1,4 +1,4 @@ -namespace FixedWidthParser.Readers +namespace FixedWidthParser.Infrastructure { /// /// Asynchronous version of for diff --git a/src/FixedWidthParser/Readers/FixedWidthRecordEnumerable.cs b/src/FixedWidthParser/Infrastructure/FixedWidthRecordEnumerable.cs similarity index 98% rename from src/FixedWidthParser/Readers/FixedWidthRecordEnumerable.cs rename to src/FixedWidthParser/Infrastructure/FixedWidthRecordEnumerable.cs index 203eca1..b9fa8d8 100644 --- a/src/FixedWidthParser/Readers/FixedWidthRecordEnumerable.cs +++ b/src/FixedWidthParser/Infrastructure/FixedWidthRecordEnumerable.cs @@ -1,7 +1,7 @@ using System.Collections; using System.Diagnostics.CodeAnalysis; -namespace FixedWidthParser.Readers +namespace FixedWidthParser.Infrastructure { /// /// A lazily-read sequence of models from a stream or a file, without a string per line: bytes are diff --git a/src/FixedWidthParser/FixedWidthRuntime.cs b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs similarity index 99% rename from src/FixedWidthParser/FixedWidthRuntime.cs rename to src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs index 7f79c11..0f8d672 100644 --- a/src/FixedWidthParser/FixedWidthRuntime.cs +++ b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs @@ -1,15 +1,13 @@ -using System.Buffers; using System.Buffers.Text; +using System.Buffers; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Runtime.CompilerServices; -using System.Text; using System.Text.Unicode; -using FixedWidthParser.Attributes; -using FixedWidthParser.Formatters; +using System.Text; using FixedWidthParser.Processors; -namespace FixedWidthParser +namespace FixedWidthParser.Infrastructure { /// /// Low-level helpers called by the source-generated diff --git a/src/FixedWidthParser/Readers/RecordEnumerator.cs b/src/FixedWidthParser/Infrastructure/RecordEnumerator.cs similarity index 97% rename from src/FixedWidthParser/Readers/RecordEnumerator.cs rename to src/FixedWidthParser/Infrastructure/RecordEnumerator.cs index cf27cb6..71fb436 100644 --- a/src/FixedWidthParser/Readers/RecordEnumerator.cs +++ b/src/FixedWidthParser/Infrastructure/RecordEnumerator.cs @@ -1,7 +1,8 @@ using System.Collections; using System.Diagnostics.CodeAnalysis; +using FixedWidthParser.Readers; -namespace FixedWidthParser.Readers +namespace FixedWidthParser.Infrastructure { /// /// Synchronous record enumerator over a stream. A specialized by diff --git a/src/FixedWidthParser/Readers/ILineParser.cs b/src/FixedWidthParser/Infrastructure/RecordLineParsers.cs similarity index 97% rename from src/FixedWidthParser/Readers/ILineParser.cs rename to src/FixedWidthParser/Infrastructure/RecordLineParsers.cs index 0fcb850..1974e20 100644 --- a/src/FixedWidthParser/Readers/ILineParser.cs +++ b/src/FixedWidthParser/Infrastructure/RecordLineParsers.cs @@ -1,4 +1,4 @@ -namespace FixedWidthParser.Readers +namespace FixedWidthParser.Infrastructure { /// /// Parse strategy the record enumerators are specialized by. Implemented by diff --git a/src/FixedWidthParser/Layouts/FixedWidthColumnOptions.cs b/src/FixedWidthParser/Layouts/FixedWidthColumnOptions.cs index 281b955..e19d853 100644 --- a/src/FixedWidthParser/Layouts/FixedWidthColumnOptions.cs +++ b/src/FixedWidthParser/Layouts/FixedWidthColumnOptions.cs @@ -1,5 +1,3 @@ -using FixedWidthParser.Attributes; - namespace FixedWidthParser { /// diff --git a/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs b/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs index a7b89c2..8413345 100644 --- a/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs +++ b/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs @@ -1,6 +1,7 @@ using System.Runtime.InteropServices; using FixedWidthParser.Attributes; using FixedWidthParser.Formatters; +using FixedWidthParser.Infrastructure; using FixedWidthParser.Parsers; using FixedWidthParser.Processors; diff --git a/src/FixedWidthParser/Processors/ColumnParserFactory.cs b/src/FixedWidthParser/Processors/ColumnParserFactory.cs index 37cdf21..3594cf1 100644 --- a/src/FixedWidthParser/Processors/ColumnParserFactory.cs +++ b/src/FixedWidthParser/Processors/ColumnParserFactory.cs @@ -1,5 +1,5 @@ using System.Reflection; -using FixedWidthParser.Attributes; +using FixedWidthParser.Infrastructure; namespace FixedWidthParser.Processors { diff --git a/src/FixedWidthParser/Processors/CultureHelpers.cs b/src/FixedWidthParser/Processors/CultureHelpers.cs index bed26ac..25fed3f 100644 --- a/src/FixedWidthParser/Processors/CultureHelpers.cs +++ b/src/FixedWidthParser/Processors/CultureHelpers.cs @@ -4,6 +4,7 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Text; +using FixedWidthParser.Infrastructure; using csFastFloat; namespace FixedWidthParser.Processors @@ -50,7 +51,7 @@ private static char Resolve(IFormatProvider formatProvider) } /// Parses a (not yet trimmed) column as , honoring the context's provider. - public static bool TryParseDouble(ReadOnlySpan column, in FixedWidthContext context, out double value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) + public static bool TryParseDouble(ReadOnlySpan column, in FixedWidthContext context, out double value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) { // Also trim leading spaces: csFastFloat's characters_consumed does not reliably include // skipped leading whitespace, so a right-aligned column would otherwise fail the @@ -67,7 +68,7 @@ public static bool TryParseDouble(ReadOnlySpan column, in FixedWidthContex } /// Parses a (not yet trimmed) column as , honoring the context's provider. - public static bool TryParseFloat(ReadOnlySpan column, in FixedWidthContext context, out float value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) + public static bool TryParseFloat(ReadOnlySpan column, in FixedWidthContext context, out float value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) { var trimmed = FixedWidthRuntime.TrimColumn(column, trimChar, trimMode).TrimStart((byte)' '); var formatProvider = context.FormatProvider; diff --git a/src/FixedWidthParser/PublicAPI.Unshipped.txt b/src/FixedWidthParser/PublicAPI.Unshipped.txt index a6f5b1c..419d9ac 100644 --- a/src/FixedWidthParser/PublicAPI.Unshipped.txt +++ b/src/FixedWidthParser/PublicAPI.Unshipped.txt @@ -1,33 +1,4 @@ #nullable enable -FixedWidthParser.Attributes.Alignment -FixedWidthParser.Attributes.Alignment.Left = 0 -> FixedWidthParser.Attributes.Alignment -FixedWidthParser.Attributes.Alignment.Right = 1 -> FixedWidthParser.Attributes.Alignment -FixedWidthParser.Attributes.FixedColumnAttribute -FixedWidthParser.Attributes.FixedColumnAttribute.Alignment.get -> FixedWidthParser.Attributes.Alignment -FixedWidthParser.Attributes.FixedColumnAttribute.Alignment.set -> void -FixedWidthParser.Attributes.FixedColumnAttribute.Converter.get -> System.Type? -FixedWidthParser.Attributes.FixedColumnAttribute.Converter.set -> void -FixedWidthParser.Attributes.FixedColumnAttribute.FixedColumnAttribute(int start, int length) -> void -FixedWidthParser.Attributes.FixedColumnAttribute.Format.get -> string? -FixedWidthParser.Attributes.FixedColumnAttribute.Format.set -> void -FixedWidthParser.Attributes.FixedColumnAttribute.Length.get -> int -FixedWidthParser.Attributes.FixedColumnAttribute.Overflow.get -> FixedWidthParser.Attributes.OverflowBehavior -FixedWidthParser.Attributes.FixedColumnAttribute.Overflow.set -> void -FixedWidthParser.Attributes.FixedColumnAttribute.Padding.get -> char -FixedWidthParser.Attributes.FixedColumnAttribute.Padding.set -> void -FixedWidthParser.Attributes.FixedColumnAttribute.Start.get -> int -FixedWidthParser.Attributes.FixedColumnAttribute.TrimChar.get -> char -FixedWidthParser.Attributes.FixedColumnAttribute.TrimChar.set -> void -FixedWidthParser.Attributes.FixedColumnAttribute.TrimMode.get -> FixedWidthParser.Attributes.TrimMode -FixedWidthParser.Attributes.FixedColumnAttribute.TrimMode.set -> void -FixedWidthParser.Attributes.OverflowBehavior -FixedWidthParser.Attributes.OverflowBehavior.Default = 0 -> FixedWidthParser.Attributes.OverflowBehavior -FixedWidthParser.Attributes.OverflowBehavior.Throw = 2 -> FixedWidthParser.Attributes.OverflowBehavior -FixedWidthParser.Attributes.OverflowBehavior.Truncate = 1 -> FixedWidthParser.Attributes.OverflowBehavior -FixedWidthParser.Attributes.TrimMode -FixedWidthParser.Attributes.TrimMode.Both = 2 -> FixedWidthParser.Attributes.TrimMode -FixedWidthParser.Attributes.TrimMode.Leading = 1 -> FixedWidthParser.Attributes.TrimMode -FixedWidthParser.Attributes.TrimMode.Trailing = 0 -> FixedWidthParser.Attributes.TrimMode FixedWidthParser.FixedWidth FixedWidthParser.FixedWidthContext FixedWidthParser.FixedWidthContext.Encoding.get -> System.Text.Encoding! @@ -63,7 +34,6 @@ FixedWidthParser.FixedWidthReaderOptions.OnParseError.get -> System.Func void FixedWidthParser.FixedWidthReaderOptions.StringPool.get -> CommunityToolkit.HighPerformance.Buffers.StringPool? FixedWidthParser.FixedWidthReaderOptions.StringPool.init -> void -FixedWidthParser.FixedWidthRuntime FixedWidthParser.FixedWidthWriterOptions FixedWidthParser.FixedWidthWriterOptions.Encoding.get -> System.Text.Encoding? FixedWidthParser.FixedWidthWriterOptions.Encoding.init -> void @@ -72,77 +42,16 @@ FixedWidthParser.FixedWidthWriterOptions.FormatProvider.get -> System.IFormatPro FixedWidthParser.FixedWidthWriterOptions.FormatProvider.init -> void FixedWidthParser.FixedWidthWriterOptions.NewLine.get -> string! FixedWidthParser.FixedWidthWriterOptions.NewLine.init -> void -FixedWidthParser.Formatters.ColumnFormatOptions -FixedWidthParser.Formatters.ColumnFormatOptions.Alignment.get -> FixedWidthParser.Attributes.Alignment -FixedWidthParser.Formatters.ColumnFormatOptions.Alignment.init -> void -FixedWidthParser.Formatters.ColumnFormatOptions.ColumnFormatOptions() -> void -FixedWidthParser.Formatters.ColumnFormatOptions.ColumnFormatOptions(FixedWidthParser.Attributes.Alignment Alignment, char Padding, string? Format, FixedWidthParser.Attributes.OverflowBehavior Overflow) -> void -FixedWidthParser.Formatters.ColumnFormatOptions.Deconstruct(out FixedWidthParser.Attributes.Alignment Alignment, out char Padding, out string? Format, out FixedWidthParser.Attributes.OverflowBehavior Overflow) -> void -FixedWidthParser.Formatters.ColumnFormatOptions.Equals(FixedWidthParser.Formatters.ColumnFormatOptions other) -> bool -FixedWidthParser.Formatters.ColumnFormatOptions.Format.get -> string? -FixedWidthParser.Formatters.ColumnFormatOptions.Format.init -> void -FixedWidthParser.Formatters.ColumnFormatOptions.Overflow.get -> FixedWidthParser.Attributes.OverflowBehavior -FixedWidthParser.Formatters.ColumnFormatOptions.Overflow.init -> void -FixedWidthParser.Formatters.ColumnFormatOptions.Padding.get -> char -FixedWidthParser.Formatters.ColumnFormatOptions.Padding.init -> void -FixedWidthParser.Formatters.ColumnFormatOptions.WriteInto(System.ReadOnlySpan content, System.Span slice, string! columnName) -> void FixedWidthParser.IFixedWidthModel FixedWidthParser.IFixedWidthModel.TryFormat(in TSelf model, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int bytesWritten) -> bool FixedWidthParser.IFixedWidthModel.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TSelf model) -> bool -FixedWidthParser.Processors.IFixedWidthConverter -FixedWidthParser.Processors.IFixedWidthConverter.TryFormat(T value, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int written) -> bool -FixedWidthParser.Processors.IFixedWidthConverter.TryParse(System.ReadOnlySpan field, in FixedWidthParser.FixedWidthContext context, out T value) -> bool -FixedWidthParser.Readers.AsyncRecordEnumerator -FixedWidthParser.Readers.AsyncRecordEnumerator.Current.get -> TModel -FixedWidthParser.Readers.AsyncRecordEnumerator.DisposeAsync() -> System.Threading.Tasks.ValueTask -FixedWidthParser.Readers.AsyncRecordEnumerator.MoveNextAsync() -> System.Threading.Tasks.ValueTask -FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable -FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable.GetAsyncEnumerator(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> FixedWidthParser.Readers.AsyncRecordEnumerator! -FixedWidthParser.Readers.FixedWidthRecordEnumerable -FixedWidthParser.Readers.FixedWidthRecordEnumerable.GetEnumerator() -> FixedWidthParser.Readers.RecordEnumerator -FixedWidthParser.Readers.GeneratedLineParser -FixedWidthParser.Readers.GeneratedLineParser.GeneratedLineParser() -> void -FixedWidthParser.Readers.GeneratedLineParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool -FixedWidthParser.Readers.IRecordLineParser -FixedWidthParser.Readers.IRecordLineParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool -FixedWidthParser.Readers.RecordEnumerator -FixedWidthParser.Readers.RecordEnumerator.Current.get -> TModel -FixedWidthParser.Readers.RecordEnumerator.Dispose() -> void -FixedWidthParser.Readers.RecordEnumerator.MoveNext() -> bool -FixedWidthParser.Readers.RecordEnumerator.RecordEnumerator() -> void -FixedWidthParser.Readers.RecordEnumerator.Reset() -> void override FixedWidthParser.FixedWidthContext.GetHashCode() -> int -override FixedWidthParser.Formatters.ColumnFormatOptions.GetHashCode() -> int -static FixedWidthParser.FixedWidth.Read(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! -static FixedWidthParser.FixedWidth.ReadAsync(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! -static FixedWidthParser.FixedWidth.ReadFile(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! -static FixedWidthParser.FixedWidth.ReadFileAsync(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! static FixedWidthParser.FixedWidth.TryFormat(in TModel model, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int bytesWritten) -> bool static FixedWidthParser.FixedWidth.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool static FixedWidthParser.FixedWidthContext.operator !=(FixedWidthParser.FixedWidthContext left, FixedWidthParser.FixedWidthContext right) -> bool static FixedWidthParser.FixedWidthContext.operator ==(FixedWidthParser.FixedWidthContext left, FixedWidthParser.FixedWidthContext right) -> bool -static FixedWidthParser.FixedWidthRuntime.FormatConvert(TValue value, System.Span slice, in FixedWidthParser.FixedWidthContext context, TConverter converter, FixedWidthParser.Formatters.ColumnFormatOptions options, string! columnName) -> void -static FixedWidthParser.FixedWidthRuntime.FormatString(string! value, System.Span slice, in FixedWidthParser.FixedWidthContext context, FixedWidthParser.Formatters.ColumnFormatOptions options, string! columnName) -> void -static FixedWidthParser.FixedWidthRuntime.FormatUtf8Value(TValue value, System.Span slice, in FixedWidthParser.FixedWidthContext context, FixedWidthParser.Formatters.ColumnFormatOptions options, string! columnName) -> void -static FixedWidthParser.FixedWidthRuntime.FormatValue(TValue value, System.Span slice, in FixedWidthParser.FixedWidthContext context, FixedWidthParser.Formatters.ColumnFormatOptions options, string! columnName) -> void -static FixedWidthParser.FixedWidthRuntime.IsBlank(System.ReadOnlySpan column, byte trimChar) -> bool -static FixedWidthParser.FixedWidthRuntime.String(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> string! -static FixedWidthParser.FixedWidthRuntime.TrimColumn(System.ReadOnlySpan column, byte trimChar, FixedWidthParser.Attributes.TrimMode trimMode) -> System.ReadOnlySpan -static FixedWidthParser.FixedWidthRuntime.TryConvert(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, TConverter converter, out TValue value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool -static FixedWidthParser.FixedWidthRuntime.TryDateOnlyExact(System.ReadOnlySpan column, string! format, in FixedWidthParser.FixedWidthContext context, out System.DateOnly value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool -static FixedWidthParser.FixedWidthRuntime.TryDateTimeExact(System.ReadOnlySpan column, string! format, in FixedWidthParser.FixedWidthContext context, out System.DateTime value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool -static FixedWidthParser.FixedWidthRuntime.TryDateTimeOffsetExact(System.ReadOnlySpan column, string! format, in FixedWidthParser.FixedWidthContext context, out System.DateTimeOffset value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool -static FixedWidthParser.FixedWidthRuntime.TryDouble(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out double value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool -static FixedWidthParser.FixedWidthRuntime.TryFloat(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out float value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool -static FixedWidthParser.FixedWidthRuntime.TryParse(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out TValue value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool -static FixedWidthParser.FixedWidthRuntime.TryParseDecoded(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out TValue value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool -static FixedWidthParser.FixedWidthRuntime.TryTimeOnlyExact(System.ReadOnlySpan column, string! format, in FixedWidthParser.FixedWidthContext context, out System.TimeOnly value, byte trimChar = 32, FixedWidthParser.Attributes.TrimMode trimMode = FixedWidthParser.Attributes.TrimMode.Trailing) -> bool -static FixedWidthParser.Formatters.ColumnFormatOptions.operator !=(FixedWidthParser.Formatters.ColumnFormatOptions left, FixedWidthParser.Formatters.ColumnFormatOptions right) -> bool -static FixedWidthParser.Formatters.ColumnFormatOptions.operator ==(FixedWidthParser.Formatters.ColumnFormatOptions left, FixedWidthParser.Formatters.ColumnFormatOptions right) -> bool ~override FixedWidthParser.FixedWidthContext.Equals(object obj) -> bool ~override FixedWidthParser.FixedWidthContext.ToString() -> string -~override FixedWidthParser.Formatters.ColumnFormatOptions.Equals(object obj) -> bool -~override FixedWidthParser.Formatters.ColumnFormatOptions.ToString() -> string FixedWidthParser.FixedWidthParallelOptions FixedWidthParser.FixedWidthParallelOptions.DegreeOfParallelism.get -> int FixedWidthParser.FixedWidthParallelOptions.DegreeOfParallelism.init -> void @@ -161,15 +70,8 @@ static FixedWidthParser.FixedWidth.TryParse(System.ReadOnlySpan li static FixedWidthParser.FixedWidthLayout.FromAttributes() -> FixedWidthParser.FixedWidthLayout! virtual FixedWidthParser.ColumnGetter.Invoke(in TModel model) -> TValue virtual FixedWidthParser.ColumnSetter.Invoke(ref TModel model, TValue value) -> void -FixedWidthParser.Readers.LayoutLineParser -FixedWidthParser.Readers.LayoutLineParser.LayoutLineParser() -> void -FixedWidthParser.Readers.LayoutLineParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool static FixedWidthParser.FixedWidth.AggregateAsync(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! static FixedWidthParser.FixedWidth.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! -static FixedWidthParser.FixedWidth.Read(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! -static FixedWidthParser.FixedWidth.ReadAsync(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! -static FixedWidthParser.FixedWidth.ReadFile(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthRecordEnumerable>! -static FixedWidthParser.FixedWidth.ReadFileAsync(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable>! static FixedWidthParser.FixedWidth.Write(System.IO.Stream! stream, in TModel model, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthWriterOptions? options = null) -> void static FixedWidthParser.FixedWidth.Write(System.IO.Stream! stream, in TModel model, FixedWidthParser.FixedWidthWriterOptions? options = null) -> void static FixedWidthParser.FixedWidth.WriteAsync(System.IO.Stream! stream, TModel model, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthWriterOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! @@ -181,26 +83,124 @@ static FixedWidthParser.FixedWidth.WriteMany(System.IO.Stream! stream, S static FixedWidthParser.FixedWidth.WriteManyAsync(System.IO.Stream! stream, System.Collections.Generic.IEnumerable! models, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthWriterOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! static FixedWidthParser.FixedWidth.WriteManyAsync(System.IO.Stream! stream, System.Collections.Generic.IEnumerable! models, FixedWidthParser.FixedWidthWriterOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! FixedWidthParser.FixedWidthColumnOptions -FixedWidthParser.FixedWidthColumnOptions.Alignment.get -> FixedWidthParser.Attributes.Alignment FixedWidthParser.FixedWidthColumnOptions.Alignment.init -> void FixedWidthParser.FixedWidthColumnOptions.FixedWidthColumnOptions() -> void FixedWidthParser.FixedWidthColumnOptions.Format.get -> string? FixedWidthParser.FixedWidthColumnOptions.Format.init -> void -FixedWidthParser.FixedWidthColumnOptions.Overflow.get -> FixedWidthParser.Attributes.OverflowBehavior FixedWidthParser.FixedWidthColumnOptions.Overflow.init -> void FixedWidthParser.FixedWidthColumnOptions.Padding.get -> char FixedWidthParser.FixedWidthColumnOptions.Padding.init -> void FixedWidthParser.FixedWidthColumnOptions.TrimChar.get -> char FixedWidthParser.FixedWidthColumnOptions.TrimChar.init -> void -FixedWidthParser.FixedWidthColumnOptions.TrimMode.get -> FixedWidthParser.Attributes.TrimMode FixedWidthParser.FixedWidthColumnOptions.TrimMode.init -> void FixedWidthParser.FixedWidthLayoutBuilder FixedWidthParser.FixedWidthLayoutBuilder.Column(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! FixedWidthParser.FixedWidthLayoutBuilder.Column(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! -FixedWidthParser.FixedWidthLayoutBuilder.ConvertedColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.Processors.IFixedWidthConverter! converter, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! FixedWidthParser.FixedWidthLayoutBuilder.DecodedColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! FixedWidthParser.FixedWidthLayoutBuilder.Factory(System.Func! factory) -> FixedWidthParser.FixedWidthLayoutBuilder! FixedWidthParser.FixedWidthLayoutBuilder.NullableColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! static FixedWidthParser.FixedWidthLayout.Build(System.Action!>! configure) -> FixedWidthParser.FixedWidthLayout! -FixedWidthParser.FixedWidthLayoutBuilder.NullableConvertedColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.Processors.IFixedWidthConverter! converter, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! FixedWidthParser.FixedWidthLayoutBuilder.NullableDecodedColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.Alignment +FixedWidthParser.Alignment.Left = 0 -> FixedWidthParser.Alignment +FixedWidthParser.Alignment.Right = 1 -> FixedWidthParser.Alignment +FixedWidthParser.FixedColumnAttribute +FixedWidthParser.FixedColumnAttribute.Alignment.get -> FixedWidthParser.Alignment +FixedWidthParser.FixedColumnAttribute.Alignment.set -> void +FixedWidthParser.FixedColumnAttribute.Converter.get -> System.Type? +FixedWidthParser.FixedColumnAttribute.Converter.set -> void +FixedWidthParser.FixedColumnAttribute.FixedColumnAttribute(int start, int length) -> void +FixedWidthParser.FixedColumnAttribute.Format.get -> string? +FixedWidthParser.FixedColumnAttribute.Format.set -> void +FixedWidthParser.FixedColumnAttribute.Length.get -> int +FixedWidthParser.FixedColumnAttribute.Overflow.get -> FixedWidthParser.OverflowBehavior +FixedWidthParser.FixedColumnAttribute.Overflow.set -> void +FixedWidthParser.FixedColumnAttribute.Padding.get -> char +FixedWidthParser.FixedColumnAttribute.Padding.set -> void +FixedWidthParser.FixedColumnAttribute.Start.get -> int +FixedWidthParser.FixedColumnAttribute.TrimChar.get -> char +FixedWidthParser.FixedColumnAttribute.TrimChar.set -> void +FixedWidthParser.FixedColumnAttribute.TrimMode.get -> FixedWidthParser.TrimMode +FixedWidthParser.FixedColumnAttribute.TrimMode.set -> void +FixedWidthParser.FixedWidthColumnOptions.Alignment.get -> FixedWidthParser.Alignment +FixedWidthParser.FixedWidthColumnOptions.Overflow.get -> FixedWidthParser.OverflowBehavior +FixedWidthParser.FixedWidthColumnOptions.TrimMode.get -> FixedWidthParser.TrimMode +FixedWidthParser.FixedWidthLayoutBuilder.ConvertedColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.IFixedWidthConverter! converter, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.FixedWidthLayoutBuilder.NullableConvertedColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.IFixedWidthConverter! converter, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.IFixedWidthConverter +FixedWidthParser.IFixedWidthConverter.TryFormat(T value, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int written) -> bool +FixedWidthParser.IFixedWidthConverter.TryParse(System.ReadOnlySpan field, in FixedWidthParser.FixedWidthContext context, out T value) -> bool +FixedWidthParser.Infrastructure.AsyncRecordEnumerator +FixedWidthParser.Infrastructure.AsyncRecordEnumerator.Current.get -> TModel +FixedWidthParser.Infrastructure.AsyncRecordEnumerator.DisposeAsync() -> System.Threading.Tasks.ValueTask +FixedWidthParser.Infrastructure.AsyncRecordEnumerator.MoveNextAsync() -> System.Threading.Tasks.ValueTask +FixedWidthParser.Infrastructure.ColumnFormatOptions +FixedWidthParser.Infrastructure.ColumnFormatOptions.Alignment.get -> FixedWidthParser.Alignment +FixedWidthParser.Infrastructure.ColumnFormatOptions.Alignment.init -> void +FixedWidthParser.Infrastructure.ColumnFormatOptions.ColumnFormatOptions() -> void +FixedWidthParser.Infrastructure.ColumnFormatOptions.ColumnFormatOptions(FixedWidthParser.Alignment Alignment, char Padding, string? Format, FixedWidthParser.OverflowBehavior Overflow) -> void +FixedWidthParser.Infrastructure.ColumnFormatOptions.Deconstruct(out FixedWidthParser.Alignment Alignment, out char Padding, out string? Format, out FixedWidthParser.OverflowBehavior Overflow) -> void +FixedWidthParser.Infrastructure.ColumnFormatOptions.Equals(FixedWidthParser.Infrastructure.ColumnFormatOptions other) -> bool +FixedWidthParser.Infrastructure.ColumnFormatOptions.Format.get -> string? +FixedWidthParser.Infrastructure.ColumnFormatOptions.Format.init -> void +FixedWidthParser.Infrastructure.ColumnFormatOptions.Overflow.get -> FixedWidthParser.OverflowBehavior +FixedWidthParser.Infrastructure.ColumnFormatOptions.Overflow.init -> void +FixedWidthParser.Infrastructure.ColumnFormatOptions.Padding.get -> char +FixedWidthParser.Infrastructure.ColumnFormatOptions.Padding.init -> void +FixedWidthParser.Infrastructure.ColumnFormatOptions.WriteInto(System.ReadOnlySpan content, System.Span slice, string! columnName) -> void +FixedWidthParser.Infrastructure.FixedWidthAsyncRecordEnumerable +FixedWidthParser.Infrastructure.FixedWidthAsyncRecordEnumerable.GetAsyncEnumerator(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> FixedWidthParser.Infrastructure.AsyncRecordEnumerator! +FixedWidthParser.Infrastructure.FixedWidthRecordEnumerable +FixedWidthParser.Infrastructure.FixedWidthRecordEnumerable.GetEnumerator() -> FixedWidthParser.Infrastructure.RecordEnumerator +FixedWidthParser.Infrastructure.FixedWidthRuntime +FixedWidthParser.Infrastructure.GeneratedLineParser +FixedWidthParser.Infrastructure.GeneratedLineParser.GeneratedLineParser() -> void +FixedWidthParser.Infrastructure.GeneratedLineParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool +FixedWidthParser.Infrastructure.IRecordLineParser +FixedWidthParser.Infrastructure.IRecordLineParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool +FixedWidthParser.Infrastructure.LayoutLineParser +FixedWidthParser.Infrastructure.LayoutLineParser.LayoutLineParser() -> void +FixedWidthParser.Infrastructure.LayoutLineParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool +FixedWidthParser.Infrastructure.RecordEnumerator +FixedWidthParser.Infrastructure.RecordEnumerator.Current.get -> TModel +FixedWidthParser.Infrastructure.RecordEnumerator.Dispose() -> void +FixedWidthParser.Infrastructure.RecordEnumerator.MoveNext() -> bool +FixedWidthParser.Infrastructure.RecordEnumerator.RecordEnumerator() -> void +FixedWidthParser.Infrastructure.RecordEnumerator.Reset() -> void +FixedWidthParser.OverflowBehavior +FixedWidthParser.OverflowBehavior.Default = 0 -> FixedWidthParser.OverflowBehavior +FixedWidthParser.OverflowBehavior.Throw = 2 -> FixedWidthParser.OverflowBehavior +FixedWidthParser.OverflowBehavior.Truncate = 1 -> FixedWidthParser.OverflowBehavior +FixedWidthParser.TrimMode +FixedWidthParser.TrimMode.Both = 2 -> FixedWidthParser.TrimMode +FixedWidthParser.TrimMode.Leading = 1 -> FixedWidthParser.TrimMode +FixedWidthParser.TrimMode.Trailing = 0 -> FixedWidthParser.TrimMode +override FixedWidthParser.Infrastructure.ColumnFormatOptions.GetHashCode() -> int +static FixedWidthParser.FixedWidth.Read(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthRecordEnumerable>! +static FixedWidthParser.FixedWidth.Read(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadAsync(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthAsyncRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadAsync(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthAsyncRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadFile(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadFile(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadFileAsync(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthAsyncRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadFileAsync(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthAsyncRecordEnumerable>! +static FixedWidthParser.Infrastructure.ColumnFormatOptions.operator !=(FixedWidthParser.Infrastructure.ColumnFormatOptions left, FixedWidthParser.Infrastructure.ColumnFormatOptions right) -> bool +static FixedWidthParser.Infrastructure.ColumnFormatOptions.operator ==(FixedWidthParser.Infrastructure.ColumnFormatOptions left, FixedWidthParser.Infrastructure.ColumnFormatOptions right) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.FormatConvert(TValue value, System.Span slice, in FixedWidthParser.FixedWidthContext context, TConverter converter, FixedWidthParser.Infrastructure.ColumnFormatOptions options, string! columnName) -> void +static FixedWidthParser.Infrastructure.FixedWidthRuntime.FormatString(string! value, System.Span slice, in FixedWidthParser.FixedWidthContext context, FixedWidthParser.Infrastructure.ColumnFormatOptions options, string! columnName) -> void +static FixedWidthParser.Infrastructure.FixedWidthRuntime.FormatUtf8Value(TValue value, System.Span slice, in FixedWidthParser.FixedWidthContext context, FixedWidthParser.Infrastructure.ColumnFormatOptions options, string! columnName) -> void +static FixedWidthParser.Infrastructure.FixedWidthRuntime.FormatValue(TValue value, System.Span slice, in FixedWidthParser.FixedWidthContext context, FixedWidthParser.Infrastructure.ColumnFormatOptions options, string! columnName) -> void +static FixedWidthParser.Infrastructure.FixedWidthRuntime.IsBlank(System.ReadOnlySpan column, byte trimChar) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.String(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> string! +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TrimColumn(System.ReadOnlySpan column, byte trimChar, FixedWidthParser.TrimMode trimMode) -> System.ReadOnlySpan +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryConvert(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, TConverter converter, out TValue value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryDateOnlyExact(System.ReadOnlySpan column, string! format, in FixedWidthParser.FixedWidthContext context, out System.DateOnly value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryDateTimeExact(System.ReadOnlySpan column, string! format, in FixedWidthParser.FixedWidthContext context, out System.DateTime value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryDateTimeOffsetExact(System.ReadOnlySpan column, string! format, in FixedWidthParser.FixedWidthContext context, out System.DateTimeOffset value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryDouble(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out double value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryFloat(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out float value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryParse(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out TValue value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryParseDecoded(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out TValue value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryTimeOnlyExact(System.ReadOnlySpan column, string! format, in FixedWidthParser.FixedWidthContext context, out System.TimeOnly value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool +~override FixedWidthParser.Infrastructure.ColumnFormatOptions.Equals(object obj) -> bool +~override FixedWidthParser.Infrastructure.ColumnFormatOptions.ToString() -> string diff --git a/src/FixedWidthParser/Readers/ParallelAggregator.cs b/src/FixedWidthParser/Readers/ParallelAggregator.cs index d94b16a..336f312 100644 --- a/src/FixedWidthParser/Readers/ParallelAggregator.cs +++ b/src/FixedWidthParser/Readers/ParallelAggregator.cs @@ -1,5 +1,6 @@ using System.Buffers; using System.Runtime.ExceptionServices; +using FixedWidthParser.Infrastructure; using Microsoft.Win32.SafeHandles; namespace FixedWidthParser.Readers diff --git a/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs b/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs index 7ab1285..9b848d2 100644 --- a/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs +++ b/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs @@ -2,6 +2,7 @@ using System.Runtime.ExceptionServices; using System.Runtime.InteropServices; using System.Threading.Channels; +using FixedWidthParser.Infrastructure; namespace FixedWidthParser.Readers { diff --git a/src/FixedWidthParser/Readers/ParseFailure.cs b/src/FixedWidthParser/Readers/ParseFailure.cs index 0494f4a..b043376 100644 --- a/src/FixedWidthParser/Readers/ParseFailure.cs +++ b/src/FixedWidthParser/Readers/ParseFailure.cs @@ -1,4 +1,5 @@ using System.Runtime.CompilerServices; +using FixedWidthParser.Infrastructure; namespace FixedWidthParser.Readers { diff --git a/src/FixedWidthParser/Writers/RecordFormatters.cs b/src/FixedWidthParser/Writers/RecordFormatters.cs index 52ba08a..50ab1a4 100644 --- a/src/FixedWidthParser/Writers/RecordFormatters.cs +++ b/src/FixedWidthParser/Writers/RecordFormatters.cs @@ -1,8 +1,10 @@ +using FixedWidthParser.Infrastructure; + namespace FixedWidthParser.Writers { /// /// Format strategy is specialized by, the write-side twin of - /// : implemented by types so each + /// : implemented by types so each /// (TModel, TFormatter) pair is compiled once and the call devirtualizes. /// internal interface IRecordFormatter diff --git a/tests/AotSmoke/Program.cs b/tests/AotSmoke/Program.cs index f947ca6..fcb08fb 100644 --- a/tests/AotSmoke/Program.cs +++ b/tests/AotSmoke/Program.cs @@ -1,6 +1,5 @@ using AotSmoke; using FixedWidthParser; -using FixedWidthParser.Attributes; var layout = FixedWidthLayout.Build(b => b .Factory(() => new Order()) diff --git a/tests/Benchmarks/Perf/AsyncReaderBenchmarks.cs b/tests/Benchmarks/Perf/AsyncReaderBenchmarks.cs index 35ceee0..8f8c08d 100644 --- a/tests/Benchmarks/Perf/AsyncReaderBenchmarks.cs +++ b/tests/Benchmarks/Perf/AsyncReaderBenchmarks.cs @@ -2,7 +2,6 @@ using BenchmarkDotNet.Attributes; using CommunityToolkit.HighPerformance.Buffers; using FixedWidthParser; -using FixedWidthParser.Readers; namespace Benchmarks.Perf { diff --git a/tests/Benchmarks/Perf/ComparisonBenchmarks.cs b/tests/Benchmarks/Perf/ComparisonBenchmarks.cs index f33ec83..38d5dad 100644 --- a/tests/Benchmarks/Perf/ComparisonBenchmarks.cs +++ b/tests/Benchmarks/Perf/ComparisonBenchmarks.cs @@ -1,9 +1,8 @@ using System.Text; using BenchmarkDotNet.Attributes; using FileHelpers; -using FlatFiles; using FixedWidthParser; -using FixedWidthParser.Readers; +using FlatFiles; using RecordParser.Builders.Reader; using RecordParser.Extensions; @@ -14,13 +13,13 @@ namespace Benchmarks.Perf public sealed class FileHelpersRecord { [FieldFixedLength(10)] - [FieldTrim(TrimMode.Both)] + [FieldTrim(FileHelpers.TrimMode.Both)] public string? Name; [FieldFixedLength(5)] - [FieldTrim(TrimMode.Both)] + [FieldTrim(FileHelpers.TrimMode.Both)] public int Age; [FieldFixedLength(10)] - [FieldTrim(TrimMode.Both)] + [FieldTrim(FileHelpers.TrimMode.Both)] public double Salary; } #pragma warning restore CA1051 diff --git a/tests/Benchmarks/Perf/ReaderBenchmarks.cs b/tests/Benchmarks/Perf/ReaderBenchmarks.cs index d0cd6e7..141cc28 100644 --- a/tests/Benchmarks/Perf/ReaderBenchmarks.cs +++ b/tests/Benchmarks/Perf/ReaderBenchmarks.cs @@ -2,7 +2,6 @@ using BenchmarkDotNet.Attributes; using CommunityToolkit.HighPerformance.Buffers; using FixedWidthParser; -using FixedWidthParser.Readers; namespace Benchmarks.Perf { diff --git a/tests/Benchmarks/Perf/ReaderPoolingBenchmarks.cs b/tests/Benchmarks/Perf/ReaderPoolingBenchmarks.cs index 76ecd3b..54de4b3 100644 --- a/tests/Benchmarks/Perf/ReaderPoolingBenchmarks.cs +++ b/tests/Benchmarks/Perf/ReaderPoolingBenchmarks.cs @@ -1,7 +1,6 @@ using BenchmarkDotNet.Attributes; using CommunityToolkit.HighPerformance.Buffers; using FixedWidthParser; -using FixedWidthParser.Readers; namespace Benchmarks.Perf { diff --git a/tests/Benchmarks/Perf/ReaderStartupBenchmarks.cs b/tests/Benchmarks/Perf/ReaderStartupBenchmarks.cs index 48c3bae..2bc8a41 100644 --- a/tests/Benchmarks/Perf/ReaderStartupBenchmarks.cs +++ b/tests/Benchmarks/Perf/ReaderStartupBenchmarks.cs @@ -1,6 +1,5 @@ using BenchmarkDotNet.Attributes; using FixedWidthParser; -using FixedWidthParser.Readers; namespace Benchmarks.Perf { diff --git a/tests/Benchmarks/Perf/SourceGenParserBenchmarks.cs b/tests/Benchmarks/Perf/SourceGenParserBenchmarks.cs index 0918408..18c86f5 100644 --- a/tests/Benchmarks/Perf/SourceGenParserBenchmarks.cs +++ b/tests/Benchmarks/Perf/SourceGenParserBenchmarks.cs @@ -1,7 +1,6 @@ using System.Text; using BenchmarkDotNet.Attributes; using FixedWidthParser; -using FixedWidthParser.Attributes; namespace Benchmarks.Perf { diff --git a/tests/Benchmarks/SampleModel.cs b/tests/Benchmarks/SampleModel.cs index 0198732..5dbcc17 100644 --- a/tests/Benchmarks/SampleModel.cs +++ b/tests/Benchmarks/SampleModel.cs @@ -1,4 +1,4 @@ -using FixedWidthParser.Attributes; +using FixedWidthParser; namespace Benchmarks { diff --git a/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs b/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs index f905aff..d37dce6 100644 --- a/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs +++ b/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs @@ -1,7 +1,7 @@ using System.Collections.Immutable; using CommunityToolkit.HighPerformance.Buffers; -using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis; namespace FixedWidthParser.Generator.Tests { @@ -61,7 +61,7 @@ public void EveryColumnKind_CompilesCleanly() var result = Run(""" public readonly record struct Cents(long Value); - public sealed class CentsConverter : FixedWidthParser.Processors.IFixedWidthConverter + public sealed class CentsConverter : FixedWidthParser.IFixedWidthConverter { public bool TryParse(System.ReadOnlySpan field, in FixedWidthContext context, out Cents value) { value = default; return true; } public bool TryFormat(Cents value, System.Span destination, in FixedWidthContext context, out int written) { written = 0; return true; } @@ -83,7 +83,7 @@ public sealed class CentsConverter : FixedWidthParser.Processors.IFixedWidthConv var generated = Assert.Single(result.GeneratedSources); Assert.Contains("FixedWidthRuntime.IsBlank(line.Slice(5, 5), (byte)' ')", generated, StringComparison.Ordinal); Assert.Contains("TryDateOnlyExact(line.Slice(10, 8), \"yyyyMMdd\", context", generated, StringComparison.Ordinal); - Assert.Contains("(byte)'0', global::FixedWidthParser.Attributes.TrimMode.Leading", generated, StringComparison.Ordinal); + Assert.Contains("(byte)'0', global::FixedWidthParser.TrimMode.Leading", generated, StringComparison.Ordinal); } [Fact] @@ -140,7 +140,7 @@ public void ColumnNotFormattable_ReportsFwp010_AndEmitsNothing() public void ConverterForAnotherType_ReportsFwp008() { var result = Run(""" - public sealed class IntConverter : FixedWidthParser.Processors.IFixedWidthConverter + public sealed class IntConverter : FixedWidthParser.IFixedWidthConverter { public bool TryParse(System.ReadOnlySpan field, in FixedWidthContext context, out int value) { value = 0; return true; } public bool TryFormat(int value, System.Span destination, in FixedWidthContext context, out int written) { written = 0; return true; } @@ -417,8 +417,7 @@ private static CSharpCompilation CreateCompilation(string modelSource, CSharpPar { string source = """ using FixedWidthParser; - using FixedWidthParser.Attributes; - + namespace GeneratorSmoke; """ + modelSource; @@ -435,8 +434,7 @@ private static GeneratorRunResult Run(string modelSource) { string source = """ using FixedWidthParser; - using FixedWidthParser.Attributes; - + namespace GeneratorSmoke; """ + modelSource; diff --git a/tests/FixedWidthParser.Tests/ConverterAttributeTests.cs b/tests/FixedWidthParser.Tests/ConverterAttributeTests.cs index 288be68..d521278 100644 --- a/tests/FixedWidthParser.Tests/ConverterAttributeTests.cs +++ b/tests/FixedWidthParser.Tests/ConverterAttributeTests.cs @@ -3,7 +3,7 @@ namespace FixedWidthParser.Tests { /// - /// : a per-property converter that takes + /// : a per-property converter that takes /// priority over the built-in parsers, on the reflection and generated paths, for parse and write, /// plus the build-time mismatch check. /// diff --git a/tests/FixedWidthParser.Tests/CoverageTests.cs b/tests/FixedWidthParser.Tests/CoverageTests.cs index c22acb8..a95ca9e 100644 --- a/tests/FixedWidthParser.Tests/CoverageTests.cs +++ b/tests/FixedWidthParser.Tests/CoverageTests.cs @@ -1,7 +1,6 @@ using System.Globalization; using System.Text; -using FixedWidthParser.Attributes; -using FixedWidthParser.Formatters; +using FixedWidthParser.Infrastructure; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests diff --git a/tests/FixedWidthParser.Tests/DoubleFormatTests.cs b/tests/FixedWidthParser.Tests/DoubleFormatTests.cs index 6222297..82aaca6 100644 --- a/tests/FixedWidthParser.Tests/DoubleFormatTests.cs +++ b/tests/FixedWidthParser.Tests/DoubleFormatTests.cs @@ -1,8 +1,7 @@ using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Text; -using FixedWidthParser.Attributes; -using FixedWidthParser.Formatters; +using FixedWidthParser.Infrastructure; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests diff --git a/tests/FixedWidthParser.Tests/FluentLayoutTests.cs b/tests/FixedWidthParser.Tests/FluentLayoutTests.cs index 2a19e63..2546938 100644 --- a/tests/FixedWidthParser.Tests/FluentLayoutTests.cs +++ b/tests/FixedWidthParser.Tests/FluentLayoutTests.cs @@ -1,5 +1,4 @@ using System.Text; -using FixedWidthParser.Attributes; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests diff --git a/tests/FixedWidthParser.Tests/GeneratedModels.cs b/tests/FixedWidthParser.Tests/GeneratedModels.cs index d5d4432..6d17eaf 100644 --- a/tests/FixedWidthParser.Tests/GeneratedModels.cs +++ b/tests/FixedWidthParser.Tests/GeneratedModels.cs @@ -1,5 +1,3 @@ -using FixedWidthParser.Attributes; - namespace FixedWidthParser.Tests { // Models that opt into the source-generated parser and writer. Each mirrors a reflection-based diff --git a/tests/FixedWidthParser.Tests/GeneratedReaderTests.cs b/tests/FixedWidthParser.Tests/GeneratedReaderTests.cs index 7523550..6bd316a 100644 --- a/tests/FixedWidthParser.Tests/GeneratedReaderTests.cs +++ b/tests/FixedWidthParser.Tests/GeneratedReaderTests.cs @@ -1,6 +1,6 @@ using System.Text; using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Readers; +using FixedWidthParser.Infrastructure; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests diff --git a/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs b/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs index 631e80f..be69a8a 100644 --- a/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs +++ b/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs @@ -1,6 +1,5 @@ using System.Collections.Concurrent; using System.Text; -using FixedWidthParser.Readers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests diff --git a/tests/FixedWidthParser.Tests/PropertyRoundTripTests.cs b/tests/FixedWidthParser.Tests/PropertyRoundTripTests.cs index 8d73154..58f06d6 100644 --- a/tests/FixedWidthParser.Tests/PropertyRoundTripTests.cs +++ b/tests/FixedWidthParser.Tests/PropertyRoundTripTests.cs @@ -1,5 +1,4 @@ using CsCheck; -using FixedWidthParser.Attributes; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests diff --git a/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs b/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs index 24a0e0c..065edef 100644 --- a/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs +++ b/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs @@ -1,5 +1,4 @@ using System.Text; -using FixedWidthParser.Readers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests diff --git a/tests/FixedWidthParser.Tests/ReaderTests.cs b/tests/FixedWidthParser.Tests/ReaderTests.cs index bf94d2a..94ecb3f 100644 --- a/tests/FixedWidthParser.Tests/ReaderTests.cs +++ b/tests/FixedWidthParser.Tests/ReaderTests.cs @@ -1,5 +1,5 @@ using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Readers; +using FixedWidthParser.Infrastructure; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests diff --git a/tests/FixedWidthParser.Tests/TestModels.cs b/tests/FixedWidthParser.Tests/TestModels.cs index c068350..e5fbed6 100644 --- a/tests/FixedWidthParser.Tests/TestModels.cs +++ b/tests/FixedWidthParser.Tests/TestModels.cs @@ -1,7 +1,6 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime.InteropServices; -using FixedWidthParser.Attributes; -using FixedWidthParser.Processors; +using FixedWidthParser.Infrastructure; namespace FixedWidthParser.Tests { diff --git a/tests/FixedWidthParser.Tests/WriterAsyncTests.cs b/tests/FixedWidthParser.Tests/WriterAsyncTests.cs index 8121612..6009b21 100644 --- a/tests/FixedWidthParser.Tests/WriterAsyncTests.cs +++ b/tests/FixedWidthParser.Tests/WriterAsyncTests.cs @@ -1,5 +1,4 @@ using System.Text; -using FixedWidthParser.Readers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests From 9eee4acdfdbdc10f5956cede9a749dc6c5ee65ca Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 23 Oct 2026 23:15:11 -0300 Subject: [PATCH 27/85] refactor!: name parameters the same way across the API Aggregate methods take cancellationToken, like the write methods, instead of ct. IFixedWidthConverter takes column and bytesWritten, like the rest of the library, instead of field and written. Named arguments make parameter names part of the contract, so they are settled before 2.0 ships. Co-Authored-By: Claude Opus 5.5 --- src/FixedWidthParser/FixedWidth.cs | 16 ++++++------ src/FixedWidthParser/IFixedWidthConverter.cs | 10 +++---- src/FixedWidthParser/PublicAPI.Unshipped.txt | 12 ++++----- .../FixedWidthModelGeneratorTests.cs | 8 +++--- .../FluentLayoutTests.cs | 2 +- .../FixedWidthParser.Tests/LayoutReadTests.cs | 4 +-- .../ParallelAggregateTests.cs | 4 +-- tests/FixedWidthParser.Tests/TestModels.cs | 26 +++++++++---------- 8 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/FixedWidthParser/FixedWidth.cs b/src/FixedWidthParser/FixedWidth.cs index 03fc55e..25eaec5 100644 --- a/src/FixedWidthParser/FixedWidth.cs +++ b/src/FixedWidthParser/FixedWidth.cs @@ -128,13 +128,13 @@ public static FixedWidthAsyncRecordEnumerable> /// possibly concurrently, and the first failure in file order is the one thrown. /// public static Task AggregateFileAsync( - string path, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken ct = default) + string path, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken cancellationToken = default) where TModel : IFixedWidthModel where TAccumulator : IRecordAccumulator, new() { ArgumentException.ThrowIfNullOrEmpty(path); return ParallelAggregator.RunAsync, TAccumulator>( - default, path, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), ct); + default, path, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), cancellationToken); } /// @@ -144,12 +144,12 @@ public static Task AggregateFileAsync( /// Carries the same contract as . /// has a column without a setter. public static Task AggregateFileAsync( - string path, FixedWidthLayout layout, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken ct = default) + string path, FixedWidthLayout layout, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken cancellationToken = default) where TAccumulator : IRecordAccumulator, new() { ArgumentException.ThrowIfNullOrEmpty(path); return ParallelAggregator.RunAsync, TAccumulator>( - ReadableLayout(layout), path, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), ct); + ReadableLayout(layout), path, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), cancellationToken); } /// @@ -158,13 +158,13 @@ public static Task AggregateFileAsync( /// /// Carries the same contract as . public static Task AggregateAsync( - Stream stream, bool leaveOpen = false, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken ct = default) + Stream stream, bool leaveOpen = false, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken cancellationToken = default) where TModel : IFixedWidthModel where TAccumulator : IRecordAccumulator, new() { ArgumentNullException.ThrowIfNull(stream); return ParallelStreamAggregator.RunAsync, TAccumulator>( - default, stream, leaveOpen, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), ct); + default, stream, leaveOpen, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), cancellationToken); } /// @@ -175,12 +175,12 @@ public static Task AggregateAsync( /// Carries the same contract as . /// has a column without a setter. public static Task AggregateAsync( - Stream stream, FixedWidthLayout layout, bool leaveOpen = false, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken ct = default) + Stream stream, FixedWidthLayout layout, bool leaveOpen = false, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken cancellationToken = default) where TAccumulator : IRecordAccumulator, new() { ArgumentNullException.ThrowIfNull(stream); return ParallelStreamAggregator.RunAsync, TAccumulator>( - ReadableLayout(layout), stream, leaveOpen, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), ct); + ReadableLayout(layout), stream, leaveOpen, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), cancellationToken); } diff --git a/src/FixedWidthParser/IFixedWidthConverter.cs b/src/FixedWidthParser/IFixedWidthConverter.cs index 60c3f41..9aba327 100644 --- a/src/FixedWidthParser/IFixedWidthConverter.cs +++ b/src/FixedWidthParser/IFixedWidthConverter.cs @@ -1,9 +1,9 @@ namespace FixedWidthParser { /// - /// Per-property custom conversion for a fixed-width column, wired via - /// FixedColumnAttribute.Converter. A single instance is created once (when the owning - /// model's parser/writer is built) and reused for every row, so implementations must be stateless. + /// Custom conversion for a fixed-width column, wired through + /// or a fluent ConvertedColumn. One instance serves every line of the layout, on every thread + /// that reads or writes in parallel, so implementations must be stateless. /// One interface covers both directions: a fixed-width column always round-trips. /// /// A later version never adds a member to this interface; new settings arrive on . @@ -13,7 +13,7 @@ public interface IFixedWidthConverter /// Parses the already-sliced, trimmed column bytes. Decode text with /// . Returns to reject the line. /// - bool TryParse(ReadOnlySpan field, in FixedWidthContext context, out T value); + bool TryParse(ReadOnlySpan column, in FixedWidthContext context, out T value); /// /// Formats into , encoding text with @@ -21,6 +21,6 @@ public interface IFixedWidthConverter /// when is too small; the writer retries /// with a larger buffer. /// - bool TryFormat(T value, Span destination, in FixedWidthContext context, out int written); + bool TryFormat(T value, Span destination, in FixedWidthContext context, out int bytesWritten); } } diff --git a/src/FixedWidthParser/PublicAPI.Unshipped.txt b/src/FixedWidthParser/PublicAPI.Unshipped.txt index 419d9ac..5bafecf 100644 --- a/src/FixedWidthParser/PublicAPI.Unshipped.txt +++ b/src/FixedWidthParser/PublicAPI.Unshipped.txt @@ -59,8 +59,6 @@ FixedWidthParser.FixedWidthParallelOptions.FixedWidthParallelOptions() -> void FixedWidthParser.IRecordAccumulator FixedWidthParser.IRecordAccumulator.Add(in TModel model) -> void FixedWidthParser.IRecordAccumulator.Merge(TSelf following) -> void -static FixedWidthParser.FixedWidth.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! -static FixedWidthParser.FixedWidth.AggregateAsync(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! FixedWidthParser.ColumnGetter FixedWidthParser.ColumnSetter FixedWidthParser.FixedWidthLayout @@ -70,8 +68,6 @@ static FixedWidthParser.FixedWidth.TryParse(System.ReadOnlySpan li static FixedWidthParser.FixedWidthLayout.FromAttributes() -> FixedWidthParser.FixedWidthLayout! virtual FixedWidthParser.ColumnGetter.Invoke(in TModel model) -> TValue virtual FixedWidthParser.ColumnSetter.Invoke(ref TModel model, TValue value) -> void -static FixedWidthParser.FixedWidth.AggregateAsync(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! -static FixedWidthParser.FixedWidth.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! static FixedWidthParser.FixedWidth.Write(System.IO.Stream! stream, in TModel model, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthWriterOptions? options = null) -> void static FixedWidthParser.FixedWidth.Write(System.IO.Stream! stream, in TModel model, FixedWidthParser.FixedWidthWriterOptions? options = null) -> void static FixedWidthParser.FixedWidth.WriteAsync(System.IO.Stream! stream, TModel model, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthWriterOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! @@ -128,8 +124,6 @@ FixedWidthParser.FixedWidthColumnOptions.TrimMode.get -> FixedWidthParser.TrimMo FixedWidthParser.FixedWidthLayoutBuilder.ConvertedColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.IFixedWidthConverter! converter, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! FixedWidthParser.FixedWidthLayoutBuilder.NullableConvertedColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.IFixedWidthConverter! converter, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! FixedWidthParser.IFixedWidthConverter -FixedWidthParser.IFixedWidthConverter.TryFormat(T value, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int written) -> bool -FixedWidthParser.IFixedWidthConverter.TryParse(System.ReadOnlySpan field, in FixedWidthParser.FixedWidthContext context, out T value) -> bool FixedWidthParser.Infrastructure.AsyncRecordEnumerator FixedWidthParser.Infrastructure.AsyncRecordEnumerator.Current.get -> TModel FixedWidthParser.Infrastructure.AsyncRecordEnumerator.DisposeAsync() -> System.Threading.Tasks.ValueTask @@ -204,3 +198,9 @@ static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryParseDecoded static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryTimeOnlyExact(System.ReadOnlySpan column, string! format, in FixedWidthParser.FixedWidthContext context, out System.TimeOnly value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool ~override FixedWidthParser.Infrastructure.ColumnFormatOptions.Equals(object obj) -> bool ~override FixedWidthParser.Infrastructure.ColumnFormatOptions.ToString() -> string +FixedWidthParser.IFixedWidthConverter.TryFormat(T value, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int bytesWritten) -> bool +FixedWidthParser.IFixedWidthConverter.TryParse(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out T value) -> bool +static FixedWidthParser.FixedWidth.AggregateAsync(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.AggregateAsync(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! diff --git a/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs b/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs index d37dce6..9fd865b 100644 --- a/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs +++ b/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs @@ -63,8 +63,8 @@ public void EveryColumnKind_CompilesCleanly() public sealed class CentsConverter : FixedWidthParser.IFixedWidthConverter { - public bool TryParse(System.ReadOnlySpan field, in FixedWidthContext context, out Cents value) { value = default; return true; } - public bool TryFormat(Cents value, System.Span destination, in FixedWidthContext context, out int written) { written = 0; return true; } + public bool TryParse(System.ReadOnlySpan column, in FixedWidthContext context, out Cents value) { value = default; return true; } + public bool TryFormat(Cents value, System.Span destination, in FixedWidthContext context, out int bytesWritten) { bytesWritten = 0; return true; } } public readonly partial record struct AllKinds : IFixedWidthModel @@ -142,8 +142,8 @@ public void ConverterForAnotherType_ReportsFwp008() var result = Run(""" public sealed class IntConverter : FixedWidthParser.IFixedWidthConverter { - public bool TryParse(System.ReadOnlySpan field, in FixedWidthContext context, out int value) { value = 0; return true; } - public bool TryFormat(int value, System.Span destination, in FixedWidthContext context, out int written) { written = 0; return true; } + public bool TryParse(System.ReadOnlySpan column, in FixedWidthContext context, out int value) { value = 0; return true; } + public bool TryFormat(int value, System.Span destination, in FixedWidthContext context, out int bytesWritten) { bytesWritten = 0; return true; } } public readonly partial record struct MismatchModel : IFixedWidthModel diff --git a/tests/FixedWidthParser.Tests/FluentLayoutTests.cs b/tests/FixedWidthParser.Tests/FluentLayoutTests.cs index 2546938..2b43898 100644 --- a/tests/FixedWidthParser.Tests/FluentLayoutTests.cs +++ b/tests/FixedWidthParser.Tests/FluentLayoutTests.cs @@ -270,7 +270,7 @@ public async Task Fluent_AggregateFileAsync_MatchesSequential() var layout = Person(); var parallel = await FixedWidth.AggregateFileAsync( - _path, layout, parallel: new FixedWidthParallelOptions { DegreeOfParallelism = 4 }, ct: TestContext.Current.CancellationToken); + _path, layout, parallel: new FixedWidthParallelOptions { DegreeOfParallelism = 4 }, cancellationToken: TestContext.Current.CancellationToken); Assert.Equal(FixedWidth.ReadFile(_path, layout).Select(p => p.Age), parallel.Values); Assert.Equal(100_000, parallel.Values.Count); diff --git a/tests/FixedWidthParser.Tests/LayoutReadTests.cs b/tests/FixedWidthParser.Tests/LayoutReadTests.cs index 293a1c7..1774d1d 100644 --- a/tests/FixedWidthParser.Tests/LayoutReadTests.cs +++ b/tests/FixedWidthParser.Tests/LayoutReadTests.cs @@ -52,7 +52,7 @@ public async Task AggregateFileAsync_WithLayout_MatchesSequentialFold() var layout = FixedWidthLayout.FromAttributes(); var parallel = await FixedWidth.AggregateFileAsync( - _path, layout, parallel: new FixedWidthParallelOptions { DegreeOfParallelism = 4 }, ct: TestContext.Current.CancellationToken); + _path, layout, parallel: new FixedWidthParallelOptions { DegreeOfParallelism = 4 }, cancellationToken: TestContext.Current.CancellationToken); Assert.Equal(SequentialAges(layout), parallel.Values); } @@ -65,7 +65,7 @@ public async Task AggregateAsync_WithLayout_MatchesSequentialFold() using var stream = new MemoryStream(File.ReadAllBytes(_path)); var parallel = await FixedWidth.AggregateAsync( - stream, layout, parallel: new FixedWidthParallelOptions { DegreeOfParallelism = 4 }, ct: TestContext.Current.CancellationToken); + stream, layout, parallel: new FixedWidthParallelOptions { DegreeOfParallelism = 4 }, cancellationToken: TestContext.Current.CancellationToken); Assert.Equal(SequentialAges(layout), parallel.Values); } diff --git a/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs b/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs index be69a8a..c6bc67b 100644 --- a/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs +++ b/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs @@ -109,8 +109,8 @@ private Task AggregateGenerated(Source source, FixedWidthParallelOption var ct = TestContext.Current.CancellationToken; return source switch { - Source.File => FixedWidth.AggregateFileAsync(_path, parallel: parallel, ct: ct), - _ => FixedWidth.AggregateAsync(File.OpenRead(_path), parallel: parallel, ct: ct), + Source.File => FixedWidth.AggregateFileAsync(_path, parallel: parallel, cancellationToken: ct), + _ => FixedWidth.AggregateAsync(File.OpenRead(_path), parallel: parallel, cancellationToken: ct), }; } diff --git a/tests/FixedWidthParser.Tests/TestModels.cs b/tests/FixedWidthParser.Tests/TestModels.cs index e5fbed6..f44428c 100644 --- a/tests/FixedWidthParser.Tests/TestModels.cs +++ b/tests/FixedWidthParser.Tests/TestModels.cs @@ -10,9 +10,9 @@ namespace FixedWidthParser.Tests /// Stateless for : stores cents as a plain integer. public sealed class CentsConverter : IFixedWidthConverter { - public bool TryParse(ReadOnlySpan field, in FixedWidthContext context, out CentsValue value) + public bool TryParse(ReadOnlySpan column, in FixedWidthContext context, out CentsValue value) { - if (!long.TryParse(field, context.FormatProvider, out long cents)) + if (!long.TryParse(column, context.FormatProvider, out long cents)) { value = default; return false; @@ -21,23 +21,23 @@ public bool TryParse(ReadOnlySpan field, in FixedWidthContext context, out return true; } - public bool TryFormat(CentsValue value, Span destination, in FixedWidthContext context, out int written) + public bool TryFormat(CentsValue value, Span destination, in FixedWidthContext context, out int bytesWritten) { - return value.Cents.TryFormat(destination, out written, provider: context.FormatProvider); + return value.Cents.TryFormat(destination, out bytesWritten, provider: context.FormatProvider); } } /// Converter for int, attached to a string column to prove the mismatch is rejected. public sealed class IntOnlyConverter : IFixedWidthConverter { - public bool TryParse(ReadOnlySpan field, in FixedWidthContext context, out int value) + public bool TryParse(ReadOnlySpan column, in FixedWidthContext context, out int value) { - return int.TryParse(field, context.FormatProvider, out value); + return int.TryParse(column, context.FormatProvider, out value); } - public bool TryFormat(int value, Span destination, in FixedWidthContext context, out int written) + public bool TryFormat(int value, Span destination, in FixedWidthContext context, out int bytesWritten) { - return value.TryFormat(destination, out written, provider: context.FormatProvider); + return value.TryFormat(destination, out bytesWritten, provider: context.FormatProvider); } } /// Single custom-converted column, exercised by both the reflection parser/writer and the generator. @@ -312,21 +312,21 @@ public readonly record struct WideUtf8NumericModel /// Converter for ; emits N 'x' bytes on format. public sealed class BigConverter : IFixedWidthConverter { - public bool TryParse(ReadOnlySpan field, in FixedWidthContext context, out BigValue value) + public bool TryParse(ReadOnlySpan column, in FixedWidthContext context, out BigValue value) { - value = new BigValue(field.Trim((byte)' ').Length); + value = new BigValue(column.Trim((byte)' ').Length); return true; } - public bool TryFormat(BigValue value, Span destination, in FixedWidthContext context, out int written) + public bool TryFormat(BigValue value, Span destination, in FixedWidthContext context, out int bytesWritten) { if (destination.Length < value.Length) { - written = 0; + bytesWritten = 0; return false; } destination[..value.Length].Fill((byte)'x'); - written = value.Length; + bytesWritten = value.Length; return true; } } From cb5ee86efaf9813073836441d16b105d9348ffce Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 23 Oct 2026 23:17:22 -0300 Subject: [PATCH 28/85] refactor!: fold FixedWidthParallelOptions into FixedWidthReaderOptions DegreeOfParallelism is now a property of FixedWidthReaderOptions, ignored by reads that are not aggregations. The aggregate methods lose their parallel parameter, and a later parallelism setting is one more property instead of a new type or parameter. BREAKING CHANGE: pass new FixedWidthReaderOptions { DegreeOfParallelism = n } instead of new FixedWidthParallelOptions { DegreeOfParallelism = n }. Co-Authored-By: Claude Opus 5.5 --- src/FixedWidthParser/FixedWidth.cs | 22 ++++----- .../FixedWidthParallelOptions.cs | 39 --------------- .../FixedWidthReaderOptions.cs | 26 ++++++++++ src/FixedWidthParser/PublicAPI.Unshipped.txt | 14 +++--- .../Readers/ParallelAggregator.cs | 20 ++++---- .../Readers/ParallelStreamAggregator.cs | 12 ++--- .../FluentLayoutTests.cs | 2 +- .../FixedWidthParser.Tests/LayoutReadTests.cs | 4 +- .../ParallelAggregateTests.cs | 48 ++++++++++--------- 9 files changed, 88 insertions(+), 99 deletions(-) delete mode 100644 src/FixedWidthParser/FixedWidthParallelOptions.cs diff --git a/src/FixedWidthParser/FixedWidth.cs b/src/FixedWidthParser/FixedWidth.cs index 25eaec5..4a5ca31 100644 --- a/src/FixedWidthParser/FixedWidth.cs +++ b/src/FixedWidthParser/FixedWidth.cs @@ -128,43 +128,43 @@ public static FixedWidthAsyncRecordEnumerable> /// possibly concurrently, and the first failure in file order is the one thrown. /// public static Task AggregateFileAsync( - string path, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken cancellationToken = default) + string path, FixedWidthReaderOptions? options = null, CancellationToken cancellationToken = default) where TModel : IFixedWidthModel where TAccumulator : IRecordAccumulator, new() { ArgumentException.ThrowIfNullOrEmpty(path); return ParallelAggregator.RunAsync, TAccumulator>( - default, path, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), cancellationToken); + default, path, FixedWidthReaderOptions.Validate(options), cancellationToken); } /// /// Reads a file through across several threads and folds every record into /// a , one instance per partition, merged in file order. /// - /// Carries the same contract as . + /// Carries the same contract as . /// has a column without a setter. public static Task AggregateFileAsync( - string path, FixedWidthLayout layout, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken cancellationToken = default) + string path, FixedWidthLayout layout, FixedWidthReaderOptions? options = null, CancellationToken cancellationToken = default) where TAccumulator : IRecordAccumulator, new() { ArgumentException.ThrowIfNullOrEmpty(path); return ParallelAggregator.RunAsync, TAccumulator>( - ReadableLayout(layout), path, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), cancellationToken); + ReadableLayout(layout), path, FixedWidthReaderOptions.Validate(options), cancellationToken); } /// /// Reads a stream once and folds every record into a across /// several threads. controls whether the stream is disposed afterwards. /// - /// Carries the same contract as . + /// Carries the same contract as . public static Task AggregateAsync( - Stream stream, bool leaveOpen = false, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken cancellationToken = default) + Stream stream, bool leaveOpen = false, FixedWidthReaderOptions? options = null, CancellationToken cancellationToken = default) where TModel : IFixedWidthModel where TAccumulator : IRecordAccumulator, new() { ArgumentNullException.ThrowIfNull(stream); return ParallelStreamAggregator.RunAsync, TAccumulator>( - default, stream, leaveOpen, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), cancellationToken); + default, stream, leaveOpen, FixedWidthReaderOptions.Validate(options), cancellationToken); } /// @@ -172,15 +172,15 @@ public static Task AggregateAsync( /// across several threads. controls /// whether the stream is disposed afterwards. /// - /// Carries the same contract as . + /// Carries the same contract as . /// has a column without a setter. public static Task AggregateAsync( - Stream stream, FixedWidthLayout layout, bool leaveOpen = false, FixedWidthReaderOptions? options = null, FixedWidthParallelOptions? parallel = null, CancellationToken cancellationToken = default) + Stream stream, FixedWidthLayout layout, bool leaveOpen = false, FixedWidthReaderOptions? options = null, CancellationToken cancellationToken = default) where TAccumulator : IRecordAccumulator, new() { ArgumentNullException.ThrowIfNull(stream); return ParallelStreamAggregator.RunAsync, TAccumulator>( - ReadableLayout(layout), stream, leaveOpen, FixedWidthReaderOptions.Validate(options), FixedWidthParallelOptions.Validate(parallel), cancellationToken); + ReadableLayout(layout), stream, leaveOpen, FixedWidthReaderOptions.Validate(options), cancellationToken); } diff --git a/src/FixedWidthParser/FixedWidthParallelOptions.cs b/src/FixedWidthParser/FixedWidthParallelOptions.cs deleted file mode 100644 index 09d59e3..0000000 --- a/src/FixedWidthParser/FixedWidthParallelOptions.cs +++ /dev/null @@ -1,39 +0,0 @@ -namespace FixedWidthParser -{ - /// Options for reading a file across several threads with AggregateFileAsync. - public sealed class FixedWidthParallelOptions - { - internal static readonly FixedWidthParallelOptions Default = new(); - - /// - /// The maximum number of reading threads. 0, the default, means - /// ; 1 reads the file as one partition. - /// - public int DegreeOfParallelism { get; init; } - - // Bytes per partition; 0 sizes partitions from the file length. Tests set it to cut every boundary. - internal int ChunkSize { get; init; } - - internal int Workers - { - get - { - if (DegreeOfParallelism == 0) - { - return Environment.ProcessorCount; - } - return DegreeOfParallelism; - } - } - - internal static FixedWidthParallelOptions Validate(FixedWidthParallelOptions? options) - { - options ??= Default; - if (options.DegreeOfParallelism < 0) - { - throw new ArgumentOutOfRangeException(nameof(options), options.DegreeOfParallelism, "DegreeOfParallelism must not be negative."); - } - return options; - } - } -} diff --git a/src/FixedWidthParser/FixedWidthReaderOptions.cs b/src/FixedWidthParser/FixedWidthReaderOptions.cs index 1b195bc..9db4697 100644 --- a/src/FixedWidthParser/FixedWidthReaderOptions.cs +++ b/src/FixedWidthParser/FixedWidthReaderOptions.cs @@ -40,6 +40,28 @@ public sealed class FixedWidthReaderOptions /// public Func? OnParseError { get; init; } + /// + /// The maximum number of threads AggregateFileAsync and AggregateAsync read with. + /// 0, the default, means ; 1 reads in one pass. + /// Other reads ignore it. + /// + public int DegreeOfParallelism { get; init; } + + // Bytes per partition or chunk; 0 sizes them from the source. Tests set it to cut every boundary. + internal int ChunkSize { get; init; } + + internal int Workers + { + get + { + if (DegreeOfParallelism == 0) + { + return Environment.ProcessorCount; + } + return DegreeOfParallelism; + } + } + internal FixedWidthContext Context => new() { FormatProvider = FormatProvider!, Encoding = Encoding!, StringPool = StringPool }; internal bool SkipsByteOrderMark => Encoding is null || Encoding.CodePage == Encoding.UTF8.CodePage; @@ -55,6 +77,10 @@ internal static FixedWidthReaderOptions Validate(FixedWidthReaderOptions? option { throw new ArgumentOutOfRangeException(nameof(options), options.MaxLineLength, "MaxLineLength must be at least 1."); } + if (options.DegreeOfParallelism < 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.DegreeOfParallelism, "DegreeOfParallelism must not be negative."); + } return options; } } diff --git a/src/FixedWidthParser/PublicAPI.Unshipped.txt b/src/FixedWidthParser/PublicAPI.Unshipped.txt index 5bafecf..3526499 100644 --- a/src/FixedWidthParser/PublicAPI.Unshipped.txt +++ b/src/FixedWidthParser/PublicAPI.Unshipped.txt @@ -52,10 +52,6 @@ static FixedWidthParser.FixedWidthContext.operator !=(FixedWidthParser.FixedWidt static FixedWidthParser.FixedWidthContext.operator ==(FixedWidthParser.FixedWidthContext left, FixedWidthParser.FixedWidthContext right) -> bool ~override FixedWidthParser.FixedWidthContext.Equals(object obj) -> bool ~override FixedWidthParser.FixedWidthContext.ToString() -> string -FixedWidthParser.FixedWidthParallelOptions -FixedWidthParser.FixedWidthParallelOptions.DegreeOfParallelism.get -> int -FixedWidthParser.FixedWidthParallelOptions.DegreeOfParallelism.init -> void -FixedWidthParser.FixedWidthParallelOptions.FixedWidthParallelOptions() -> void FixedWidthParser.IRecordAccumulator FixedWidthParser.IRecordAccumulator.Add(in TModel model) -> void FixedWidthParser.IRecordAccumulator.Merge(TSelf following) -> void @@ -200,7 +196,9 @@ static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryTimeOnlyExact(System ~override FixedWidthParser.Infrastructure.ColumnFormatOptions.ToString() -> string FixedWidthParser.IFixedWidthConverter.TryFormat(T value, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int bytesWritten) -> bool FixedWidthParser.IFixedWidthConverter.TryParse(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out T value) -> bool -static FixedWidthParser.FixedWidth.AggregateAsync(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! -static FixedWidthParser.FixedWidth.AggregateAsync(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! -static FixedWidthParser.FixedWidth.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! -static FixedWidthParser.FixedWidth.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null, FixedWidthParser.FixedWidthParallelOptions? parallel = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +FixedWidthParser.FixedWidthReaderOptions.DegreeOfParallelism.get -> int +FixedWidthParser.FixedWidthReaderOptions.DegreeOfParallelism.init -> void +static FixedWidthParser.FixedWidth.AggregateAsync(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.AggregateAsync(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! diff --git a/src/FixedWidthParser/Readers/ParallelAggregator.cs b/src/FixedWidthParser/Readers/ParallelAggregator.cs index 336f312..5293d93 100644 --- a/src/FixedWidthParser/Readers/ParallelAggregator.cs +++ b/src/FixedWidthParser/Readers/ParallelAggregator.cs @@ -20,15 +20,15 @@ internal static class ParallelAggregator private const int LinesPerCancellationCheck = 4096; internal static async Task RunAsync( - TParser parser, string path, FixedWidthReaderOptions options, FixedWidthParallelOptions parallel, CancellationToken ct) + TParser parser, string path, FixedWidthReaderOptions options, CancellationToken ct) where TParser : struct, IRecordLineParser where TAccumulator : IRecordAccumulator, new() { using var handle = File.OpenHandle(path, FileMode.Open, FileAccess.Read, FileShare.Read); - var plan = new ChunkPlan(RandomAccess.GetLength(handle), parallel); + var plan = new ChunkPlan(RandomAccess.GetLength(handle), options); var results = new TAccumulator[plan.Count]; var failures = new ExceptionDispatchInfo?[plan.Count]; - var workers = new Task[Math.Min(parallel.Workers, plan.Count)]; + var workers = new Task[Math.Min(options.Workers, plan.Count)]; foreach (ref var worker in workers.AsSpan()) { worker = Task.Run(() => Drain(parser, handle, plan, options, results, failures, ct), ct); @@ -125,10 +125,10 @@ private sealed class ChunkPlan private readonly long _chunkSize; private int _next = -1; - internal ChunkPlan(long length, FixedWidthParallelOptions parallel) + internal ChunkPlan(long length, FixedWidthReaderOptions options) { _length = length; - _chunkSize = ChunkSize(length, parallel); + _chunkSize = ChunkSize(length, options); Count = (int)Math.Max(1, (length + _chunkSize - 1) / _chunkSize); } @@ -150,17 +150,17 @@ internal bool TryTakeNext(out int index) return index < Count; } - private static long ChunkSize(long length, FixedWidthParallelOptions parallel) + private static long ChunkSize(long length, FixedWidthReaderOptions options) { - if (parallel.ChunkSize > 0) + if (options.ChunkSize > 0) { - return parallel.ChunkSize; + return options.ChunkSize; } - if (parallel.Workers == 1) + if (options.Workers == 1) { return Math.Max(1, length); } - return Math.Clamp(length / ((long)parallel.Workers * ChunksPerWorker), MinChunkBytes, MaxChunkBytes); + return Math.Clamp(length / ((long)options.Workers * ChunksPerWorker), MinChunkBytes, MaxChunkBytes); } } diff --git a/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs b/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs index 9b848d2..94c4007 100644 --- a/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs +++ b/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs @@ -26,17 +26,17 @@ internal static class ParallelStreamAggregator /// Folds , disposing it afterwards unless . internal static async Task RunAsync( - TParser parser, Stream source, bool leaveOpen, FixedWidthReaderOptions options, FixedWidthParallelOptions parallel, CancellationToken ct) + TParser parser, Stream source, bool leaveOpen, FixedWidthReaderOptions options, CancellationToken ct) where TParser : struct, IRecordLineParser where TAccumulator : IRecordAccumulator, new() { try { - if (parallel.Workers == 1) + if (options.Workers == 1) { return await SequentialAsync(parser, source, options, ct).ConfigureAwait(false); } - return await PartitionedAsync(parser, source, options, parallel, ct).ConfigureAwait(false); + return await PartitionedAsync(parser, source, options, ct).ConfigureAwait(false); } finally { @@ -62,11 +62,11 @@ private static async Task SequentialAsync PartitionedAsync( - TParser parser, Stream source, FixedWidthReaderOptions options, FixedWidthParallelOptions parallel, CancellationToken ct) + TParser parser, Stream source, FixedWidthReaderOptions options, CancellationToken ct) where TParser : struct, IRecordLineParser where TAccumulator : IRecordAccumulator, new() { - int workerCount = parallel.Workers; + int workerCount = options.Workers; var queue = Channel.CreateBounded(new BoundedChannelOptions(workerCount * 2) { SingleWriter = true }); var workers = new Task>>[workerCount]; foreach (ref var worker in workers.AsSpan()) @@ -77,7 +77,7 @@ private static async Task PartitionedAsync 0 ? parallel.ChunkSize : DefaultChunkSize, ct).ConfigureAwait(false); + await ProduceAsync(source, queue.Writer, options, options.ChunkSize > 0 ? options.ChunkSize : DefaultChunkSize, ct).ConfigureAwait(false); } // Cancellation too: the workers must finish before the chunks left in the queue are returned. #pragma warning disable CA1031 // Rethrown by Merge once the workers have finished. diff --git a/tests/FixedWidthParser.Tests/FluentLayoutTests.cs b/tests/FixedWidthParser.Tests/FluentLayoutTests.cs index 2b43898..f254099 100644 --- a/tests/FixedWidthParser.Tests/FluentLayoutTests.cs +++ b/tests/FixedWidthParser.Tests/FluentLayoutTests.cs @@ -270,7 +270,7 @@ public async Task Fluent_AggregateFileAsync_MatchesSequential() var layout = Person(); var parallel = await FixedWidth.AggregateFileAsync( - _path, layout, parallel: new FixedWidthParallelOptions { DegreeOfParallelism = 4 }, cancellationToken: TestContext.Current.CancellationToken); + _path, layout, new FixedWidthReaderOptions { DegreeOfParallelism = 4 }, cancellationToken: TestContext.Current.CancellationToken); Assert.Equal(FixedWidth.ReadFile(_path, layout).Select(p => p.Age), parallel.Values); Assert.Equal(100_000, parallel.Values.Count); diff --git a/tests/FixedWidthParser.Tests/LayoutReadTests.cs b/tests/FixedWidthParser.Tests/LayoutReadTests.cs index 1774d1d..ac60137 100644 --- a/tests/FixedWidthParser.Tests/LayoutReadTests.cs +++ b/tests/FixedWidthParser.Tests/LayoutReadTests.cs @@ -52,7 +52,7 @@ public async Task AggregateFileAsync_WithLayout_MatchesSequentialFold() var layout = FixedWidthLayout.FromAttributes(); var parallel = await FixedWidth.AggregateFileAsync( - _path, layout, parallel: new FixedWidthParallelOptions { DegreeOfParallelism = 4 }, cancellationToken: TestContext.Current.CancellationToken); + _path, layout, new FixedWidthReaderOptions { DegreeOfParallelism = 4 }, cancellationToken: TestContext.Current.CancellationToken); Assert.Equal(SequentialAges(layout), parallel.Values); } @@ -65,7 +65,7 @@ public async Task AggregateAsync_WithLayout_MatchesSequentialFold() using var stream = new MemoryStream(File.ReadAllBytes(_path)); var parallel = await FixedWidth.AggregateAsync( - stream, layout, parallel: new FixedWidthParallelOptions { DegreeOfParallelism = 4 }, cancellationToken: TestContext.Current.CancellationToken); + stream, layout, options: new FixedWidthReaderOptions { DegreeOfParallelism = 4 }, cancellationToken: TestContext.Current.CancellationToken); Assert.Equal(SequentialAges(layout), parallel.Values); } diff --git a/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs b/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs index c6bc67b..a34c346 100644 --- a/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs +++ b/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs @@ -93,24 +93,24 @@ private List Sequential(FixedWidthReaderOptions? options = null) return [.. FixedWidth.ReadFile(_path, FixedWidthLayout.FromAttributes(), options).Select(m => m.Age)]; } - private Task Aggregate(Source source, FixedWidthParallelOptions parallel, FixedWidthReaderOptions? options = null) + private Task Aggregate(Source source, FixedWidthReaderOptions options) { var layout = FixedWidthLayout.FromAttributes(); var ct = TestContext.Current.CancellationToken; return source switch { - Source.File => FixedWidth.AggregateFileAsync(_path, layout, options, parallel, ct), - _ => FixedWidth.AggregateAsync(File.OpenRead(_path), layout, leaveOpen: false, options, parallel, ct), + Source.File => FixedWidth.AggregateFileAsync(_path, layout, options, ct), + _ => FixedWidth.AggregateAsync(File.OpenRead(_path), layout, leaveOpen: false, options, ct), }; } - private Task AggregateGenerated(Source source, FixedWidthParallelOptions parallel) + private Task AggregateGenerated(Source source, FixedWidthReaderOptions options) { var ct = TestContext.Current.CancellationToken; return source switch { - Source.File => FixedWidth.AggregateFileAsync(_path, parallel: parallel, cancellationToken: ct), - _ => FixedWidth.AggregateAsync(File.OpenRead(_path), parallel: parallel, cancellationToken: ct), + Source.File => FixedWidth.AggregateFileAsync(_path, options, ct), + _ => FixedWidth.AggregateAsync(File.OpenRead(_path), options: options, cancellationToken: ct), }; } @@ -119,7 +119,7 @@ private Task AggregateGenerated(Source source, FixedWidthParallelOption public async Task EveryPartitionSize_EqualsTheSequentialRead(Source source, int chunkSize) { File.WriteAllText(_path, Content(2000)); - var parallel = new FixedWidthParallelOptions { DegreeOfParallelism = 4, ChunkSize = chunkSize }; + var parallel = new FixedWidthReaderOptions { DegreeOfParallelism = 4, ChunkSize = chunkSize }; var runtime = await Aggregate(source, parallel); var generated = await AggregateGenerated(source, parallel); @@ -136,7 +136,7 @@ public async Task OneWorker_EqualsTheSequentialRead(Source source) { File.WriteAllText(_path, Content(500)); - var result = await Aggregate(source, new FixedWidthParallelOptions { DegreeOfParallelism = 1 }); + var result = await Aggregate(source, new FixedWidthReaderOptions { DegreeOfParallelism = 1 }); Assert.Equal(Sequential(), result.Values); } @@ -147,7 +147,7 @@ public async Task EmptySource_ReturnsAnEmptyAccumulator(Source source) { File.WriteAllBytes(_path, []); - var result = await Aggregate(source, new FixedWidthParallelOptions { DegreeOfParallelism = 4, ChunkSize = 8 }); + var result = await Aggregate(source, new FixedWidthReaderOptions { DegreeOfParallelism = 4, ChunkSize = 8 }); Assert.Empty(result.Values); } @@ -158,7 +158,7 @@ public async Task ParseFailure_ReportsTheSequentialLineNumber(Source source, int { File.WriteAllText(_path, Content(2000, badLine: 1537)); var expected = Assert.Throws(() => Sequential()); - var parallel = new FixedWidthParallelOptions { DegreeOfParallelism = 4, ChunkSize = chunkSize }; + var parallel = new FixedWidthReaderOptions { DegreeOfParallelism = 4, ChunkSize = chunkSize }; var actual = await Assert.ThrowsAsync(() => Aggregate(source, parallel)); @@ -174,9 +174,14 @@ public async Task OnParseError_SkipsEachBadLineOnce_WithItsLineNumber(Source sou var sequentialErrors = new List(); var expected = Sequential(new FixedWidthReaderOptions { OnParseError = ex => { sequentialErrors.Add(ex.LineNumber); return true; } }); var parallelErrors = new ConcurrentBag(); - var options = new FixedWidthReaderOptions { OnParseError = ex => { parallelErrors.Add(ex.LineNumber); return true; } }; + var options = new FixedWidthReaderOptions + { + OnParseError = ex => { parallelErrors.Add(ex.LineNumber); return true; }, + DegreeOfParallelism = 4, + ChunkSize = 9, + }; - var result = await Aggregate(source, new FixedWidthParallelOptions { DegreeOfParallelism = 4, ChunkSize = 9 }, options); + var result = await Aggregate(source, options); Assert.Equal(3, sequentialErrors.Count); Assert.Equal(expected, result.Values); @@ -188,11 +193,10 @@ public async Task OnParseError_SkipsEachBadLineOnce_WithItsLineNumber(Source sou public async Task LineTooLong_ReportsTheSequentialLineNumber(Source source, int chunkSize) { File.WriteAllText(_path, Content(300) + "\n" + new string('9', 50) + "\n" + Content(3)[1..]); - var options = new FixedWidthReaderOptions { MaxLineLength = 20 }; + var options = new FixedWidthReaderOptions { MaxLineLength = 20, DegreeOfParallelism = 4, ChunkSize = chunkSize }; var expected = Assert.Throws(() => Sequential(options)); - var parallel = new FixedWidthParallelOptions { DegreeOfParallelism = 4, ChunkSize = chunkSize }; - var actual = await Assert.ThrowsAsync(() => Aggregate(source, parallel, options)); + var actual = await Assert.ThrowsAsync(() => Aggregate(source, options)); Assert.Equal(expected.Message, actual.Message); } @@ -205,7 +209,7 @@ public async Task Stream_IsDisposedUnlessLeftOpen(bool leaveOpen) File.WriteAllText(_path, Content(10)); await using var stream = File.OpenRead(_path); - await FixedWidth.AggregateAsync(stream, FixedWidthLayout.FromAttributes(), leaveOpen, null, new FixedWidthParallelOptions { DegreeOfParallelism = 2, ChunkSize = 16 }, TestContext.Current.CancellationToken); + await FixedWidth.AggregateAsync(stream, FixedWidthLayout.FromAttributes(), leaveOpen, new FixedWidthReaderOptions { DegreeOfParallelism = 2, ChunkSize = 16 }, TestContext.Current.CancellationToken); Assert.Equal(leaveOpen, stream.CanRead); } @@ -218,12 +222,12 @@ public async Task CancelledToken_Throws(Source source) using var cts = new CancellationTokenSource(); await cts.CancelAsync(); var layout = FixedWidthLayout.FromAttributes(); - var parallel = new FixedWidthParallelOptions { DegreeOfParallelism = 2 }; + var parallel = new FixedWidthReaderOptions { DegreeOfParallelism = 2 }; await Assert.ThrowsAnyAsync(() => source switch { - Source.File => FixedWidth.AggregateFileAsync(_path, layout, null, parallel, cts.Token), - _ => FixedWidth.AggregateAsync(File.OpenRead(_path), layout, leaveOpen: false, null, parallel, cts.Token), + Source.File => FixedWidth.AggregateFileAsync(_path, layout, parallel, cts.Token), + _ => FixedWidth.AggregateAsync(File.OpenRead(_path), layout, leaveOpen: false, parallel, cts.Token), }); } @@ -233,12 +237,12 @@ public async Task NegativeDegreeOfParallelism_Throws() { File.WriteAllText(_path, Content(10)); var layout = FixedWidthLayout.FromAttributes(); - var parallel = new FixedWidthParallelOptions { DegreeOfParallelism = -1 }; + var parallel = new FixedWidthReaderOptions { DegreeOfParallelism = -1 }; var ct = TestContext.Current.CancellationToken; await using var stream = File.OpenRead(_path); - await Assert.ThrowsAsync(() => FixedWidth.AggregateFileAsync(_path, layout, null, parallel, ct)); - await Assert.ThrowsAsync(() => FixedWidth.AggregateAsync(stream, layout, leaveOpen: false, null, parallel, ct)); + await Assert.ThrowsAsync(() => FixedWidth.AggregateFileAsync(_path, layout, parallel, ct)); + await Assert.ThrowsAsync(() => FixedWidth.AggregateAsync(stream, layout, leaveOpen: false, parallel, ct)); Assert.True(stream.CanRead); } } From 18c1545b364e06805d048418386879622269a361 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 23 Oct 2026 23:25:35 -0300 Subject: [PATCH 29/85] refactor!: pass per-column parse and format settings as extensible structs ColumnFormatOptions is a plain readonly struct with an explicit constructor instead of a positional record struct, and WriteInto is internal. The new ColumnParseOptions (trim character, trim mode, format) replaces the optional trimChar/trimMode parameters and the format argument of every FixedWidthRuntime parse helper. Generated code stores one of each per column and passes them by in reference, so a later setting is a new constructor overload and property, not a change to helpers that compiled consumer code already binds to. Co-Authored-By: Claude Opus 5.5 --- .../FixedWidthModelGenerator.cs | 34 ++--- .../Infrastructure/ColumnFormatOptions.cs | 74 +++++++---- .../Infrastructure/ColumnParseOptions.cs | 35 +++++ .../Infrastructure/FixedWidthRuntime.cs | 70 +++++----- .../Infrastructure/RecordLineParsers.cs | 2 +- .../Layouts/FixedWidthLayoutBuilder.cs | 33 +++-- .../Processors/ColumnParserFactory.cs | 120 +++++++++--------- .../Processors/CultureHelpers.cs | 8 +- src/FixedWidthParser/PublicAPI.Unshipped.txt | 43 +++---- .../FixedWidthModelGeneratorTests.cs | 15 ++- 10 files changed, 244 insertions(+), 190 deletions(-) create mode 100644 src/FixedWidthParser/Infrastructure/ColumnParseOptions.cs diff --git a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs index 5e93d58..8585ea3 100644 --- a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs +++ b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs @@ -469,8 +469,8 @@ private static string BuildSource(ModelInfo model) return sb.ToString(); } - // One converter instance per converter column (created once, shared by parse and format) and - // one ColumnFormatOptions per column, resolved at compile time. + // One converter instance per converter column (created once, shared by parse and format), and one + // ColumnParseOptions and one ColumnFormatOptions per column, resolved at compile time. private static void AppendFields(StringBuilder sb, string body, ModelInfo model) { var columns = model.Columns.AsImmutableArray(); @@ -482,6 +482,9 @@ private static void AppendFields(StringBuilder sb, string body, ModelInfo model) sb.Append(body).Append("private static readonly ").Append(c.ConverterFqn) .Append(" __converter").Append(i).Append(" = new ").Append(c.ConverterFqn).AppendLine("();"); } + sb.Append(body).Append("private static readonly global::FixedWidthParser.Infrastructure.ColumnParseOptions __parse").Append(i) + .Append(" = new(").Append(SymbolDisplay.FormatLiteral(c.TrimChar, true)).Append(", ").Append(TrimModeLiteral(c.TrimMode)) + .Append(", ").Append(c.Format is null ? "null" : SymbolDisplay.FormatLiteral(c.Format, true)).AppendLine(");"); sb.Append(body).Append("private static readonly global::FixedWidthParser.Infrastructure.ColumnFormatOptions __options").Append(i) .Append(" = new(").Append(AlignmentLiteral(c.Alignment)).Append(", ").Append(SymbolDisplay.FormatLiteral(c.Padding, true)) .Append(", ").Append(c.Format is null ? "null" : SymbolDisplay.FormatLiteral(c.Format, true)) @@ -529,7 +532,7 @@ private static void AppendParseMethod(StringBuilder sb, string body, ModelInfo m // T?: a blank column assigns null without invoking the underlying parser. sb.Append(stmt).Append(c.TypeFqn).Append("? __v").Append(i).AppendLine(";"); - sb.Append(stmt).Append("if (").Append(RuntimeFqn).Append(".IsBlank(").Append(col).Append(", ").Append(ByteLiteral(c.TrimChar)).AppendLine("))"); + sb.Append(stmt).Append("if (").Append(RuntimeFqn).Append(".IsBlank(").Append(col).Append(", in __parse").Append(i).AppendLine("))"); sb.Append(stmt).AppendLine("{"); sb.Append(stmt).Append(" __v").Append(i).AppendLine(" = null;"); sb.Append(stmt).AppendLine("}"); @@ -554,7 +557,7 @@ private static void AppendParseMethod(StringBuilder sb, string body, ModelInfo m // Emits the kind-specific parse statement for one column into a local named localName. private static void AppendColumnParse(StringBuilder sb, string stmt, ColumnInfo c, string col, int i, string localName) { - string trimArg = TrimArgSuffix(c); + string parse = ", in __parse" + i; string fail = ")) { context.ReportFailedColumn(" + SymbolDisplay.FormatLiteral(c.Name, true) + ", " + c.Start + ", " + c.Length + "); model = default!; return false; }"; @@ -562,29 +565,29 @@ private static void AppendColumnParse(StringBuilder sb, string stmt, ColumnInfo { case ColumnKind.Exact: sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append('.').Append(ExactHelper(c.TypeFqn)).Append('(').Append(col) - .Append(", ").Append(SymbolDisplay.FormatLiteral(c.Format!, true)).Append(", context, out var ").Append(localName).Append(trimArg).AppendLine(fail); + .Append(", context").Append(parse).Append(", out var ").Append(localName).AppendLine(fail); break; case ColumnKind.String: sb.Append(stmt).Append("string ").Append(localName) - .Append(" = ").Append(RuntimeFqn).Append(".String(").Append(col).Append(", context").Append(trimArg).AppendLine(");"); + .Append(" = ").Append(RuntimeFqn).Append(".String(").Append(col).Append(", context").Append(parse).AppendLine(");"); break; case ColumnKind.Double: sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append(".TryDouble(").Append(col) - .Append(", context, out double ").Append(localName).Append(trimArg).AppendLine(fail); + .Append(", context").Append(parse).Append(", out double ").Append(localName).AppendLine(fail); break; case ColumnKind.Float: sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append(".TryFloat(").Append(col) - .Append(", context, out float ").Append(localName).Append(trimArg).AppendLine(fail); + .Append(", context").Append(parse).Append(", out float ").Append(localName).AppendLine(fail); break; case ColumnKind.Utf8Parsable: case ColumnKind.DecodedParsable: sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append(c.Kind == ColumnKind.Utf8Parsable ? ".TryParse<" : ".TryParseDecoded<") - .Append(c.TypeFqn).Append(">(").Append(col).Append(", context, out ").Append(c.TypeFqn).Append(' ').Append(localName).Append(trimArg).AppendLine(fail); + .Append(c.TypeFqn).Append(">(").Append(col).Append(", context").Append(parse).Append(", out ").Append(c.TypeFqn).Append(' ').Append(localName).AppendLine(fail); break; case ColumnKind.Converter: sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append(".TryConvert<").Append(c.TypeFqn).Append(", ").Append(c.ConverterFqn) - .Append(">(").Append(col).Append(", context, __converter").Append(i).Append(", out ").Append(c.TypeFqn) - .Append(' ').Append(localName).Append(trimArg).AppendLine(fail); + .Append(">(").Append(col).Append(", context, __converter").Append(i).Append(parse).Append(", out ").Append(c.TypeFqn) + .Append(' ').Append(localName).AppendLine(fail); break; default: // Unsupported kinds were reported in Emit, which then skips generation. @@ -664,15 +667,6 @@ private static void AppendColumnKindFormat(StringBuilder sb, string stmt, Column } // Trailing trim arguments for a runtime parse call, omitted for the default (space, trailing). - private static string TrimArgSuffix(ColumnInfo c) - { - if (c.TrimChar == ' ' && c.TrimMode == 0) - { - return string.Empty; - } - return ", " + ByteLiteral(c.TrimChar) + ", " + TrimModeLiteral(c.TrimMode); - } - private static string ByteLiteral(char value) { return "(byte)" + SymbolDisplay.FormatLiteral(value, true); diff --git a/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs b/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs index 2596246..528afb1 100644 --- a/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs +++ b/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs @@ -5,55 +5,81 @@ namespace FixedWidthParser.Infrastructure /// /// Resolved formatting options for a column (alignment, padding, format string and an overflow /// policy already resolved to or - /// ). Centralizes placing the content into the line slice, - /// shared by the generated and runtime writers. must be ASCII. + /// ), passed by generated code and layouts to + /// . must be ASCII. + /// Public because generated code constructs it; not intended for direct use. /// - public readonly record struct ColumnFormatOptions(Alignment Alignment, char Padding, string? Format, OverflowBehavior Overflow) + /// + /// Generated code in consumer assemblies calls the constructor, so a new setting arrives as a new + /// constructor overload and a new property; the existing constructor is never changed. + /// +#pragma warning disable CA1815 // Never compared; value equality is meaningless for it. + public readonly struct ColumnFormatOptions +#pragma warning restore CA1815 { + /// Creates the options for one column. + public ColumnFormatOptions(Alignment alignment, char padding, string? format, OverflowBehavior overflow) + { + Alignment = alignment; + Padding = padding; + Format = format; + Overflow = overflow; + } + + /// Which side a written value is aligned to. + public Alignment Alignment { get; } + + /// The ASCII character that fills the column around its value. + public char Padding { get; } + + /// The format string for the value, or for the type's default. + public string? Format { get; } + + /// The resolved overflow policy: or . + public OverflowBehavior Overflow { get; } + /// /// Writes the encoded into the column slice , /// applying alignment, padding and overflow. /// - public void WriteInto(ReadOnlySpan content, Span slice, string columnName) + internal void WriteInto(ReadOnlySpan content, Span slice, string columnName) { int width = slice.Length; int length = content.Length; byte padding = (byte)Padding; - if (length <= width) + if (length > width) { - if (Alignment != Alignment.Right) + if (Overflow == OverflowBehavior.Throw) { - content.CopyTo(slice); - slice[length..].Fill(padding); - return; + throw new InvalidOperationException( + $"Value \"{Encoding.UTF8.GetString(content)}\" ({length} bytes) exceeds the width {width} of column \"{columnName}\"."); } - int pad = width - length; - if (Padding != ' ' && content.Length > 0 && (content[0] == (byte)'-' || content[0] == (byte)'+')) + if (Alignment == Alignment.Right) { - slice[0] = content[0]; - slice[1..(pad + 1)].Fill(padding); - content[1..].CopyTo(slice[(pad + 1)..]); + content[(length - width)..].CopyTo(slice); return; } - slice[..pad].Fill(padding); - content.CopyTo(slice[pad..]); + content[..width].CopyTo(slice); return; } - if (Overflow == OverflowBehavior.Throw) + if (Alignment != Alignment.Right) { - throw new InvalidOperationException( - $"Value \"{Encoding.UTF8.GetString(content)}\" ({length} bytes) exceeds the width {width} of column \"{columnName}\"."); + content.CopyTo(slice); + slice[length..].Fill(padding); + return; } - - // Truncate: keep the bytes on the alignment side. - if (Alignment == Alignment.Right) + int pad = width - length; + if (Padding != ' ' && content.Length > 0 && (content[0] == (byte)'-' || content[0] == (byte)'+')) { - content[(length - width)..].CopyTo(slice); + slice[0] = content[0]; + slice[1..(pad + 1)].Fill(padding); + content[1..].CopyTo(slice[(pad + 1)..]); return; } - content[..width].CopyTo(slice); + slice[..pad].Fill(padding); + content.CopyTo(slice[pad..]); } /// diff --git a/src/FixedWidthParser/Infrastructure/ColumnParseOptions.cs b/src/FixedWidthParser/Infrastructure/ColumnParseOptions.cs new file mode 100644 index 0000000..5467199 --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/ColumnParseOptions.cs @@ -0,0 +1,35 @@ +namespace FixedWidthParser.Infrastructure +{ + /// + /// How a column is trimmed and, for a date or time, the exact format it is parsed with, passed by + /// generated code and layouts to . must be ASCII. + /// Public because generated code constructs it; not intended for direct use. + /// + /// + /// Generated code in consumer assemblies calls the constructor, so a new setting arrives as a new + /// constructor overload and a new property; the existing constructor is never changed. + /// +#pragma warning disable CA1815 // Never compared; value equality is meaningless for it. + public readonly struct ColumnParseOptions +#pragma warning restore CA1815 + { + /// Creates the options for one column. + public ColumnParseOptions(char trimChar, TrimMode trimMode, string? format) + { + TrimChar = trimChar; + TrimMode = trimMode; + Format = format; + } + + /// The ASCII character trimmed from the column before it is parsed, and that a value is written with. + public char TrimChar { get; } + + /// Which ends of the column is trimmed from. + public TrimMode TrimMode { get; } + + /// The exact format of a date or time column, or . + public string? Format { get; } + + internal byte TrimByte => (byte)TrimChar; + } +} diff --git a/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs index 0f8d672..b45b01b 100644 --- a/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs +++ b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs @@ -20,60 +20,60 @@ public static class FixedWidthRuntime private const int StackBufferSize = 256; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ReadOnlySpan TrimColumn(ReadOnlySpan column, byte trimChar, TrimMode trimMode) + internal static ReadOnlySpan TrimColumn(ReadOnlySpan column, in ColumnParseOptions options) { - return trimMode switch + return options.TrimMode switch { - TrimMode.Leading => column.TrimStart(trimChar), - TrimMode.Both => column.Trim(trimChar), - _ => column.TrimEnd(trimChar), + TrimMode.Leading => column.TrimStart(options.TrimByte), + TrimMode.Both => column.Trim(options.TrimByte), + _ => column.TrimEnd(options.TrimByte), }; } - /// True when the column holds only and spaces, so a nullable column reads as . + /// True when the column holds only the trim character and spaces, so a nullable column reads as . [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool IsBlank(ReadOnlySpan column, byte trimChar) + public static bool IsBlank(ReadOnlySpan column, in ColumnParseOptions options) { - return column.Trim(trimChar).Trim((byte)' ').IsEmpty; + return column.Trim(options.TrimByte).Trim((byte)' ').IsEmpty; } /// Decodes a string column with the context's encoding, interning it through the context's pool when there is one. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static string String(ReadOnlySpan column, in FixedWidthContext context, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) + public static string String(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options) { - var trimmed = TrimColumn(column, trimChar, trimMode); + var trimmed = TrimColumn(column, in options); return context.StringPool is { } pool ? pool.GetOrAdd(trimmed, context.Encoding) : context.Encoding.GetString(trimmed); } /// Parses a column, honoring the provider's decimal separator. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryDouble(ReadOnlySpan column, in FixedWidthContext context, out double value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) + public static bool TryDouble(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out double value) { - return CultureHelpers.TryParseDouble(column, in context, out value, trimChar, trimMode); + return CultureHelpers.TryParseDouble(column, in context, in options, out value); } /// Parses a column, honoring the provider's decimal separator. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryFloat(ReadOnlySpan column, in FixedWidthContext context, out float value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) + public static bool TryFloat(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out float value) { - return CultureHelpers.TryParseFloat(column, in context, out value, trimChar, trimMode); + return CultureHelpers.TryParseFloat(column, in context, in options, out value); } /// Parses an column straight from its bytes. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryParse(ReadOnlySpan column, in FixedWidthContext context, [MaybeNullWhen(false)] out TValue value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) + public static bool TryParse(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, [MaybeNullWhen(false)] out TValue value) where TValue : IUtf8SpanParsable { - return TValue.TryParse(TrimColumn(column, trimChar, trimMode), context.FormatProvider, out value); + return TValue.TryParse(TrimColumn(column, in options), context.FormatProvider, out value); } /// Parses an column that has no UTF-8 parser, decoding it with the context's encoding first. [SkipLocalsInit] - public static bool TryParseDecoded(ReadOnlySpan column, in FixedWidthContext context, [MaybeNullWhen(false)] out TValue value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) + public static bool TryParseDecoded(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, [MaybeNullWhen(false)] out TValue value) where TValue : ISpanParsable { Span stack = stackalloc char[StackBufferSize]; - var text = Decode(TrimColumn(column, trimChar, trimMode), context.Encoding, stack, out char[]? rented); + var text = Decode(TrimColumn(column, in options), context.Encoding, stack, out char[]? rented); try { return TValue.TryParse(text, context.FormatProvider, out value); @@ -84,23 +84,24 @@ public static bool TryParseDecoded(ReadOnlySpan column, in FixedWi } } - /// Parses a column via a FixedColumnAttribute.Converter instance. + /// Parses a column through an instance. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool TryConvert( - ReadOnlySpan column, in FixedWidthContext context, TConverter converter, [MaybeNullWhen(false)] out TValue value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) + ReadOnlySpan column, in FixedWidthContext context, TConverter converter, in ColumnParseOptions options, [MaybeNullWhen(false)] out TValue value) where TConverter : IFixedWidthConverter { - return converter.TryParse(TrimColumn(column, trimChar, trimMode), in context, out value); + return converter.TryParse(TrimColumn(column, in options), in context, out value); } + /// Parses a column with exactly , decoding it first. [SkipLocalsInit] - public static bool TryDateTimeExact(ReadOnlySpan column, string format, in FixedWidthContext context, out DateTime value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) + public static bool TryDateTimeExact(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out DateTime value) { Span stack = stackalloc char[StackBufferSize]; - var text = Decode(TrimColumn(column, trimChar, trimMode), context.Encoding, stack, out char[]? rented); + var text = Decode(TrimColumn(column, in options), context.Encoding, stack, out char[]? rented); try { - return DateTime.TryParseExact(text, format, context.FormatProvider, DateTimeStyles.None, out value); + return DateTime.TryParseExact(text, options.Format, context.FormatProvider, DateTimeStyles.None, out value); } finally { @@ -108,14 +109,15 @@ public static bool TryDateTimeExact(ReadOnlySpan column, string format, in } } + /// Parses a column with exactly , decoding it first. [SkipLocalsInit] - public static bool TryDateOnlyExact(ReadOnlySpan column, string format, in FixedWidthContext context, out DateOnly value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) + public static bool TryDateOnlyExact(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out DateOnly value) { Span stack = stackalloc char[StackBufferSize]; - var text = Decode(TrimColumn(column, trimChar, trimMode), context.Encoding, stack, out char[]? rented); + var text = Decode(TrimColumn(column, in options), context.Encoding, stack, out char[]? rented); try { - return DateOnly.TryParseExact(text, format, context.FormatProvider, DateTimeStyles.None, out value); + return DateOnly.TryParseExact(text, options.Format, context.FormatProvider, DateTimeStyles.None, out value); } finally { @@ -123,14 +125,15 @@ public static bool TryDateOnlyExact(ReadOnlySpan column, string format, in } } + /// Parses a column with exactly , decoding it first. [SkipLocalsInit] - public static bool TryTimeOnlyExact(ReadOnlySpan column, string format, in FixedWidthContext context, out TimeOnly value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) + public static bool TryTimeOnlyExact(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out TimeOnly value) { Span stack = stackalloc char[StackBufferSize]; - var text = Decode(TrimColumn(column, trimChar, trimMode), context.Encoding, stack, out char[]? rented); + var text = Decode(TrimColumn(column, in options), context.Encoding, stack, out char[]? rented); try { - return TimeOnly.TryParseExact(text, format, context.FormatProvider, DateTimeStyles.None, out value); + return TimeOnly.TryParseExact(text, options.Format, context.FormatProvider, DateTimeStyles.None, out value); } finally { @@ -138,14 +141,15 @@ public static bool TryTimeOnlyExact(ReadOnlySpan column, string format, in } } + /// Parses a column with exactly , decoding it first. [SkipLocalsInit] - public static bool TryDateTimeOffsetExact(ReadOnlySpan column, string format, in FixedWidthContext context, out DateTimeOffset value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) + public static bool TryDateTimeOffsetExact(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out DateTimeOffset value) { Span stack = stackalloc char[StackBufferSize]; - var text = Decode(TrimColumn(column, trimChar, trimMode), context.Encoding, stack, out char[]? rented); + var text = Decode(TrimColumn(column, in options), context.Encoding, stack, out char[]? rented); try { - return DateTimeOffset.TryParseExact(text, format, context.FormatProvider, DateTimeStyles.None, out value); + return DateTimeOffset.TryParseExact(text, options.Format, context.FormatProvider, DateTimeStyles.None, out value); } finally { diff --git a/src/FixedWidthParser/Infrastructure/RecordLineParsers.cs b/src/FixedWidthParser/Infrastructure/RecordLineParsers.cs index 1974e20..e4bd882 100644 --- a/src/FixedWidthParser/Infrastructure/RecordLineParsers.cs +++ b/src/FixedWidthParser/Infrastructure/RecordLineParsers.cs @@ -6,7 +6,7 @@ namespace FixedWidthParser.Infrastructure /// devirtualizes into a direct call. Public only because it appears in the /// enumerator types; the library is its only implementer. /// - public interface IRecordLineParser + public interface IRecordLineParser where TModel : allows ref struct { /// Parses a buffered line into a model. bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TModel model); diff --git a/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs b/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs index 8413345..ca6bb95 100644 --- a/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs +++ b/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs @@ -44,7 +44,7 @@ public FixedWidthLayoutBuilder Column( ColumnParser? parser = null; if (set is not null) { - parser = ColumnParserFactory.ForString(set, column.Trim, column.TrimMode); + parser = ColumnParserFactory.ForString(set, column.Parse); } IColumnFormatter? formatter = null; if (get is not null) @@ -68,7 +68,7 @@ public FixedWidthLayoutBuilder Column( ColumnParser? parser = null; if (set is not null) { - parser = ColumnParserFactory.ForUtf8Parsable(set, column.Trim, column.TrimMode, column.Format); + parser = ColumnParserFactory.ForUtf8Parsable(set, column.Parse); } IColumnFormatter? formatter = null; if (get is not null) @@ -92,7 +92,7 @@ public FixedWidthLayoutBuilder DecodedColumn( ColumnParser? parser = null; if (set is not null) { - parser = ColumnParserFactory.ForDecodedParsable(set, column.Trim, column.TrimMode, column.Format); + parser = ColumnParserFactory.ForDecodedParsable(set, column.Parse); } IColumnFormatter? formatter = null; if (get is not null) @@ -115,15 +115,15 @@ public FixedWidthLayoutBuilder NullableColumn( ColumnParser? parser = null; if (set is not null) { - parser = ColumnParserFactory.ForNullable(set, column.Trim, - adapted => ColumnParserFactory.ForUtf8Parsable(adapted, column.Trim, column.TrimMode, column.Format)); + parser = ColumnParserFactory.ForNullable(set, column.Parse, + adapted => ColumnParserFactory.ForUtf8Parsable(adapted, column.Parse)); } IColumnFormatter? formatter = null; if (get is not null) { var inner = new Utf8SpanFormattableColumnFormatter( start, length, column.FormatOptions, name, (in TModel model) => get(in model).GetValueOrDefault()); - formatter = new NullableColumnFormatter(start, length, column.Trim, get, inner); + formatter = new NullableColumnFormatter(start, length, column.Parse.TrimByte, get, inner); } return Add(column, parser, formatter); } @@ -143,15 +143,15 @@ public FixedWidthLayoutBuilder NullableDecodedColumn( ColumnParser? parser = null; if (set is not null) { - parser = ColumnParserFactory.ForNullable(set, column.Trim, - adapted => ColumnParserFactory.ForDecodedParsable(adapted, column.Trim, column.TrimMode, column.Format)); + parser = ColumnParserFactory.ForNullable(set, column.Parse, + adapted => ColumnParserFactory.ForDecodedParsable(adapted, column.Parse)); } IColumnFormatter? formatter = null; if (get is not null) { var inner = new SpanFormattableColumnFormatter( start, length, column.FormatOptions, name, (in TModel model) => get(in model).GetValueOrDefault()); - formatter = new NullableColumnFormatter(start, length, column.Trim, get, inner); + formatter = new NullableColumnFormatter(start, length, column.Parse.TrimByte, get, inner); } return Add(column, parser, formatter); } @@ -172,15 +172,15 @@ public FixedWidthLayoutBuilder NullableConvertedColumn( ColumnParser? parser = null; if (set is not null) { - parser = ColumnParserFactory.ForNullable(set, column.Trim, - adapted => ColumnParserFactory.ForConverter(adapted, converter, column.Trim, column.TrimMode)); + parser = ColumnParserFactory.ForNullable(set, column.Parse, + adapted => ColumnParserFactory.ForConverter(adapted, converter, column.Parse)); } IColumnFormatter? formatter = null; if (get is not null) { var inner = new ConverterColumnFormatter>( start, length, column.FormatOptions, name, (in TModel model) => get(in model).GetValueOrDefault(), converter); - formatter = new NullableColumnFormatter(start, length, column.Trim, get, inner); + formatter = new NullableColumnFormatter(start, length, column.Parse.TrimByte, get, inner); } return Add(column, parser, formatter); } @@ -203,7 +203,7 @@ public FixedWidthLayoutBuilder ConvertedColumn( ColumnParser? parser = null; if (set is not null) { - parser = ColumnParserFactory.ForConverter(set, converter, column.Trim, column.TrimMode); + parser = ColumnParserFactory.ForConverter(set, converter, column.Parse); } IColumnFormatter? formatter = null; if (get is not null) @@ -269,9 +269,9 @@ private PendingColumn Begin(string name, int start, int length, Type valueType, ArgumentException.ThrowIfNullOrEmpty(name); options ??= FixedWidthColumnOptions.Default; var formatOptions = FormatterBuilder.ResolveOptions(name, valueType, options.Alignment, options.Padding, options.Format, options.Overflow); - byte trim = FixedWidthRuntime.ToAsciiByte(options.TrimChar, name, nameof(FixedWidthColumnOptions.TrimChar)); + FixedWidthRuntime.ToAsciiByte(options.TrimChar, name, nameof(FixedWidthColumnOptions.TrimChar)); _columns.Add((start, length, name)); - return new(name, start, length, trim, options.TrimMode, options.Format, formatOptions); + return new(name, start, length, new ColumnParseOptions(options.TrimChar, options.TrimMode, options.Format), formatOptions); } private FixedWidthLayoutBuilder Add(in PendingColumn column, ColumnParser? parser, IColumnFormatter? formatter) @@ -290,7 +290,6 @@ private FixedWidthLayoutBuilder Add(in PendingColumn column, ColumnParse } [StructLayout(LayoutKind.Auto)] - private readonly record struct PendingColumn( - string Name, int Start, int Length, byte Trim, TrimMode TrimMode, string? Format, ColumnFormatOptions FormatOptions); + private readonly record struct PendingColumn(string Name, int Start, int Length, ColumnParseOptions Parse, ColumnFormatOptions FormatOptions); } } diff --git a/src/FixedWidthParser/Processors/ColumnParserFactory.cs b/src/FixedWidthParser/Processors/ColumnParserFactory.cs index 3594cf1..c04d17c 100644 --- a/src/FixedWidthParser/Processors/ColumnParserFactory.cs +++ b/src/FixedWidthParser/Processors/ColumnParserFactory.cs @@ -4,11 +4,12 @@ namespace FixedWidthParser.Processors { /// - /// Combines a column's compiled setter and a value parser into a single - /// closure that calls the same - /// helpers as generated code. Resolution order: a FixedColumnAttribute.Converter, a date or - /// time with a Format, string, double/float, then - /// , then over the decoded text. + /// Combines a column's setter and a value parser into a single + /// closure that calls the same helpers as generated code. Resolution + /// order: a converter, a date or time with a Format, string, double/float, + /// then , then over the decoded + /// text. Attribute layouts reach it by reflection (), fluent layouts through + /// the typed For* entry points. /// internal static class ColumnParserFactory { @@ -17,17 +18,18 @@ internal static class ColumnParserFactory public static ColumnParser Create(Type valueType, Delegate setter, Type? converterType, string memberName, char trimChar, TrimMode trimMode, string? format) where TModel : allows ref struct { - byte trim = FixedWidthRuntime.ToAsciiByte(trimChar, memberName, nameof(FixedColumnAttribute.TrimChar)); + FixedWidthRuntime.ToAsciiByte(trimChar, memberName, nameof(FixedColumnAttribute.TrimChar)); + var options = new ColumnParseOptions(trimChar, trimMode, format); var underlyingType = Nullable.GetUnderlyingType(valueType); if (underlyingType is not null) { return (ColumnParser)Method(nameof(BuildNullable), typeof(TModel), underlyingType) - .Invoke(null, [setter, converterType, memberName, trim, trimMode, format])!; + .Invoke(null, [setter, converterType, memberName, options])!; } - return CreateValue(valueType, setter, converterType, memberName, trim, trimMode, format); + return CreateValue(valueType, setter, converterType, memberName, options); } - private static ColumnParser CreateValue(Type valueType, Delegate setter, Type? converterType, string memberName, byte trim, TrimMode trimMode, string? format) + private static ColumnParser CreateValue(Type valueType, Delegate setter, Type? converterType, string memberName, ColumnParseOptions options) where TModel : allows ref struct { if (converterType is not null) @@ -41,74 +43,74 @@ private static ColumnParser CreateValue(Type valueType, Delegate var converter = Activator.CreateInstance(converterType) ?? throw new InvalidOperationException($"Could not instantiate converter '{converterType}' for column '{memberName}'."); return (ColumnParser)Method(nameof(BuildConverter), typeof(TModel), valueType, converterType) - .Invoke(null, [setter, converter, trim, trimMode])!; + .Invoke(null, [setter, converter, options])!; } - var special = TryBuildSpecial(valueType, setter, trim, trimMode, format); + var special = TryBuildSpecial(valueType, setter, options); if (special is not null) { return special; } if (valueType == typeof(string)) { - return BuildString((ColumnSetter)setter, trim, trimMode); + return BuildString((ColumnSetter)setter, options); } if (Implements(valueType, typeof(IUtf8SpanParsable<>))) { - return (ColumnParser)Method(nameof(BuildUtf8Parsable), typeof(TModel), valueType).Invoke(null, [setter, trim, trimMode])!; + return (ColumnParser)Method(nameof(BuildUtf8Parsable), typeof(TModel), valueType).Invoke(null, [setter, options])!; } if (Implements(valueType, typeof(ISpanParsable<>))) { - return (ColumnParser)Method(nameof(BuildDecodedParsable), typeof(TModel), valueType).Invoke(null, [setter, trim, trimMode])!; + return (ColumnParser)Method(nameof(BuildDecodedParsable), typeof(TModel), valueType).Invoke(null, [setter, options])!; } throw new InvalidOperationException( $"Column '{memberName}' has type '{valueType}', which is not string, double, float, IUtf8SpanParsable or ISpanParsable; set a Converter for it."); } - internal static ColumnParser ForString(ColumnSetter setter, byte trim, TrimMode trimMode) + internal static ColumnParser ForString(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct { - return BuildString(setter, trim, trimMode); + return BuildString(setter, options); } - internal static ColumnParser ForUtf8Parsable(ColumnSetter setter, byte trim, TrimMode trimMode, string? format) + internal static ColumnParser ForUtf8Parsable(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct where TValue : IUtf8SpanParsable { - return TryBuildSpecial(typeof(TValue), setter, trim, trimMode, format) ?? BuildUtf8Parsable(setter, trim, trimMode); + return TryBuildSpecial(typeof(TValue), setter, options) ?? BuildUtf8Parsable(setter, options); } - internal static ColumnParser ForDecodedParsable(ColumnSetter setter, byte trim, TrimMode trimMode, string? format) + internal static ColumnParser ForDecodedParsable(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct where TValue : ISpanParsable { - return TryBuildSpecial(typeof(TValue), setter, trim, trimMode, format) ?? BuildDecodedParsable(setter, trim, trimMode); + return TryBuildSpecial(typeof(TValue), setter, options) ?? BuildDecodedParsable(setter, options); } - internal static ColumnParser ForConverter(ColumnSetter setter, IFixedWidthConverter converter, byte trim, TrimMode trimMode) + internal static ColumnParser ForConverter(ColumnSetter setter, IFixedWidthConverter converter, ColumnParseOptions options) where TModel : allows ref struct { - return BuildConverter(setter, converter, trim, trimMode); + return BuildConverter(setter, converter, options); } internal static ColumnParser ForNullable( - ColumnSetter setter, byte trim, Func, ColumnParser> buildInner) + ColumnSetter setter, ColumnParseOptions options, Func, ColumnParser> buildInner) where TModel : allows ref struct where TValue : struct { ColumnSetter adapted = (ref TModel model, TValue value) => setter(ref model, value); - return WrapNullable(setter, buildInner(adapted), trim); + return WrapNullable(setter, buildInner(adapted), options); } // The types the attribute path special-cases ahead of the generic parsers, in its order: a date or // time with a Format, then double and float. - private static ColumnParser? TryBuildSpecial(Type valueType, Delegate setter, byte trim, TrimMode trimMode, string? format) + private static ColumnParser? TryBuildSpecial(Type valueType, Delegate setter, ColumnParseOptions options) where TModel : allows ref struct { - if (format is not null) + if (options.Format is not null) { - var date = TryBuildDate(valueType, setter, trim, trimMode, format); + var date = TryBuildDate(valueType, setter, options); if (date is not null) { return date; @@ -116,33 +118,33 @@ internal static ColumnParser ForNullable( } if (valueType == typeof(double)) { - return BuildDouble((ColumnSetter)setter, trim, trimMode); + return BuildDouble((ColumnSetter)setter, options); } if (valueType == typeof(float)) { - return BuildFloat((ColumnSetter)setter, trim, trimMode); + return BuildFloat((ColumnSetter)setter, options); } return null; } - private static ColumnParser? TryBuildDate(Type valueType, Delegate setter, byte trim, TrimMode trimMode, string format) + private static ColumnParser? TryBuildDate(Type valueType, Delegate setter, ColumnParseOptions options) where TModel : allows ref struct { if (valueType == typeof(DateTime)) { - return BuildDateTime((ColumnSetter)setter, format, trim, trimMode); + return BuildDateTime((ColumnSetter)setter, options); } if (valueType == typeof(DateOnly)) { - return BuildDateOnly((ColumnSetter)setter, format, trim, trimMode); + return BuildDateOnly((ColumnSetter)setter, options); } if (valueType == typeof(TimeOnly)) { - return BuildTimeOnly((ColumnSetter)setter, format, trim, trimMode); + return BuildTimeOnly((ColumnSetter)setter, options); } if (valueType == typeof(DateTimeOffset)) { - return BuildDateTimeOffset((ColumnSetter)setter, format, trim, trimMode); + return BuildDateTimeOffset((ColumnSetter)setter, options); } return null; } @@ -160,21 +162,21 @@ private static MethodInfo Method(string name, params Type[] typeArguments) } // A nullable column (T?): a blank column assigns null without invoking the underlying parser. - private static ColumnParser BuildNullable(Delegate setter, Type? converterType, string memberName, byte trim, TrimMode trimMode, string? format) + private static ColumnParser BuildNullable(Delegate setter, Type? converterType, string memberName, ColumnParseOptions options) where TModel : allows ref struct where TUnderlying : struct { - return ForNullable((ColumnSetter)setter, trim, - adapted => CreateValue(typeof(TUnderlying), adapted, converterType, memberName, trim, trimMode, format)); + return ForNullable((ColumnSetter)setter, options, + adapted => CreateValue(typeof(TUnderlying), adapted, converterType, memberName, options)); } - private static ColumnParser WrapNullable(ColumnSetter nullableSetter, ColumnParser inner, byte trim) + private static ColumnParser WrapNullable(ColumnSetter nullableSetter, ColumnParser inner, ColumnParseOptions options) where TModel : allows ref struct where TUnderlying : struct { return (column, in context, ref model) => { - if (FixedWidthRuntime.IsBlank(column, trim)) + if (FixedWidthRuntime.IsBlank(column, in options)) { nullableSetter(ref model, null); return true; @@ -183,13 +185,13 @@ private static ColumnParser WrapNullable(ColumnSett }; } - private static ColumnParser BuildUtf8Parsable(ColumnSetter setter, byte trim, TrimMode trimMode) + private static ColumnParser BuildUtf8Parsable(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct where TValue : IUtf8SpanParsable { return (column, in context, ref model) => { - if (!FixedWidthRuntime.TryParse(column, in context, out var value, trim, trimMode)) + if (!FixedWidthRuntime.TryParse(column, in context, in options, out var value)) { return false; } @@ -198,13 +200,13 @@ private static ColumnParser BuildUtf8Parsable(ColumnSett }; } - private static ColumnParser BuildDecodedParsable(ColumnSetter setter, byte trim, TrimMode trimMode) + private static ColumnParser BuildDecodedParsable(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct where TValue : ISpanParsable { return (column, in context, ref model) => { - if (!FixedWidthRuntime.TryParseDecoded(column, in context, out var value, trim, trimMode)) + if (!FixedWidthRuntime.TryParseDecoded(column, in context, in options, out var value)) { return false; } @@ -213,13 +215,13 @@ private static ColumnParser BuildDecodedParsable(ColumnS }; } - private static ColumnParser BuildConverter(ColumnSetter setter, TConverter converter, byte trim, TrimMode trimMode) + private static ColumnParser BuildConverter(ColumnSetter setter, TConverter converter, ColumnParseOptions options) where TModel : allows ref struct where TConverter : IFixedWidthConverter { return (column, in context, ref model) => { - if (!FixedWidthRuntime.TryConvert(column, in context, converter, out var value, trim, trimMode)) + if (!FixedWidthRuntime.TryConvert(column, in context, converter, in options, out var value)) { return false; } @@ -228,22 +230,22 @@ private static ColumnParser BuildConverter(C }; } - private static ColumnParser BuildString(ColumnSetter setter, byte trim, TrimMode trimMode) + private static ColumnParser BuildString(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct { return (column, in context, ref model) => { - setter(ref model, FixedWidthRuntime.String(column, in context, trim, trimMode)); + setter(ref model, FixedWidthRuntime.String(column, in context, in options)); return true; }; } - private static ColumnParser BuildDouble(ColumnSetter setter, byte trim, TrimMode trimMode) + private static ColumnParser BuildDouble(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct { return (column, in context, ref model) => { - if (!FixedWidthRuntime.TryDouble(column, in context, out double value, trim, trimMode)) + if (!FixedWidthRuntime.TryDouble(column, in context, in options, out double value)) { return false; } @@ -252,12 +254,12 @@ private static ColumnParser BuildDouble(ColumnSetter BuildFloat(ColumnSetter setter, byte trim, TrimMode trimMode) + private static ColumnParser BuildFloat(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct { return (column, in context, ref model) => { - if (!FixedWidthRuntime.TryFloat(column, in context, out float value, trim, trimMode)) + if (!FixedWidthRuntime.TryFloat(column, in context, in options, out float value)) { return false; } @@ -266,12 +268,12 @@ private static ColumnParser BuildFloat(ColumnSetter BuildDateTime(ColumnSetter setter, string format, byte trim, TrimMode trimMode) + private static ColumnParser BuildDateTime(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct { return (column, in context, ref model) => { - if (!FixedWidthRuntime.TryDateTimeExact(column, format, in context, out var value, trim, trimMode)) + if (!FixedWidthRuntime.TryDateTimeExact(column, in context, in options, out DateTime value)) { return false; } @@ -280,12 +282,12 @@ private static ColumnParser BuildDateTime(ColumnSetter BuildDateOnly(ColumnSetter setter, string format, byte trim, TrimMode trimMode) + private static ColumnParser BuildDateOnly(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct { return (column, in context, ref model) => { - if (!FixedWidthRuntime.TryDateOnlyExact(column, format, in context, out var value, trim, trimMode)) + if (!FixedWidthRuntime.TryDateOnlyExact(column, in context, in options, out DateOnly value)) { return false; } @@ -294,12 +296,12 @@ private static ColumnParser BuildDateOnly(ColumnSetter BuildTimeOnly(ColumnSetter setter, string format, byte trim, TrimMode trimMode) + private static ColumnParser BuildTimeOnly(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct { return (column, in context, ref model) => { - if (!FixedWidthRuntime.TryTimeOnlyExact(column, format, in context, out var value, trim, trimMode)) + if (!FixedWidthRuntime.TryTimeOnlyExact(column, in context, in options, out TimeOnly value)) { return false; } @@ -308,12 +310,12 @@ private static ColumnParser BuildTimeOnly(ColumnSetter BuildDateTimeOffset(ColumnSetter setter, string format, byte trim, TrimMode trimMode) + private static ColumnParser BuildDateTimeOffset(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct { return (column, in context, ref model) => { - if (!FixedWidthRuntime.TryDateTimeOffsetExact(column, format, in context, out var value, trim, trimMode)) + if (!FixedWidthRuntime.TryDateTimeOffsetExact(column, in context, in options, out DateTimeOffset value)) { return false; } diff --git a/src/FixedWidthParser/Processors/CultureHelpers.cs b/src/FixedWidthParser/Processors/CultureHelpers.cs index 25fed3f..7dde005 100644 --- a/src/FixedWidthParser/Processors/CultureHelpers.cs +++ b/src/FixedWidthParser/Processors/CultureHelpers.cs @@ -51,12 +51,12 @@ private static char Resolve(IFormatProvider formatProvider) } /// Parses a (not yet trimmed) column as , honoring the context's provider. - public static bool TryParseDouble(ReadOnlySpan column, in FixedWidthContext context, out double value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) + public static bool TryParseDouble(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out double value) { // Also trim leading spaces: csFastFloat's characters_consumed does not reliably include // skipped leading whitespace, so a right-aligned column would otherwise fail the // full-consumption check below. - var trimmed = FixedWidthRuntime.TrimColumn(column, trimChar, trimMode).TrimStart((byte)' '); + var trimmed = FixedWidthRuntime.TrimColumn(column, in options).TrimStart((byte)' '); var formatProvider = context.FormatProvider; if (GetDecimalSeparator(formatProvider) == '.' && FastDoubleParser.TryParseDouble(trimmed, out int consumed, out value, decimal_separator: (byte)'.') @@ -68,9 +68,9 @@ public static bool TryParseDouble(ReadOnlySpan column, in FixedWidthContex } /// Parses a (not yet trimmed) column as , honoring the context's provider. - public static bool TryParseFloat(ReadOnlySpan column, in FixedWidthContext context, out float value, byte trimChar = (byte)' ', TrimMode trimMode = TrimMode.Trailing) + public static bool TryParseFloat(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out float value) { - var trimmed = FixedWidthRuntime.TrimColumn(column, trimChar, trimMode).TrimStart((byte)' '); + var trimmed = FixedWidthRuntime.TrimColumn(column, in options).TrimStart((byte)' '); var formatProvider = context.FormatProvider; if (GetDecimalSeparator(formatProvider) == '.' && FastFloatParser.TryParseFloat(trimmed, out int consumed, out value, decimal_separator: (byte)'.') diff --git a/src/FixedWidthParser/PublicAPI.Unshipped.txt b/src/FixedWidthParser/PublicAPI.Unshipped.txt index 3526499..f0ac24c 100644 --- a/src/FixedWidthParser/PublicAPI.Unshipped.txt +++ b/src/FixedWidthParser/PublicAPI.Unshipped.txt @@ -126,18 +126,10 @@ FixedWidthParser.Infrastructure.AsyncRecordEnumerator.DisposeAs FixedWidthParser.Infrastructure.AsyncRecordEnumerator.MoveNextAsync() -> System.Threading.Tasks.ValueTask FixedWidthParser.Infrastructure.ColumnFormatOptions FixedWidthParser.Infrastructure.ColumnFormatOptions.Alignment.get -> FixedWidthParser.Alignment -FixedWidthParser.Infrastructure.ColumnFormatOptions.Alignment.init -> void FixedWidthParser.Infrastructure.ColumnFormatOptions.ColumnFormatOptions() -> void -FixedWidthParser.Infrastructure.ColumnFormatOptions.ColumnFormatOptions(FixedWidthParser.Alignment Alignment, char Padding, string? Format, FixedWidthParser.OverflowBehavior Overflow) -> void -FixedWidthParser.Infrastructure.ColumnFormatOptions.Deconstruct(out FixedWidthParser.Alignment Alignment, out char Padding, out string? Format, out FixedWidthParser.OverflowBehavior Overflow) -> void -FixedWidthParser.Infrastructure.ColumnFormatOptions.Equals(FixedWidthParser.Infrastructure.ColumnFormatOptions other) -> bool FixedWidthParser.Infrastructure.ColumnFormatOptions.Format.get -> string? -FixedWidthParser.Infrastructure.ColumnFormatOptions.Format.init -> void FixedWidthParser.Infrastructure.ColumnFormatOptions.Overflow.get -> FixedWidthParser.OverflowBehavior -FixedWidthParser.Infrastructure.ColumnFormatOptions.Overflow.init -> void FixedWidthParser.Infrastructure.ColumnFormatOptions.Padding.get -> char -FixedWidthParser.Infrastructure.ColumnFormatOptions.Padding.init -> void -FixedWidthParser.Infrastructure.ColumnFormatOptions.WriteInto(System.ReadOnlySpan content, System.Span slice, string! columnName) -> void FixedWidthParser.Infrastructure.FixedWidthAsyncRecordEnumerable FixedWidthParser.Infrastructure.FixedWidthAsyncRecordEnumerable.GetAsyncEnumerator(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> FixedWidthParser.Infrastructure.AsyncRecordEnumerator! FixedWidthParser.Infrastructure.FixedWidthRecordEnumerable @@ -165,7 +157,6 @@ FixedWidthParser.TrimMode FixedWidthParser.TrimMode.Both = 2 -> FixedWidthParser.TrimMode FixedWidthParser.TrimMode.Leading = 1 -> FixedWidthParser.TrimMode FixedWidthParser.TrimMode.Trailing = 0 -> FixedWidthParser.TrimMode -override FixedWidthParser.Infrastructure.ColumnFormatOptions.GetHashCode() -> int static FixedWidthParser.FixedWidth.Read(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthRecordEnumerable>! static FixedWidthParser.FixedWidth.Read(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthRecordEnumerable>! static FixedWidthParser.FixedWidth.ReadAsync(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthAsyncRecordEnumerable>! @@ -174,26 +165,10 @@ static FixedWidthParser.FixedWidth.ReadFile(string! path, FixedWidthPars static FixedWidthParser.FixedWidth.ReadFile(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthRecordEnumerable>! static FixedWidthParser.FixedWidth.ReadFileAsync(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthAsyncRecordEnumerable>! static FixedWidthParser.FixedWidth.ReadFileAsync(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthAsyncRecordEnumerable>! -static FixedWidthParser.Infrastructure.ColumnFormatOptions.operator !=(FixedWidthParser.Infrastructure.ColumnFormatOptions left, FixedWidthParser.Infrastructure.ColumnFormatOptions right) -> bool -static FixedWidthParser.Infrastructure.ColumnFormatOptions.operator ==(FixedWidthParser.Infrastructure.ColumnFormatOptions left, FixedWidthParser.Infrastructure.ColumnFormatOptions right) -> bool static FixedWidthParser.Infrastructure.FixedWidthRuntime.FormatConvert(TValue value, System.Span slice, in FixedWidthParser.FixedWidthContext context, TConverter converter, FixedWidthParser.Infrastructure.ColumnFormatOptions options, string! columnName) -> void static FixedWidthParser.Infrastructure.FixedWidthRuntime.FormatString(string! value, System.Span slice, in FixedWidthParser.FixedWidthContext context, FixedWidthParser.Infrastructure.ColumnFormatOptions options, string! columnName) -> void static FixedWidthParser.Infrastructure.FixedWidthRuntime.FormatUtf8Value(TValue value, System.Span slice, in FixedWidthParser.FixedWidthContext context, FixedWidthParser.Infrastructure.ColumnFormatOptions options, string! columnName) -> void static FixedWidthParser.Infrastructure.FixedWidthRuntime.FormatValue(TValue value, System.Span slice, in FixedWidthParser.FixedWidthContext context, FixedWidthParser.Infrastructure.ColumnFormatOptions options, string! columnName) -> void -static FixedWidthParser.Infrastructure.FixedWidthRuntime.IsBlank(System.ReadOnlySpan column, byte trimChar) -> bool -static FixedWidthParser.Infrastructure.FixedWidthRuntime.String(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> string! -static FixedWidthParser.Infrastructure.FixedWidthRuntime.TrimColumn(System.ReadOnlySpan column, byte trimChar, FixedWidthParser.TrimMode trimMode) -> System.ReadOnlySpan -static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryConvert(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, TConverter converter, out TValue value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool -static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryDateOnlyExact(System.ReadOnlySpan column, string! format, in FixedWidthParser.FixedWidthContext context, out System.DateOnly value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool -static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryDateTimeExact(System.ReadOnlySpan column, string! format, in FixedWidthParser.FixedWidthContext context, out System.DateTime value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool -static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryDateTimeOffsetExact(System.ReadOnlySpan column, string! format, in FixedWidthParser.FixedWidthContext context, out System.DateTimeOffset value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool -static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryDouble(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out double value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool -static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryFloat(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out float value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool -static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryParse(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out TValue value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool -static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryParseDecoded(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out TValue value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool -static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryTimeOnlyExact(System.ReadOnlySpan column, string! format, in FixedWidthParser.FixedWidthContext context, out System.TimeOnly value, byte trimChar = 32, FixedWidthParser.TrimMode trimMode = FixedWidthParser.TrimMode.Trailing) -> bool -~override FixedWidthParser.Infrastructure.ColumnFormatOptions.Equals(object obj) -> bool -~override FixedWidthParser.Infrastructure.ColumnFormatOptions.ToString() -> string FixedWidthParser.IFixedWidthConverter.TryFormat(T value, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int bytesWritten) -> bool FixedWidthParser.IFixedWidthConverter.TryParse(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out T value) -> bool FixedWidthParser.FixedWidthReaderOptions.DegreeOfParallelism.get -> int @@ -202,3 +177,21 @@ static FixedWidthParser.FixedWidth.AggregateAsync(System.I static FixedWidthParser.FixedWidth.AggregateAsync(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! static FixedWidthParser.FixedWidth.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! static FixedWidthParser.FixedWidth.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +FixedWidthParser.Infrastructure.ColumnFormatOptions.ColumnFormatOptions(FixedWidthParser.Alignment alignment, char padding, string? format, FixedWidthParser.OverflowBehavior overflow) -> void +FixedWidthParser.Infrastructure.ColumnParseOptions +FixedWidthParser.Infrastructure.ColumnParseOptions.ColumnParseOptions() -> void +FixedWidthParser.Infrastructure.ColumnParseOptions.ColumnParseOptions(char trimChar, FixedWidthParser.TrimMode trimMode, string? format) -> void +FixedWidthParser.Infrastructure.ColumnParseOptions.Format.get -> string? +FixedWidthParser.Infrastructure.ColumnParseOptions.TrimChar.get -> char +FixedWidthParser.Infrastructure.ColumnParseOptions.TrimMode.get -> FixedWidthParser.TrimMode +static FixedWidthParser.Infrastructure.FixedWidthRuntime.IsBlank(System.ReadOnlySpan column, in FixedWidthParser.Infrastructure.ColumnParseOptions options) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.String(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options) -> string! +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryConvert(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, TConverter converter, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out TValue value) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryDateOnlyExact(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out System.DateOnly value) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryDateTimeExact(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out System.DateTime value) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryDateTimeOffsetExact(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out System.DateTimeOffset value) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryDouble(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out double value) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryFloat(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out float value) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryParse(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out TValue value) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryParseDecoded(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out TValue value) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryTimeOnlyExact(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out System.TimeOnly value) -> bool diff --git a/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs b/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs index 9fd865b..056ce1e 100644 --- a/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs +++ b/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs @@ -25,10 +25,10 @@ public void ValidModel_GeneratesTryParse() var generated = Assert.Single(result.GeneratedSources); Assert.Contains("TryParse(global::System.ReadOnlySpan line, in global::FixedWidthParser.FixedWidthContext context", generated, StringComparison.Ordinal); Assert.Contains("if (line.Length < 18) { context.ReportShortLine(18); model = default!; return false; }", generated, StringComparison.Ordinal); - Assert.Contains("FixedWidthRuntime.String(line.Slice(0, 5), context)", generated, StringComparison.Ordinal); - Assert.Contains("FixedWidthRuntime.TryParse(line.Slice(5, 3), context, out int __v1)) { context.ReportFailedColumn(\"Quantity\", 5, 3); model = default!; return false; }", generated, StringComparison.Ordinal); - Assert.Contains("FixedWidthRuntime.TryDouble(line.Slice(8, 5), context", generated, StringComparison.Ordinal); - Assert.Contains("FixedWidthRuntime.TryFloat(line.Slice(13, 5), context", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.String(line.Slice(0, 5), context, in __parse0)", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.TryParse(line.Slice(5, 3), context, in __parse1, out int __v1)) { context.ReportFailedColumn(\"Quantity\", 5, 3); model = default!; return false; }", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.TryDouble(line.Slice(8, 5), context, in __parse2", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.TryFloat(line.Slice(13, 5), context, in __parse3", generated, StringComparison.Ordinal); Assert.DoesNotContain("ReadOnlySpan", generated, StringComparison.Ordinal); Assert.Contains("Code = __v0", generated, StringComparison.Ordinal); Assert.Contains("Discount = __v3", generated, StringComparison.Ordinal); @@ -81,9 +81,10 @@ public sealed class CentsConverter : FixedWidthParser.IFixedWidthConverter Date: Fri, 23 Oct 2026 23:26:58 -0300 Subject: [PATCH 30/85] refactor: hide generated-code plumbing from IntelliSense Every type in FixedWidthParser.Infrastructure and FixedWidthContext's ReportFailedColumn/ReportShortLine are public only for generated code and allocation-free enumeration; [EditorBrowsable(Never)] keeps them out of completion lists so callers do not build on them. Co-Authored-By: Claude Opus 5.5 --- src/FixedWidthParser/FixedWidthContext.cs | 3 +++ src/FixedWidthParser/Infrastructure/AsyncRecordEnumerator.cs | 2 ++ src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs | 2 ++ src/FixedWidthParser/Infrastructure/ColumnParseOptions.cs | 3 +++ .../Infrastructure/FixedWidthAsyncRecordEnumerable.cs | 3 +++ .../Infrastructure/FixedWidthRecordEnumerable.cs | 2 ++ src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs | 2 ++ src/FixedWidthParser/Infrastructure/RecordEnumerator.cs | 2 ++ src/FixedWidthParser/Infrastructure/RecordLineParsers.cs | 5 +++++ 9 files changed, 24 insertions(+) diff --git a/src/FixedWidthParser/FixedWidthContext.cs b/src/FixedWidthParser/FixedWidthContext.cs index 124ac1f..0d8b1fd 100644 --- a/src/FixedWidthParser/FixedWidthContext.cs +++ b/src/FixedWidthParser/FixedWidthContext.cs @@ -1,3 +1,4 @@ +using System.ComponentModel; using System.Globalization; using System.Text; using CommunityToolkit.HighPerformance.Buffers; @@ -79,6 +80,7 @@ public Encoding Encoding /// . Generated parsers call this before returning /// ; outside a reader's diagnosis it does nothing. /// + [EditorBrowsable(EditorBrowsableState.Never)] public void ReportFailedColumn(string columnName, int start, int length) { Failure?.SetColumn(columnName, start, length); @@ -89,6 +91,7 @@ public void ReportFailedColumn(string columnName, int start, int length) /// Generated parsers call this before returning ; outside a reader's /// diagnosis it does nothing. /// + [EditorBrowsable(EditorBrowsableState.Never)] public void ReportShortLine(int requiredLength) { Failure?.SetShortLine(requiredLength); diff --git a/src/FixedWidthParser/Infrastructure/AsyncRecordEnumerator.cs b/src/FixedWidthParser/Infrastructure/AsyncRecordEnumerator.cs index abf8903..dd4ad5a 100644 --- a/src/FixedWidthParser/Infrastructure/AsyncRecordEnumerator.cs +++ b/src/FixedWidthParser/Infrastructure/AsyncRecordEnumerator.cs @@ -1,3 +1,4 @@ +using System.ComponentModel; using FixedWidthParser.Readers; namespace FixedWidthParser.Infrastructure @@ -6,6 +7,7 @@ namespace FixedWidthParser.Infrastructure /// Asynchronous record enumerator over a stream. The line scan stays synchronous; only the refill /// awaits the stream. /// + [EditorBrowsable(EditorBrowsableState.Never)] public sealed class AsyncRecordEnumerator : IAsyncEnumerator where TParser : struct, IRecordLineParser { diff --git a/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs b/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs index 528afb1..de302eb 100644 --- a/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs +++ b/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs @@ -1,3 +1,4 @@ +using System.ComponentModel; using System.Text; namespace FixedWidthParser.Infrastructure @@ -14,6 +15,7 @@ namespace FixedWidthParser.Infrastructure /// constructor overload and a new property; the existing constructor is never changed. /// #pragma warning disable CA1815 // Never compared; value equality is meaningless for it. + [EditorBrowsable(EditorBrowsableState.Never)] public readonly struct ColumnFormatOptions #pragma warning restore CA1815 { diff --git a/src/FixedWidthParser/Infrastructure/ColumnParseOptions.cs b/src/FixedWidthParser/Infrastructure/ColumnParseOptions.cs index 5467199..9a1c635 100644 --- a/src/FixedWidthParser/Infrastructure/ColumnParseOptions.cs +++ b/src/FixedWidthParser/Infrastructure/ColumnParseOptions.cs @@ -1,3 +1,5 @@ +using System.ComponentModel; + namespace FixedWidthParser.Infrastructure { /// @@ -10,6 +12,7 @@ namespace FixedWidthParser.Infrastructure /// constructor overload and a new property; the existing constructor is never changed. /// #pragma warning disable CA1815 // Never compared; value equality is meaningless for it. + [EditorBrowsable(EditorBrowsableState.Never)] public readonly struct ColumnParseOptions #pragma warning restore CA1815 { diff --git a/src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs b/src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs index 10ff8dd..7c603d1 100644 --- a/src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs +++ b/src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs @@ -1,3 +1,5 @@ +using System.ComponentModel; + namespace FixedWidthParser.Infrastructure { /// @@ -5,6 +7,7 @@ namespace FixedWidthParser.Infrastructure /// await foreach. A stream is read once; a file is reopened, with asynchronous I/O, on each /// enumeration. /// + [EditorBrowsable(EditorBrowsableState.Never)] public sealed class FixedWidthAsyncRecordEnumerable : IAsyncEnumerable where TParser : struct, IRecordLineParser { diff --git a/src/FixedWidthParser/Infrastructure/FixedWidthRecordEnumerable.cs b/src/FixedWidthParser/Infrastructure/FixedWidthRecordEnumerable.cs index b9fa8d8..26a47a0 100644 --- a/src/FixedWidthParser/Infrastructure/FixedWidthRecordEnumerable.cs +++ b/src/FixedWidthParser/Infrastructure/FixedWidthRecordEnumerable.cs @@ -1,4 +1,5 @@ using System.Collections; +using System.ComponentModel; using System.Diagnostics.CodeAnalysis; namespace FixedWidthParser.Infrastructure @@ -9,6 +10,7 @@ namespace FixedWidthParser.Infrastructure /// enumerator for allocation-free foreach. A stream is read once; a /// file is reopened on each enumeration. /// + [EditorBrowsable(EditorBrowsableState.Never)] public sealed class FixedWidthRecordEnumerable : IEnumerable where TParser : struct, IRecordLineParser { diff --git a/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs index b45b01b..23b9c3d 100644 --- a/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs +++ b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs @@ -1,5 +1,6 @@ using System.Buffers.Text; using System.Buffers; +using System.ComponentModel; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Runtime.CompilerServices; @@ -15,6 +16,7 @@ namespace FixedWidthParser.Infrastructure /// and writer, so both paths share one set of semantics. /// Public because the generated code lives in the consumer assembly; not intended for direct use. /// + [EditorBrowsable(EditorBrowsableState.Never)] public static class FixedWidthRuntime { private const int StackBufferSize = 256; diff --git a/src/FixedWidthParser/Infrastructure/RecordEnumerator.cs b/src/FixedWidthParser/Infrastructure/RecordEnumerator.cs index 71fb436..ee0681f 100644 --- a/src/FixedWidthParser/Infrastructure/RecordEnumerator.cs +++ b/src/FixedWidthParser/Infrastructure/RecordEnumerator.cs @@ -1,4 +1,5 @@ using System.Collections; +using System.ComponentModel; using System.Diagnostics.CodeAnalysis; using FixedWidthParser.Readers; @@ -8,6 +9,7 @@ namespace FixedWidthParser.Infrastructure /// Synchronous record enumerator over a stream. A specialized by /// , so foreach allocates nothing and parsing devirtualizes. /// + [EditorBrowsable(EditorBrowsableState.Never)] public struct RecordEnumerator : IEnumerator where TParser : struct, IRecordLineParser { diff --git a/src/FixedWidthParser/Infrastructure/RecordLineParsers.cs b/src/FixedWidthParser/Infrastructure/RecordLineParsers.cs index e4bd882..b793856 100644 --- a/src/FixedWidthParser/Infrastructure/RecordLineParsers.cs +++ b/src/FixedWidthParser/Infrastructure/RecordLineParsers.cs @@ -1,3 +1,5 @@ +using System.ComponentModel; + namespace FixedWidthParser.Infrastructure { /// @@ -6,6 +8,7 @@ namespace FixedWidthParser.Infrastructure /// devirtualizes into a direct call. Public only because it appears in the /// enumerator types; the library is its only implementer. /// + [EditorBrowsable(EditorBrowsableState.Never)] public interface IRecordLineParser where TModel : allows ref struct { /// Parses a buffered line into a model. @@ -16,6 +19,7 @@ public interface IRecordLineParser where TModel : allows ref struct // value equality is meaningless for them. #pragma warning disable CA1815 // Override equals and operator equals on value types /// Layout parse strategy: forwards to a , built from attributes or fluently. + [EditorBrowsable(EditorBrowsableState.Never)] public readonly struct LayoutLineParser : IRecordLineParser { private readonly FixedWidthLayout _layout; @@ -33,6 +37,7 @@ public bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out } /// Source-generated parse strategy: forwards to the model's static . + [EditorBrowsable(EditorBrowsableState.Never)] public readonly struct GeneratedLineParser : IRecordLineParser where TModel : IFixedWidthModel { /// From c556c08e94d23fe7802a3808946c1737fd896146 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 23 Oct 2026 23:29:37 -0300 Subject: [PATCH 31/85] docs: describe the settled public surface and the 2.x compatibility rules README's versioning section no longer claims the enumerator and strategy types can change in a minor version; it states that everything public, the Infrastructure namespace and generated-code helpers included, stays binary compatible across 2.x, and how settings are added. Samples use the root namespace and FixedWidthReaderOptions.DegreeOfParallelism. ARCHITECTURE gains a public-surface section; CHANGELOG covers the namespace move and renames. Package validation suppressions follow the moved types. Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 17 +++- CHANGELOG.md | 9 +- README.md | 17 ++-- .../CompatibilitySuppressions.xml | 91 +++++++++---------- 4 files changed, 72 insertions(+), 62 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3dfacbb..ebbbccf 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -19,7 +19,7 @@ Every model is parsed and written by one of two paths, both behind the static `F | Call | `FixedWidth.X(…, layout, …)` | `FixedWidth.X(…)`, calling the generated statics | | Built | when the layout is built: by reflection and compiled expression trees for attributes, from delegates for `Build` | at compile time by `FixedWidthModelGenerator` | -Both paths call the same helpers in `FixedWidthRuntime` (`src/FixedWidthParser/FixedWidthRuntime.cs`) +Both paths call the same helpers in `FixedWidthRuntime` (`src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs`) for every column: trim, decode a string, parse a number, format a value, pad and align. That class is public only because generated code lives in the consumer's assembly. Keeping the semantics in one place is what makes the two paths agree; `SourceGenParityTests` checks that they do. @@ -85,7 +85,7 @@ source -> line splitter -> IRecordLineParser -> model Two enumerators share one shape, each specialized by a `struct` parse strategy (`IRecordLineParser`: `LayoutLineParser` or `GeneratedLineParser`, in -`Readers/ILineParser.cs`). Because the strategy is a `struct` type argument, the JIT compiles one +`Infrastructure/RecordLineParsers.cs`). Because the strategy is a `struct` type argument, the JIT compiles one enumerator per `(TModel, TParser)` pair and the parse call devirtualizes. | Source | Enumerator | @@ -187,7 +187,18 @@ change a model's columns does not regenerate it. `Emit` validates the model — partial, top-level, a supported type for each column, layout, converter type, overflow, ASCII padding — and reports diagnostics `FWP001`–`FWP012` instead of generating code for an invalid model. Otherwise it writes `TryParse` and `TryFormat` as straight-line code: one -`FixedWidthRuntime` call per column, with every attribute setting emitted as a constant. +`FixedWidthRuntime` call per column. Each column's settings become two `static readonly` fields, a +`ColumnParseOptions` and a `ColumnFormatOptions`, passed to the helpers by `in`. + +## Public surface + +User-facing types live in the root `FixedWidthParser` namespace. `FixedWidthParser.Infrastructure` +holds what is public only because generated code in the consumer's assembly calls it, or because a +`FixedWidth` method returns it so `foreach` allocates nothing; it is marked +`[EditorBrowsable(Never)]`. Compiled consumer code binds to those signatures, so they evolve only by +addition: a new per-column setting is a new constructor overload and property on `ColumnParseOptions` +or `ColumnFormatOptions`, never a change to an existing helper. Internal types keep their folder +namespaces (`Readers`, `Writers`, `Parsers`, `Processors`, `Formatters`, `Attributes`). ## Further reading diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b6ca46..83a3c9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,8 @@ in options objects and a `FixedWidthContext`, so later versions can add to them | `IFixedWidthConverter` + `IUtf8FixedWidthConverter` with `IFormatProvider?` | `IFixedWidthConverter` over bytes, with `in FixedWidthContext` | | `FixedWidthWriter.Write*(StreamWriter or Stream, …, provider)` | `FixedWidth.Write*(Stream, …, FixedWidthLayout.FromAttributes(), new FixedWidthWriterOptions { FormatProvider = …, Encoding = …, NewLine = … })` | | `catch (FormatException)` on a bad line | still works; the exception is now `FixedWidthParseException` | +| `using FixedWidthParser.Attributes;`, `using FixedWidthParser.Processors;` | `using FixedWidthParser;` — `FixedColumnAttribute`, `Alignment`, `TrimMode`, `OverflowBehavior` and `IFixedWidthConverter` moved to the root namespace | +| `IFixedWidthConverter.TryParse(field, …)`, `TryFormat(…, out int written)` | the same, with the parameters named `column` and `bytesWritten` | A UTF-8 file with multi-byte text whose layout counts characters no longer lines up: positions are bytes. Read files written in a single-byte encoding with that encoding (`Encoding.Latin1`, or @@ -47,7 +49,7 @@ Windows-1252 after registering `CodePagesEncodingProvider`). - Parallel aggregation: `FixedWidth.AggregateFileAsync`, for generated models and layouts alike, reads a file in byte-range partitions on several threads and fold each into an `IRecordAccumulator`, merged in file order. Every line is parsed exactly once, and - errors carry the same line number as a sequential read. `FixedWidthParallelOptions.DegreeOfParallelism` + errors carry the same line number as a sequential read. `FixedWidthReaderOptions.DegreeOfParallelism` sets the thread count. `AggregateAsync` does the same over a `Stream`: one thread reads chunks, the others parse them. - `FixedWidthReaderOptions`: `FormatProvider`, `Encoding`, `StringPool`, `BufferSize`, @@ -85,6 +87,11 @@ Windows-1252 after registering `CodePagesEncodingProvider`). ### Changed - **Breaking:** bytes only; see the table above. The column positions of a model are bytes. +- **Breaking:** `FixedColumnAttribute`, `Alignment`, `TrimMode`, `OverflowBehavior` and + `IFixedWidthConverter` moved to the `FixedWidthParser` namespace. The types public only for + generated code or allocation-free enumeration (`FixedWidthRuntime`, `ColumnFormatOptions`, the new + `ColumnParseOptions`, the record enumerables and enumerators, `IRecordLineParser` and the + line-parser structs) moved to `FixedWidthParser.Infrastructure` and are hidden from IntelliSense. - **Breaking:** targets `net10.0` only. .NET 8 reaches end of support on 10 November 2026. - **Breaking:** a `null` format provider means the invariant culture for every column type, on parse and write. 1.x used it for `double`/`float` only and fell back to `CurrentCulture` for the rest. diff --git a/README.md b/README.md index 086433c..7a74203 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ The package ships the Roslyn source generator bundled as an analyzer. Models tha An attribute model needs a public parameterless constructor and mapped fields or properties; `FixedWidthLayout.FromAttributes()` turns it into a layout, built once per type by reflection: ```csharp -using FixedWidthParser.Attributes; +using FixedWidthParser; public readonly record struct Person { @@ -71,7 +71,6 @@ For source generation, make the model `partial` and implement the marker interfa ```csharp using FixedWidthParser; -using FixedWidthParser.Attributes; public readonly partial record struct GeneratedPerson : IFixedWidthModel { @@ -87,7 +86,6 @@ For a type you cannot annotate, or a layout known only at run time, describe the ```csharp using FixedWidthParser; -using FixedWidthParser.Attributes; var layout = FixedWidthLayout.Build(b => b .Factory(() => new Order()) @@ -173,8 +171,8 @@ var generated = await FixedWidth.AggregateFileAsync` (normally through the generator) and `IFixedWidthConverter` — never gain a member in a minor version. New settings arrive as new properties on `FixedWidthContext`, which every call receives. -- **Types that appear in signatures only so enumeration stays allocation-free** — `IRecordLineParser`, the `*LineParser` strategy structs and the `*RecordEnumerable`/`*RecordEnumerator` types — are implemented only by the library. Program against `IEnumerable`/`IAsyncEnumerable`; their shape can change in a minor version. -- **`FixedWidthRuntime`** and **`ColumnFormatOptions`** are public because generated code calls them; they are not meant to be called directly. +- **Settings** arrive as new properties on the options objects (`FixedWidthReaderOptions`, `FixedWidthWriterOptions`, `FixedWidthColumnOptions`) or on `FixedWidthContext`, never as a new parameter on an existing method. +- **Interfaces you implement** — `IFixedWidthModel` (normally through the generator), `IFixedWidthConverter` and `IRecordAccumulator` — never gain a required member. Anything added comes with a default implementation (`static virtual` or a default interface method). +- **`FixedWidthParser.Infrastructure`** holds what is public only because generated code calls it (`FixedWidthRuntime`, `ColumnParseOptions`, `ColumnFormatOptions`) or because a method returns it so `foreach` allocates nothing (the `*RecordEnumerable`/`*RecordEnumerator` types and the `*LineParser` strategies). These types are hidden from IntelliSense and not meant for direct use, but they follow the same rules as everything else: code compiled against one 2.x version, generated code included, keeps working with every later 2.x version. A new setting for generated code is a new constructor overload on the options structs. ## Performance diff --git a/src/FixedWidthParser/CompatibilitySuppressions.xml b/src/FixedWidthParser/CompatibilitySuppressions.xml index 1055937..96f013b 100644 --- a/src/FixedWidthParser/CompatibilitySuppressions.xml +++ b/src/FixedWidthParser/CompatibilitySuppressions.xml @@ -1,6 +1,13 @@ + + CP0001 + T:FixedWidthParser.Attributes.Alignment + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + CP0001 T:FixedWidthParser.Attributes.ColumnLayoutValidator @@ -8,6 +15,27 @@ lib/net10.0/FixedWidthParser.dll true + + CP0001 + T:FixedWidthParser.Attributes.FixedColumnAttribute + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.Attributes.OverflowBehavior + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + + + CP0001 + T:FixedWidthParser.FixedWidthRuntime + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + CP0001 T:FixedWidthParser.FixedWidthUtf8 @@ -15,6 +43,13 @@ lib/net10.0/FixedWidthParser.dll true + + CP0001 + T:FixedWidthParser.Formatters.ColumnFormatOptions + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + CP0001 T:FixedWidthParser.Formatters.IColumnFormatter`1 @@ -134,6 +169,13 @@ lib/net10.0/FixedWidthParser.dll true + + CP0001 + T:FixedWidthParser.Readers.GeneratedLineParser`1 + lib/net10.0/FixedWidthParser.dll + lib/net10.0/FixedWidthParser.dll + true + CP0001 T:FixedWidthParser.Readers.GeneratedUtf8FixedWidthAsyncRecordEnumerable`1 @@ -302,41 +344,6 @@ lib/net10.0/FixedWidthParser.dll true - - CP0002 - M:FixedWidthParser.FixedWidthRuntime.String(System.ReadOnlySpan{System.Char},CommunityToolkit.HighPerformance.Buffers.StringPool) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.FixedWidthRuntime.TryDouble(System.ReadOnlySpan{System.Char},System.IFormatProvider,System.Double@) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.FixedWidthRuntime.TryFloat(System.ReadOnlySpan{System.Char},System.IFormatProvider,System.Single@) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.FixedWidthRuntime.TryParse``1(System.ReadOnlySpan{System.Char},System.IFormatProvider,``0@) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.Formatters.ColumnFormatOptions.WriteInto(System.ReadOnlySpan{System.Char},System.Span{System.Char},System.String) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - CP0002 M:FixedWidthParser.IFixedWidthModel`1.TryParse(System.ReadOnlySpan{System.Char},System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,`0@) @@ -344,13 +351,6 @@ lib/net10.0/FixedWidthParser.dll true - - CP0002 - M:FixedWidthParser.Readers.GeneratedLineParser`1.TryParse(System.ReadOnlySpan{System.Char},System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,`0@) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - CP0006 M:FixedWidthParser.IFixedWidthModel`1.TryFormat(`0@,System.Span{System.Byte},FixedWidthParser.FixedWidthContext@,System.Int32@) @@ -365,13 +365,6 @@ lib/net10.0/FixedWidthParser.dll true - - CP0008 - T:FixedWidthParser.Readers.GeneratedLineParser`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - PKV006 net8.0 From 487d88917cf64bf5f26870269f4db35dfc5be3e5 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 23 Oct 2026 23:45:38 -0300 Subject: [PATCH 32/85] feat: accept ref struct models in the readers and aggregates Read, ReadFile, ReadAsync, ReadFileAsync, AggregateFileAsync and AggregateAsync, and IRecordAccumulator, now take `allows ref struct` models, generated and through a layout. An enumerator cannot hold a ref struct field, so, following ExcelReader, MoveNext only finds the next line and Current parses it on access. A foreach still parses each line once. With OnParseError set, MoveNext also parses the line so a bad one can be skipped before it is yielded. --- ARCHITECTURE.md | 8 +- CHANGELOG.md | 7 + README.md | 2 +- src/FixedWidthParser/FixedWidth.cs | 20 +- src/FixedWidthParser/IRecordAccumulator.cs | 2 + .../Infrastructure/AsyncRecordEnumerator.cs | 16 +- .../FixedWidthAsyncRecordEnumerable.cs | 1 + .../FixedWidthRecordEnumerable.cs | 1 + .../Infrastructure/RecordEnumerator.cs | 20 +- .../Infrastructure/RecordLineParsers.cs | 4 +- .../Readers/LineBufferState.cs | 9 +- .../Readers/ParallelAggregator.cs | 4 + .../Readers/ParallelStreamAggregator.cs | 6 + src/FixedWidthParser/Readers/ParseFailure.cs | 31 +++ .../RefStructReadTests.cs | 202 ++++++++++++++++++ 15 files changed, 307 insertions(+), 26 deletions(-) create mode 100644 tests/FixedWidthParser.Tests/RefStructReadTests.cs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ebbbccf..34c80c0 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -71,9 +71,11 @@ sources (`TryBuildSpecial`) and mirrored by the generator's `Classify`; the form string, `IUtf8SpanFormattable`, `ISpanFormattable`, a nullable wrapper) is mirrored by `ClassifyWrite`. Column options are validated by `FormatterBuilder.ResolveOptions` for both. -A layout accepts `ref struct` models (`allows ref struct`) for `TryParse` and `TryFormat`; the readers -do not, since an enumerator has to hold its current model in a field, and an attribute layout over a -`ref struct` does not write. +A layout accepts `ref struct` models (`allows ref struct`) for `TryParse`, `TryFormat`, the readers and +the aggregates; an attribute layout over a `ref struct` does not write. An enumerator cannot hold a +`ref struct` in a field, so `MoveNext` only finds the next line and `Current` +parses it on access; `foreach` reads `Current` once, so each line is still parsed once. With +`OnParseError` set, `MoveNext` parses the line too, to skip a bad one before yielding it. ## Reading diff --git a/CHANGELOG.md b/CHANGELOG.md index 83a3c9c..fc32e2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,10 @@ Windows-1252 after registering `CodePagesEncodingProvider`). `ColumnLength` and `RawValue`. A short line says how long it is and how long the layout needs it to be. A line that parses pays nothing for this: a failed line is parsed again to find the column. - `FixedWidthReaderOptions.OnParseError`: return `true` to skip a line that fails to parse. +- `ref struct` models on `Read`, `ReadFile`, `ReadAsync`, `ReadFileAsync`, `AggregateFileAsync` and + `AggregateAsync`, generated and through a layout. The enumerators parse a line when `Current` is + read, so reading it twice parses twice; with `OnParseError` set, each line is parsed once more to + decide whether to skip it. - `FixedWidthReaderOptions.MaxLineLength` (default 1 MiB): a longer line throws `InvalidDataException`. Before, a source without line breaks doubled the read buffer until memory ran out, which a request body made reachable from the network. @@ -92,6 +96,9 @@ Windows-1252 after registering `CodePagesEncodingProvider`). generated code or allocation-free enumeration (`FixedWidthRuntime`, `ColumnFormatOptions`, the new `ColumnParseOptions`, the record enumerables and enumerators, `IRecordLineParser` and the line-parser structs) moved to `FixedWidthParser.Infrastructure` and are hidden from IntelliSense. +- **Breaking:** without `OnParseError`, a line that fails to parse throws when the enumerator's + `Current` is read, not from `MoveNext`. A `foreach` sees no difference. The non-generic + `IEnumerator.Current` throws `NotSupportedException`. - **Breaking:** targets `net10.0` only. .NET 8 reaches end of support on 10 November 2026. - **Breaking:** a `null` format provider means the invariant culture for every column type, on parse and write. 1.x used it for `double`/`float` only and fell back to `CurrentCulture` for the rest. diff --git a/README.md b/README.md index 7a74203..70a4f46 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Upgrading from 1.x? See [the 2.0 entry in the changelog](CHANGELOG.md#200). - **Configurable formatting** per column: alignment, padding character, format string and explicit overflow policy. - **Culture-aware** numbers and dates, including `double`/`float` via csFastFloat and any `IUtf8SpanParsable`/`ISpanParsable` type. - **Layout validation** when a layout is built or generated: negative `Start`, non-positive `Length`, overlapping columns and non-ASCII padding fail clearly. -- **`ref struct` model support** on single-line parsing and formatting. +- **`ref struct` model support** on single-line parsing and formatting, on every reader and on parallel aggregation. ## Requirements diff --git a/src/FixedWidthParser/FixedWidth.cs b/src/FixedWidthParser/FixedWidth.cs index 4a5ca31..e041363 100644 --- a/src/FixedWidthParser/FixedWidth.cs +++ b/src/FixedWidthParser/FixedWidth.cs @@ -55,7 +55,7 @@ public static bool TryFormat(in TModel model, FixedWidthLayout l /// Reads from a stream, once. controls whether the stream is disposed when iteration ends. public static FixedWidthRecordEnumerable> Read(Stream stream, bool leaveOpen = false, FixedWidthReaderOptions? options = null) - where TModel : IFixedWidthModel + where TModel : IFixedWidthModel, allows ref struct { ArgumentNullException.ThrowIfNull(stream); return new(default, stream, ownsStream: !leaveOpen, FixedWidthReaderOptions.Validate(options)); @@ -64,6 +64,7 @@ public static FixedWidthRecordEnumerable> Re /// Reads from a stream through , once. controls whether the stream is disposed when iteration ends. /// has a column without a setter. public static FixedWidthRecordEnumerable> Read(Stream stream, FixedWidthLayout layout, bool leaveOpen = false, FixedWidthReaderOptions? options = null) + where TModel : allows ref struct { ArgumentNullException.ThrowIfNull(stream); return new(ReadableLayout(layout), stream, ownsStream: !leaveOpen, FixedWidthReaderOptions.Validate(options)); @@ -71,7 +72,7 @@ public static FixedWidthRecordEnumerable> Read< /// Reads from a file, reopening it on each enumeration. public static FixedWidthRecordEnumerable> ReadFile(string path, FixedWidthReaderOptions? options = null) - where TModel : IFixedWidthModel + where TModel : IFixedWidthModel, allows ref struct { ArgumentException.ThrowIfNullOrEmpty(path); return new(default, path, FixedWidthReaderOptions.Validate(options)); @@ -80,6 +81,7 @@ public static FixedWidthRecordEnumerable> Re /// Reads from a file through , reopening it on each enumeration. /// has a column without a setter. public static FixedWidthRecordEnumerable> ReadFile(string path, FixedWidthLayout layout, FixedWidthReaderOptions? options = null) + where TModel : allows ref struct { ArgumentException.ThrowIfNullOrEmpty(path); return new(ReadableLayout(layout), path, FixedWidthReaderOptions.Validate(options)); @@ -87,7 +89,7 @@ public static FixedWidthRecordEnumerable> ReadF /// Reads from a stream via await foreach, once. controls whether the stream is disposed when iteration ends. public static FixedWidthAsyncRecordEnumerable> ReadAsync(Stream stream, bool leaveOpen = false, FixedWidthReaderOptions? options = null) - where TModel : IFixedWidthModel + where TModel : IFixedWidthModel, allows ref struct { ArgumentNullException.ThrowIfNull(stream); return new(default, stream, ownsStream: !leaveOpen, FixedWidthReaderOptions.Validate(options)); @@ -96,6 +98,7 @@ public static FixedWidthAsyncRecordEnumerableReads from a stream through via await foreach, once. controls whether the stream is disposed when iteration ends. /// has a column without a setter. public static FixedWidthAsyncRecordEnumerable> ReadAsync(Stream stream, FixedWidthLayout layout, bool leaveOpen = false, FixedWidthReaderOptions? options = null) + where TModel : allows ref struct { ArgumentNullException.ThrowIfNull(stream); return new(ReadableLayout(layout), stream, ownsStream: !leaveOpen, FixedWidthReaderOptions.Validate(options)); @@ -103,7 +106,7 @@ public static FixedWidthAsyncRecordEnumerable> /// Reads from a file via await foreach with asynchronous I/O, reopening it on each enumeration. public static FixedWidthAsyncRecordEnumerable> ReadFileAsync(string path, FixedWidthReaderOptions? options = null) - where TModel : IFixedWidthModel + where TModel : IFixedWidthModel, allows ref struct { ArgumentException.ThrowIfNullOrEmpty(path); return new(default, path, FixedWidthReaderOptions.Validate(options)); @@ -112,6 +115,7 @@ public static FixedWidthAsyncRecordEnumerableReads from a file through via await foreach with asynchronous I/O, reopening it on each enumeration. /// has a column without a setter. public static FixedWidthAsyncRecordEnumerable> ReadFileAsync(string path, FixedWidthLayout layout, FixedWidthReaderOptions? options = null) + where TModel : allows ref struct { ArgumentException.ThrowIfNullOrEmpty(path); return new(ReadableLayout(layout), path, FixedWidthReaderOptions.Validate(options)); @@ -129,7 +133,7 @@ public static FixedWidthAsyncRecordEnumerable> /// public static Task AggregateFileAsync( string path, FixedWidthReaderOptions? options = null, CancellationToken cancellationToken = default) - where TModel : IFixedWidthModel + where TModel : IFixedWidthModel, allows ref struct where TAccumulator : IRecordAccumulator, new() { ArgumentException.ThrowIfNullOrEmpty(path); @@ -145,6 +149,7 @@ public static Task AggregateFileAsync( /// has a column without a setter. public static Task AggregateFileAsync( string path, FixedWidthLayout layout, FixedWidthReaderOptions? options = null, CancellationToken cancellationToken = default) + where TModel : allows ref struct where TAccumulator : IRecordAccumulator, new() { ArgumentException.ThrowIfNullOrEmpty(path); @@ -159,7 +164,7 @@ public static Task AggregateFileAsync( /// Carries the same contract as . public static Task AggregateAsync( Stream stream, bool leaveOpen = false, FixedWidthReaderOptions? options = null, CancellationToken cancellationToken = default) - where TModel : IFixedWidthModel + where TModel : IFixedWidthModel, allows ref struct where TAccumulator : IRecordAccumulator, new() { ArgumentNullException.ThrowIfNull(stream); @@ -176,6 +181,7 @@ public static Task AggregateAsync( /// has a column without a setter. public static Task AggregateAsync( Stream stream, FixedWidthLayout layout, bool leaveOpen = false, FixedWidthReaderOptions? options = null, CancellationToken cancellationToken = default) + where TModel : allows ref struct where TAccumulator : IRecordAccumulator, new() { ArgumentNullException.ThrowIfNull(stream); @@ -183,7 +189,6 @@ public static Task AggregateAsync( ReadableLayout(layout), stream, leaveOpen, FixedWidthReaderOptions.Validate(options), cancellationToken); } - /// Writes one record and its line break. /// A value does not fit a column whose overflow throws. public static void Write(Stream stream, in TModel model, FixedWidthWriterOptions? options = null) @@ -282,6 +287,7 @@ private static LayoutRecordFormatter WritableLayout(FixedWidthLa } private static LayoutLineParser ReadableLayout(FixedWidthLayout layout) + where TModel : allows ref struct { ArgumentNullException.ThrowIfNull(layout); layout.EnsureCanRead(); diff --git a/src/FixedWidthParser/IRecordAccumulator.cs b/src/FixedWidthParser/IRecordAccumulator.cs index 3f3f876..0bd62e8 100644 --- a/src/FixedWidthParser/IRecordAccumulator.cs +++ b/src/FixedWidthParser/IRecordAccumulator.cs @@ -20,8 +20,10 @@ namespace FixedWidthParser /// public interface IRecordAccumulator where TSelf : IRecordAccumulator + where TModel : allows ref struct { /// Folds one record into this instance. + /// The record. It is valid only for the duration of the call. void Add(in TModel model); /// Folds the records of the partition that immediately follows this one into this instance. diff --git a/src/FixedWidthParser/Infrastructure/AsyncRecordEnumerator.cs b/src/FixedWidthParser/Infrastructure/AsyncRecordEnumerator.cs index dd4ad5a..48a5cd7 100644 --- a/src/FixedWidthParser/Infrastructure/AsyncRecordEnumerator.cs +++ b/src/FixedWidthParser/Infrastructure/AsyncRecordEnumerator.cs @@ -9,6 +9,7 @@ namespace FixedWidthParser.Infrastructure /// [EditorBrowsable(EditorBrowsableState.Never)] public sealed class AsyncRecordEnumerator : IAsyncEnumerator + where TModel : allows ref struct where TParser : struct, IRecordLineParser { private readonly TParser _parser; @@ -18,7 +19,6 @@ public sealed class AsyncRecordEnumerator : IAsyncEnumerator - public TModel Current => _current; + /// The record on the line the enumerator sits on, parsed on access. + /// + /// Parsed here rather than by because the enumerator cannot hold a field + /// of a ref struct . await foreach reads it once per line, + /// so each line is still parsed once; reading it twice parses twice. + /// + /// The line does not parse. + public TModel Current => ParseErrors.Parse(_parser, _lines.CurrentLine, _lines.LineNumber, in _context); /// public ValueTask MoveNextAsync() @@ -73,11 +78,10 @@ private LineStatus TryReadFromBuffer() while (true) { var status = _lines.TryGetLine(out var line); - if (status != LineStatus.Line || _parser.TryParse(line, in _context, out _current)) + if (status != LineStatus.Line || ParseErrors.Accepts(_parser, line, _lines.LineNumber, in _context, _onParseError)) { return status; } - ParseErrors.HandleFailure(_parser, line, _lines.LineNumber, in _context, _onParseError); } } diff --git a/src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs b/src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs index 7c603d1..e311615 100644 --- a/src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs +++ b/src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs @@ -9,6 +9,7 @@ namespace FixedWidthParser.Infrastructure /// [EditorBrowsable(EditorBrowsableState.Never)] public sealed class FixedWidthAsyncRecordEnumerable : IAsyncEnumerable + where TModel : allows ref struct where TParser : struct, IRecordLineParser { private readonly TParser _parser; diff --git a/src/FixedWidthParser/Infrastructure/FixedWidthRecordEnumerable.cs b/src/FixedWidthParser/Infrastructure/FixedWidthRecordEnumerable.cs index 26a47a0..9f3f828 100644 --- a/src/FixedWidthParser/Infrastructure/FixedWidthRecordEnumerable.cs +++ b/src/FixedWidthParser/Infrastructure/FixedWidthRecordEnumerable.cs @@ -12,6 +12,7 @@ namespace FixedWidthParser.Infrastructure /// [EditorBrowsable(EditorBrowsableState.Never)] public sealed class FixedWidthRecordEnumerable : IEnumerable + where TModel : allows ref struct where TParser : struct, IRecordLineParser { private readonly TParser _parser; diff --git a/src/FixedWidthParser/Infrastructure/RecordEnumerator.cs b/src/FixedWidthParser/Infrastructure/RecordEnumerator.cs index ee0681f..838f0dc 100644 --- a/src/FixedWidthParser/Infrastructure/RecordEnumerator.cs +++ b/src/FixedWidthParser/Infrastructure/RecordEnumerator.cs @@ -1,6 +1,7 @@ using System.Collections; using System.ComponentModel; using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; using FixedWidthParser.Readers; namespace FixedWidthParser.Infrastructure @@ -11,6 +12,7 @@ namespace FixedWidthParser.Infrastructure /// [EditorBrowsable(EditorBrowsableState.Never)] public struct RecordEnumerator : IEnumerator + where TModel : allows ref struct where TParser : struct, IRecordLineParser { private readonly TParser _parser; @@ -19,7 +21,6 @@ public struct RecordEnumerator : IEnumerator private readonly Stream _stream; private readonly bool _ownsStream; private LineBufferState _lines; - private TModel _current; private bool _disposed; internal RecordEnumerator(TParser parser, Stream stream, bool ownsStream, FixedWidthReaderOptions options) @@ -31,15 +32,21 @@ internal RecordEnumerator(TParser parser, Stream stream, bool ownsStream, FixedW _ownsStream = ownsStream; _lines = default; _lines.Rent(options); - _current = default!; _disposed = false; } - /// - public readonly TModel Current => _current; + /// The record on the line the enumerator sits on, parsed on access. + /// + /// Parsed here rather than by because the enumerator cannot hold a field of a + /// ref struct . foreach reads it once per line, so each + /// line is still parsed once; reading it twice parses twice. + /// + /// The line does not parse. + // AsRef: a parser never keeps a reference to the context, so the record cannot capture this struct. + public readonly TModel Current => ParseErrors.Parse(_parser, _lines.CurrentLine, _lines.LineNumber, in Unsafe.AsRef(in _context)); [ExcludeFromCodeCoverage] - readonly object IEnumerator.Current => _current!; + readonly object IEnumerator.Current => throw new NotSupportedException("Use the generic Current."); /// public bool MoveNext() @@ -50,11 +57,10 @@ public bool MoveNext() var status = _lines.TryGetLine(out var line); if (status == LineStatus.Line) { - if (_parser.TryParse(line, in _context, out _current)) + if (ParseErrors.Accepts(_parser, line, _lines.LineNumber, in _context, _onParseError)) { return true; } - ParseErrors.HandleFailure(_parser, line, _lines.LineNumber, in _context, _onParseError); continue; } if (status == LineStatus.End) diff --git a/src/FixedWidthParser/Infrastructure/RecordLineParsers.cs b/src/FixedWidthParser/Infrastructure/RecordLineParsers.cs index b793856..64bd8c3 100644 --- a/src/FixedWidthParser/Infrastructure/RecordLineParsers.cs +++ b/src/FixedWidthParser/Infrastructure/RecordLineParsers.cs @@ -21,6 +21,7 @@ public interface IRecordLineParser where TModel : allows ref struct /// Layout parse strategy: forwards to a , built from attributes or fluently. [EditorBrowsable(EditorBrowsableState.Never)] public readonly struct LayoutLineParser : IRecordLineParser + where TModel : allows ref struct { private readonly FixedWidthLayout _layout; @@ -38,7 +39,8 @@ public bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out /// Source-generated parse strategy: forwards to the model's static . [EditorBrowsable(EditorBrowsableState.Never)] - public readonly struct GeneratedLineParser : IRecordLineParser where TModel : IFixedWidthModel + public readonly struct GeneratedLineParser : IRecordLineParser + where TModel : IFixedWidthModel, allows ref struct { /// public bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TModel model) diff --git a/src/FixedWidthParser/Readers/LineBufferState.cs b/src/FixedWidthParser/Readers/LineBufferState.cs index 8c8f215..8ae4837 100644 --- a/src/FixedWidthParser/Readers/LineBufferState.cs +++ b/src/FixedWidthParser/Readers/LineBufferState.cs @@ -28,12 +28,17 @@ internal struct LineBufferState internal long LineStart; private long _nextLineStart; private long _compacted; + private int _lineOffset; + private int _lineLength; private int _maxLineLength; private bool _preambleChecked; /// Offset, from the first byte read, of the first byte no line has consumed yet. internal readonly long Consumed => _compacted + Start; + /// The line last returned, valid until the next . + internal readonly ReadOnlySpan CurrentLine => Buffer.AsSpan(_lineOffset, _lineLength); + internal void Rent(FixedWidthReaderOptions options) { Rent(options, options.SkipsByteOrderMark); @@ -139,7 +144,9 @@ private bool TrySliceLine(int rawEnd, int nextStart, out ReadOnlySpan line throw ParseErrors.LineTooLong(LineNumber + 1L, _maxLineLength); } - line = Buffer.AsSpan(Start, contentEnd - Start); + _lineOffset = Start; + _lineLength = contentEnd - Start; + line = Buffer.AsSpan(_lineOffset, _lineLength); LineStart = _nextLineStart; _nextLineStart = _compacted + nextStart; Start = nextStart; diff --git a/src/FixedWidthParser/Readers/ParallelAggregator.cs b/src/FixedWidthParser/Readers/ParallelAggregator.cs index 5293d93..7ae4b48 100644 --- a/src/FixedWidthParser/Readers/ParallelAggregator.cs +++ b/src/FixedWidthParser/Readers/ParallelAggregator.cs @@ -21,6 +21,7 @@ internal static class ParallelAggregator internal static async Task RunAsync( TParser parser, string path, FixedWidthReaderOptions options, CancellationToken ct) + where TModel : allows ref struct where TParser : struct, IRecordLineParser where TAccumulator : IRecordAccumulator, new() { @@ -48,6 +49,7 @@ internal static async Task RunAsync private static void Drain( TParser parser, SafeFileHandle handle, ChunkPlan plan, FixedWidthReaderOptions options, TAccumulator[] results, ExceptionDispatchInfo?[] failures, CancellationToken ct) + where TModel : allows ref struct where TParser : struct, IRecordLineParser where TAccumulator : IRecordAccumulator, new() { @@ -67,6 +69,7 @@ private static void Drain( private static TAccumulator AggregateChunk( TParser parser, SafeFileHandle handle, long start, long end, FixedWidthReaderOptions options, CancellationToken ct) + where TModel : allows ref struct where TParser : struct, IRecordLineParser where TAccumulator : IRecordAccumulator, new() { @@ -108,6 +111,7 @@ private static TAccumulator AggregateChunk( /// Parses one line and folds it in; when it does not parse. internal static bool TryAdd(TParser parser, ReadOnlySpan line, in FixedWidthContext context, ref TAccumulator accumulator) + where TModel : allows ref struct where TParser : struct, IRecordLineParser where TAccumulator : IRecordAccumulator { diff --git a/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs b/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs index 94c4007..d20b11b 100644 --- a/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs +++ b/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs @@ -27,6 +27,7 @@ internal static class ParallelStreamAggregator /// Folds , disposing it afterwards unless . internal static async Task RunAsync( TParser parser, Stream source, bool leaveOpen, FixedWidthReaderOptions options, CancellationToken ct) + where TModel : allows ref struct where TParser : struct, IRecordLineParser where TAccumulator : IRecordAccumulator, new() { @@ -49,6 +50,7 @@ internal static async Task RunAsync private static async Task SequentialAsync( TParser parser, Stream source, FixedWidthReaderOptions options, CancellationToken ct) + where TModel : allows ref struct where TParser : struct, IRecordLineParser where TAccumulator : IRecordAccumulator, new() { @@ -63,6 +65,7 @@ private static async Task SequentialAsync PartitionedAsync( TParser parser, Stream source, FixedWidthReaderOptions options, CancellationToken ct) + where TModel : allows ref struct where TParser : struct, IRecordLineParser where TAccumulator : IRecordAccumulator, new() { @@ -107,6 +110,7 @@ private static async Task PartitionedAsync(IEnumerable> outcomes, ExceptionDispatchInfo? producerFailure) + where TModel : allows ref struct where TAccumulator : IRecordAccumulator, new() { TAccumulator? merged = default; @@ -204,6 +208,7 @@ private static byte[] GrowUnterminated(byte[] buffer, long linesBefore, int maxL private static async Task>> ConsumeAsync( TParser parser, ChannelReader queue, FixedWidthReaderOptions options, CancellationToken ct) + where TModel : allows ref struct where TParser : struct, IRecordLineParser where TAccumulator : IRecordAccumulator, new() { @@ -225,6 +230,7 @@ private static async Task>> ConsumeAsync( TParser parser, Chunk chunk, FixedWidthReaderOptions options, in FixedWidthContext context, CancellationToken ct) + where TModel : allows ref struct where TParser : struct, IRecordLineParser where TAccumulator : IRecordAccumulator, new() { diff --git a/src/FixedWidthParser/Readers/ParseFailure.cs b/src/FixedWidthParser/Readers/ParseFailure.cs index b043376..325a0bd 100644 --- a/src/FixedWidthParser/Readers/ParseFailure.cs +++ b/src/FixedWidthParser/Readers/ParseFailure.cs @@ -30,10 +30,41 @@ internal void SetShortLine(int requiredLength) /// internal static class ParseErrors { + /// + /// Whether an enumerator should yield . Without + /// every line is yielded and a bad one throws when + /// reads it; with it, the line is parsed here so a bad one can be skipped. + /// + internal static bool Accepts( + TParser parser, ReadOnlySpan line, long lineNumber, in FixedWidthContext context, Func? onParseError) + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + { + if (onParseError is null || parser.TryParse(line, in context, out _)) + { + return true; + } + HandleFailure(parser, line, lineNumber, in context, onParseError); + return false; + } + + /// Parses the line an enumerator sits on, throwing as a reader does when it does not parse. + internal static TModel Parse(TParser parser, ReadOnlySpan line, long lineNumber, in FixedWidthContext context) + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + { + if (!parser.TryParse(line, in context, out var model)) + { + HandleFailure(parser, line, lineNumber, in context, onParseError: null); + } + return model; + } + /// Returns when asks to skip the line; throws otherwise. [MethodImpl(MethodImplOptions.NoInlining)] internal static void HandleFailure( TParser parser, ReadOnlySpan line, long lineNumber, in FixedWidthContext context, Func? onParseError) + where TModel : allows ref struct where TParser : struct, IRecordLineParser { var failure = new ParseFailure(); diff --git a/tests/FixedWidthParser.Tests/RefStructReadTests.cs b/tests/FixedWidthParser.Tests/RefStructReadTests.cs new file mode 100644 index 0000000..937a471 --- /dev/null +++ b/tests/FixedWidthParser.Tests/RefStructReadTests.cs @@ -0,0 +1,202 @@ +using System.Text; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// Ref struct models read through every enumerator and aggregate, generated and through a layout alike. + public sealed class RefStructReadTests : IDisposable + { + private const string Content = "Ann 30 1000.00 \nBob 41 2000.50 \r\nCid 52 3000.25 "; + private const string BadContent = "Ann 30 1000.00 \nBob XX 2000.50 \nCid 52 3000.25 "; + + private static readonly FixedWidthLayout Layout = FixedWidthLayout.FromAttributes(); + + private readonly string _path = Path.GetTempFileName(); + + public RefStructReadTests() + { + File.WriteAllText(_path, Content); + } + + public void Dispose() + { + File.Delete(_path); + } + + public sealed class GenAges : IRecordAccumulator + { + public List Values { get; } = []; + + public void Add(in GenRefPersonModel model) + { + Values.Add(model.Age); + } + + public void Merge(GenAges following) + { + Values.AddRange(following.Values); + } + } + + public sealed class Ages : IRecordAccumulator + { + public List Values { get; } = []; + + public void Add(in RefPersonModel model) + { + Values.Add(model.Age); + } + + public void Merge(Ages following) + { + Values.AddRange(following.Values); + } + } + + private static MemoryStream Stream(string content) + { + return new MemoryStream(Encoding.UTF8.GetBytes(content)); + } + + [Fact] + public void Read_Generated_YieldsEveryRecord() + { + var names = new List(); + foreach (var person in FixedWidth.Read(Stream(Content))) + { + names.Add($"{person.Name}:{person.Age}:{person.Salary.ToString(Inv)}"); + } + + Assert.Equal(["Ann:30:1000", "Bob:41:2000.5", "Cid:52:3000.25"], names); + } + + [Fact] + public void ReadFile_Layout_YieldsEveryRecord() + { + var ages = new List(); + foreach (var person in FixedWidth.ReadFile(_path, Layout)) + { + ages.Add(person.Age); + } + + Assert.Equal([30, 41, 52], ages); + } + + [Fact] + public void Read_ThroughIEnumerable_YieldsEveryRecord() + { + IEnumerable records = FixedWidth.Read(Stream(Content)); + var ages = new List(); + using var enumerator = records.GetEnumerator(); + while (enumerator.MoveNext()) + { + ages.Add(enumerator.Current.Age); + } + + Assert.Equal([30, 41, 52], ages); + } + + [Fact] + public void Read_CurrentReadTwice_ReturnsTheSameRecord() + { + using var enumerator = FixedWidth.Read(Stream(Content)).GetEnumerator(); + Assert.True(enumerator.MoveNext()); + + Assert.Equal(enumerator.Current.Name, enumerator.Current.Name); + Assert.Equal(30, enumerator.Current.Age); + } + + [Fact] + public async Task ReadAsync_Generated_YieldsEveryRecord() + { + var ages = new List(); + await foreach (var person in FixedWidth.ReadAsync(Stream(Content))) + { + ages.Add(person.Age); + } + + Assert.Equal([30, 41, 52], ages); + } + + [Fact] + public async Task ReadFileAsync_Layout_YieldsEveryRecord() + { + var ages = new List(); + await foreach (var person in FixedWidth.ReadFileAsync(_path, Layout)) + { + ages.Add(person.Age); + } + + Assert.Equal([30, 41, 52], ages); + } + + [Fact] + public void Read_BadLineWithoutHandler_Throws() + { + var ex = Assert.Throws(() => + { + foreach (var person in FixedWidth.Read(Stream(BadContent))) + { + _ = person.Age; + } + }); + + Assert.Equal(2, ex.LineNumber); + Assert.Equal("Age", ex.ColumnName); + } + + [Fact] + public async Task ReadAsync_BadLineSkipped_YieldsTheRest() + { + var options = new FixedWidthReaderOptions { OnParseError = _ => true }; + var ages = new List(); + await foreach (var person in FixedWidth.ReadAsync(Stream(BadContent), Layout, options: options)) + { + ages.Add(person.Age); + } + + Assert.Equal([30, 52], ages); + } + + [Fact] + public void Read_BadLineSkipped_YieldsTheRest() + { + var options = new FixedWidthReaderOptions { OnParseError = _ => true }; + var ages = new List(); + foreach (var person in FixedWidth.Read(Stream(BadContent), options: options)) + { + ages.Add(person.Age); + } + + Assert.Equal([30, 52], ages); + } + + [Theory] + [InlineData(1)] + [InlineData(4)] + public async Task AggregateFileAsync_FoldsEveryRecord(int degreeOfParallelism) + { + var options = new FixedWidthReaderOptions { DegreeOfParallelism = degreeOfParallelism, ChunkSize = 7 }; + + var generated = await FixedWidth.AggregateFileAsync(_path, options); + var layout = await FixedWidth.AggregateFileAsync(_path, Layout, options); + + Assert.Equal([30, 41, 52], generated.Values); + Assert.Equal([30, 41, 52], layout.Values); + } + + [Theory] + [InlineData(1)] + [InlineData(4)] + public async Task AggregateAsync_FoldsEveryRecord(int degreeOfParallelism) + { + var options = new FixedWidthReaderOptions { DegreeOfParallelism = degreeOfParallelism, ChunkSize = 7 }; + + var generated = await FixedWidth.AggregateAsync(Stream(Content), options: options); + var layout = await FixedWidth.AggregateAsync(Stream(Content), Layout, options: options); + + Assert.Equal([30, 41, 52], generated.Values); + Assert.Equal([30, 41, 52], layout.Values); + } + } +} From 6a1504b26eb0b4a02559a101ccf95510c0680d2e Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 00:23:52 -0300 Subject: [PATCH 33/85] refactor: run parallel aggregation over an internal line sink and source ParallelAggregator now reads through IRandomAccessSource (a file handle today, native memory next) from a data start offset, and can hand lines to an internal ILineSink instead of a model accumulator. The native library uses both; the public API does not change. The model path keeps its own loop: routing it through a generic sink made one-call reads ~40% slower under dynamic PGO. Co-Authored-By: Claude Opus 5.5 --- src/FixedWidthParser/FixedWidthParser.csproj | 1 + src/FixedWidthParser/Readers/LineSinks.cs | 18 +++ .../Readers/ParallelAggregator.cs | 117 ++++++++++++++---- src/FixedWidthParser/Readers/ParseFailure.cs | 21 +++- .../Readers/RandomAccessSources.cs | 23 ++++ tests/FixedWidthParser.Tests/LineSinkTests.cs | 101 +++++++++++++++ 6 files changed, 256 insertions(+), 25 deletions(-) create mode 100644 src/FixedWidthParser/Readers/LineSinks.cs create mode 100644 src/FixedWidthParser/Readers/RandomAccessSources.cs create mode 100644 tests/FixedWidthParser.Tests/LineSinkTests.cs diff --git a/src/FixedWidthParser/FixedWidthParser.csproj b/src/FixedWidthParser/FixedWidthParser.csproj index 870a7cc..d4fae77 100644 --- a/src/FixedWidthParser/FixedWidthParser.csproj +++ b/src/FixedWidthParser/FixedWidthParser.csproj @@ -63,6 +63,7 @@ + diff --git a/src/FixedWidthParser/Readers/LineSinks.cs b/src/FixedWidthParser/Readers/LineSinks.cs new file mode 100644 index 0000000..d7b3b82 --- /dev/null +++ b/src/FixedWidthParser/Readers/LineSinks.cs @@ -0,0 +1,18 @@ +namespace FixedWidthParser.Readers +{ + /// + /// Where the parallel reader puts each line of a partition. A , so the reader is + /// specialized per sink and the per-line call devirtualizes. + /// + internal interface ILineSink where TSelf : struct, ILineSink + { + /// Takes one line; when it does not parse, leaving the sink as it was. + bool TryAdd(ReadOnlySpan line, in FixedWidthContext context); + + /// The exception for a line rejected. + FixedWidthParseException Diagnose(ReadOnlySpan line, long lineNumber, in FixedWidthContext context); + + /// Folds the sink of the partition that follows this one into it. + void Merge(TSelf following); + } +} diff --git a/src/FixedWidthParser/Readers/ParallelAggregator.cs b/src/FixedWidthParser/Readers/ParallelAggregator.cs index 7ae4b48..f3d305f 100644 --- a/src/FixedWidthParser/Readers/ParallelAggregator.cs +++ b/src/FixedWidthParser/Readers/ParallelAggregator.cs @@ -1,7 +1,6 @@ using System.Buffers; using System.Runtime.ExceptionServices; using FixedWidthParser.Infrastructure; -using Microsoft.Win32.SafeHandles; namespace FixedWidthParser.Readers { @@ -26,13 +25,44 @@ internal static async Task RunAsync where TAccumulator : IRecordAccumulator, new() { using var handle = File.OpenHandle(path, FileMode.Open, FileAccess.Read, FileShare.Read); - var plan = new ChunkPlan(RandomAccess.GetLength(handle), options); - var results = new TAccumulator[plan.Count]; + var source = new FileSource(handle); + return await RunPartitionsAsync(source, 0, options, + (plan, index) => AggregateChunk(parser, source, plan, index, options, ct), + static (merged, following) => + { + merged.Merge(following); + return merged; + }, ct).ConfigureAwait(false); + } + + /// + /// Reads [dataStart, source.Length) in partitions, one each, merged in + /// order. must start a line; line numbers count from the start of the source. + /// + internal static Task RunAsync( + IRandomAccessSource source, long dataStart, Func create, FixedWidthReaderOptions options, CancellationToken ct) + where TSink : struct, ILineSink + { + return RunPartitionsAsync(source, dataStart, options, + (plan, index) => SinkChunk(source, plan, index, create(), options, ct), + static (merged, following) => + { + merged.Merge(following); + return merged; + }, ct); + } + + private static async Task RunPartitionsAsync( + IRandomAccessSource source, long dataStart, FixedWidthReaderOptions options, + Func aggregate, Func merge, CancellationToken ct) + { + var plan = new ChunkPlan(dataStart, source.Length, options); + var results = new TResult[plan.Count]; var failures = new ExceptionDispatchInfo?[plan.Count]; var workers = new Task[Math.Min(options.Workers, plan.Count)]; foreach (ref var worker in workers.AsSpan()) { - worker = Task.Run(() => Drain(parser, handle, plan, options, results, failures, ct), ct); + worker = Task.Run(() => Drain(plan, aggregate, results, failures, ct), ct); } await Task.WhenAll(workers).ConfigureAwait(false); @@ -41,24 +71,20 @@ internal static async Task RunAsync for (int i = 1; i < plan.Count; i++) { failures[i]?.Throw(); - merged.Merge(results[i]); + merged = merge(merged, results[i]); } return merged; } - private static void Drain( - TParser parser, SafeFileHandle handle, ChunkPlan plan, FixedWidthReaderOptions options, - TAccumulator[] results, ExceptionDispatchInfo?[] failures, CancellationToken ct) - where TModel : allows ref struct - where TParser : struct, IRecordLineParser - where TAccumulator : IRecordAccumulator, new() + private static void Drain( + ChunkPlan plan, Func aggregate, TResult[] results, ExceptionDispatchInfo?[] failures, CancellationToken ct) { while (plan.TryTakeNext(out int index)) { ct.ThrowIfCancellationRequested(); try { - results[index] = AggregateChunk(parser, handle, plan.Start(index), plan.End(index), options, ct); + results[index] = aggregate(plan, index); } catch (Exception ex) when (ex is not OperationCanceledException) { @@ -67,15 +93,20 @@ private static void Drain( } } + // ponytail: the model and sink loops are twins on purpose. Folding the model path into a ModelSink + // made one-call reads ~40% slower: dynamic PGO stopped devirtualizing IRecordAccumulator.Add through + // the shared generic sink. Keep them in step. private static TAccumulator AggregateChunk( - TParser parser, SafeFileHandle handle, long start, long end, FixedWidthReaderOptions options, CancellationToken ct) + TParser parser, IRandomAccessSource source, ChunkPlan plan, int index, FixedWidthReaderOptions options, CancellationToken ct) where TModel : allows ref struct where TParser : struct, IRecordLineParser where TAccumulator : IRecordAccumulator, new() { + long start = plan.Start(index); + long end = plan.End(index); var accumulator = new TAccumulator(); var context = options.Context; - var reader = new ChunkReader(handle, start == 0 ? 0 : start - 1, options.MaxLineLength); + var reader = new ChunkReader(source, index == 0 ? start : start - 1, options.MaxLineLength); var lines = default(LineBufferState); lines.Rent(options, start == 0 && options.SkipsByteOrderMark); long delivered = 0; @@ -109,6 +140,47 @@ private static TAccumulator AggregateChunk( return accumulator; } + private static TSink SinkChunk( + IRandomAccessSource source, ChunkPlan plan, int index, TSink sink, FixedWidthReaderOptions options, CancellationToken ct) + where TSink : struct, ILineSink + { + long start = plan.Start(index); + long end = plan.End(index); + var context = options.Context; + var reader = new ChunkReader(source, index == 0 ? start : start - 1, options.MaxLineLength); + var lines = default(LineBufferState); + lines.Rent(options, start == 0 && options.SkipsByteOrderMark); + long delivered = 0; + try + { + while (reader.Origin + lines.Consumed < end) + { + if (reader.NextLine(ref lines, out var line) == LineStatus.End) + { + break; + } + long lineStart = reader.Origin + lines.LineStart; + if (lineStart < start || lineStart >= end) + { + continue; + } + if (++delivered % LinesPerCancellationCheck == 0) + { + ct.ThrowIfCancellationRequested(); + } + if (!sink.TryAdd(line, in context)) + { + ParseErrors.Reject(sink.Diagnose(line, reader.LineNumberAt(lineStart), in context), options.OnParseError); + } + } + } + finally + { + lines.Return(); + } + return sink; + } + /// Parses one line and folds it in; when it does not parse. internal static bool TryAdd(TParser parser, ReadOnlySpan line, in FixedWidthContext context, ref TAccumulator accumulator) where TModel : allows ref struct @@ -125,22 +197,25 @@ internal static bool TryAdd(TParser parser, ReadO private sealed class ChunkPlan { + private readonly long _dataStart; private readonly long _length; private readonly long _chunkSize; private int _next = -1; - internal ChunkPlan(long length, FixedWidthReaderOptions options) + internal ChunkPlan(long dataStart, long length, FixedWidthReaderOptions options) { + _dataStart = dataStart; _length = length; - _chunkSize = ChunkSize(length, options); - Count = (int)Math.Max(1, (length + _chunkSize - 1) / _chunkSize); + long data = Math.Max(0, length - dataStart); + _chunkSize = ChunkSize(data, options); + Count = (int)Math.Max(1, (data + _chunkSize - 1) / _chunkSize); } internal int Count { get; } internal long Start(int index) { - return index * _chunkSize; + return _dataStart + (index * _chunkSize); } internal long End(int index) @@ -169,7 +244,7 @@ private static long ChunkSize(long length, FixedWidthReaderOptions options) } /// Feeds one partition's bytes to a and numbers its lines on demand. - private sealed class ChunkReader(SafeFileHandle handle, long origin, int maxLineLength) + private sealed class ChunkReader(IRandomAccessSource source, long origin, int maxLineLength) { private long _readOffset = origin; private long _countedTo; @@ -190,7 +265,7 @@ internal LineStatus NextLine(ref LineBufferState lines, out ReadOnlySpan l } lines.Compact(); lines.GrowIfFull(); - int read = RandomAccess.Read(handle, lines.Buffer.AsSpan(lines.End), _readOffset); + int read = source.Read(lines.Buffer.AsSpan(lines.End), _readOffset); _readOffset += read; lines.Advance(read); } @@ -212,7 +287,7 @@ internal long LineNumberAt(long offset) { while (_countedTo < offset) { - int read = RandomAccess.Read(handle, buffer.AsSpan(0, (int)Math.Min(buffer.Length, offset - _countedTo)), _countedTo); + int read = source.Read(buffer.AsSpan(0, (int)Math.Min(buffer.Length, offset - _countedTo)), _countedTo); if (read == 0) { break; diff --git a/src/FixedWidthParser/Readers/ParseFailure.cs b/src/FixedWidthParser/Readers/ParseFailure.cs index 325a0bd..0683343 100644 --- a/src/FixedWidthParser/Readers/ParseFailure.cs +++ b/src/FixedWidthParser/Readers/ParseFailure.cs @@ -67,17 +67,30 @@ internal static void HandleFailure( where TModel : allows ref struct where TParser : struct, IRecordLineParser { - var failure = new ParseFailure(); - var diagnosis = context with { Failure = failure }; - _ = parser.TryParse(line, in diagnosis, out _); + Reject(Diagnose(parser, line, lineNumber, in context), onParseError); + } - var exception = Create(line, lineNumber, typeof(TModel).Name, failure, in context); + /// Returns when asks to skip the line describes; throws it otherwise. + internal static void Reject(FixedWidthParseException exception, Func? onParseError) + { if (onParseError is null || !onParseError(exception)) { throw exception; } } + /// Parses a rejected line again with a failure sink and builds the exception that names its column. + [MethodImpl(MethodImplOptions.NoInlining)] + internal static FixedWidthParseException Diagnose(TParser parser, ReadOnlySpan line, long lineNumber, in FixedWidthContext context) + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + { + var failure = new ParseFailure(); + var diagnosis = context with { Failure = failure }; + _ = parser.TryParse(line, in diagnosis, out _); + return Create(line, lineNumber, typeof(TModel).Name, failure, in context); + } + [MethodImpl(MethodImplOptions.NoInlining)] internal static InvalidDataException LineTooLong(long lineNumber, int maxLineLength) { diff --git a/src/FixedWidthParser/Readers/RandomAccessSources.cs b/src/FixedWidthParser/Readers/RandomAccessSources.cs new file mode 100644 index 0000000..b7ab714 --- /dev/null +++ b/src/FixedWidthParser/Readers/RandomAccessSources.cs @@ -0,0 +1,23 @@ +using Microsoft.Win32.SafeHandles; + +namespace FixedWidthParser.Readers +{ + /// Bytes read at an offset, from any thread: what the parallel reader partitions. + internal interface IRandomAccessSource + { + long Length { get; } + + /// Copies the bytes at into ; 0 at the end. + int Read(Span destination, long offset); + } + + internal sealed class FileSource(SafeFileHandle handle) : IRandomAccessSource + { + public long Length { get; } = RandomAccess.GetLength(handle); + + public int Read(Span destination, long offset) + { + return RandomAccess.Read(handle, destination, offset); + } + } +} diff --git a/tests/FixedWidthParser.Tests/LineSinkTests.cs b/tests/FixedWidthParser.Tests/LineSinkTests.cs new file mode 100644 index 0000000..fdd5d9b --- /dev/null +++ b/tests/FixedWidthParser.Tests/LineSinkTests.cs @@ -0,0 +1,101 @@ +using System.Text; +using FixedWidthParser.Readers; + +namespace FixedWidthParser.Tests +{ + /// The internal parallel read over a line sink and a random-access source, used by the native layer. + public sealed class LineSinkTests : IDisposable + { + private readonly string _path = Path.GetTempFileName(); + + public void Dispose() + { + File.Delete(_path); + } + + private struct Lines : ILineSink + { + private List? _lines; + + internal readonly List Values => _lines ?? []; + + public bool TryAdd(ReadOnlySpan line, in FixedWidthContext context) + { + if (line.SequenceEqual("XX"u8)) + { + return false; + } + (_lines ??= []).Add(Encoding.UTF8.GetString(line)); + return true; + } + + public readonly FixedWidthParseException Diagnose(ReadOnlySpan line, long lineNumber, in FixedWidthContext context) + { + return new FixedWidthParseException("rejected", lineNumber, columnName: null, columnStart: -1, columnLength: 0, rawValue: null); + } + + public void Merge(Lines following) + { + (_lines ??= []).AddRange(following.Values); + } + } + + private sealed class ArraySource(byte[] bytes) : IRandomAccessSource + { + public long Length => bytes.Length; + + public int Read(Span destination, long offset) + { + var rest = bytes.AsSpan((int)Math.Min(offset, bytes.Length)); + int count = Math.Min(rest.Length, destination.Length); + rest[..count].CopyTo(destination); + return count; + } + } + + private static FixedWidthReaderOptions Options(int chunkSize) + { + return FixedWidthReaderOptions.Validate(new FixedWidthReaderOptions { DegreeOfParallelism = 4, ChunkSize = chunkSize }); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(64)] + public async Task RunAsync_FromDataStart_SkipsEarlierBytes(int chunkSize) + { + var source = new ArraySource("HDR\nA1\nA2\n"u8.ToArray()); + + var lines = await ParallelAggregator.RunAsync(source, dataStart: 4, () => new Lines(), Options(chunkSize), CancellationToken.None); + + Assert.Equal(["A1", "A2"], lines.Values); + } + + [Theory] + [InlineData(1)] + [InlineData(64)] + public async Task RunAsync_ErrorLineNumber_CountsFromSourceStart(int chunkSize) + { + var source = new ArraySource("HDR\nA1\nXX\nA3\n"u8.ToArray()); + + var ex = await Assert.ThrowsAsync(() => + ParallelAggregator.RunAsync(source, dataStart: 4, () => new Lines(), Options(chunkSize), CancellationToken.None)); + + Assert.Equal(3, ex.LineNumber); + } + + [Fact] + public async Task RunAsync_MemorySource_EqualsFileSource() + { + byte[] content = "A1\r\nA2\n\nA3"u8.ToArray(); + await File.WriteAllBytesAsync(_path, content); + using var handle = File.OpenHandle(_path); + + var fromFile = await ParallelAggregator.RunAsync(new FileSource(handle), 0, () => new Lines(), Options(3), CancellationToken.None); + var fromMemory = await ParallelAggregator.RunAsync(new ArraySource(content), 0, () => new Lines(), Options(3), CancellationToken.None); + + Assert.Equal(["A1", "A2", "A3"], fromFile.Values); + Assert.Equal(fromFile.Values, fromMemory.Values); + } + } +} From b291855bf0bd0492299878f7d591a343abb5237f Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 00:28:55 -0300 Subject: [PATCH 34/85] feat(native): scaffold the NativeAOT library FixedWidthParser.Native publishes as a C-ABI shared library with the same settings as ExcelReader.Native: invariant globalization, an AVX2 floor on x64, a handle table and a per-thread last error. It exports fw_abi_version, fw_last_error and fw_last_parse_error so far. Co-Authored-By: Claude Opus 5.5 --- FixedWidthParser.slnx | 1 + src/FixedWidthParser.Native/Exports.cs | 49 ++++++++ .../FixedWidthParser.Native.csproj | 50 ++++++++ .../GrowableNativeBuffer.cs | 107 ++++++++++++++++++ src/FixedWidthParser.Native/NativeApi.cs | 65 +++++++++++ .../NativeHandleTable.cs | 40 +++++++ src/FixedWidthParser.Native/NativeStatus.cs | 15 +++ src/FixedWidthParser.Native/README.md | 27 +++++ .../FixedWidthParser.Tests.csproj | 3 + .../Native/NativeApiTests.cs | 85 ++++++++++++++ 10 files changed, 442 insertions(+) create mode 100644 src/FixedWidthParser.Native/Exports.cs create mode 100644 src/FixedWidthParser.Native/FixedWidthParser.Native.csproj create mode 100644 src/FixedWidthParser.Native/GrowableNativeBuffer.cs create mode 100644 src/FixedWidthParser.Native/NativeApi.cs create mode 100644 src/FixedWidthParser.Native/NativeHandleTable.cs create mode 100644 src/FixedWidthParser.Native/NativeStatus.cs create mode 100644 src/FixedWidthParser.Native/README.md create mode 100644 tests/FixedWidthParser.Tests/Native/NativeApiTests.cs diff --git a/FixedWidthParser.slnx b/FixedWidthParser.slnx index ed74dba..b510740 100644 --- a/FixedWidthParser.slnx +++ b/FixedWidthParser.slnx @@ -2,6 +2,7 @@ + diff --git a/src/FixedWidthParser.Native/Exports.cs b/src/FixedWidthParser.Native/Exports.cs new file mode 100644 index 0000000..7fa8b76 --- /dev/null +++ b/src/FixedWidthParser.Native/Exports.cs @@ -0,0 +1,49 @@ +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +namespace FixedWidthParser.Native +{ + /// + /// The [UnmanagedCallersOnly] entry points: pointer checks and marshaling only, so the logic + /// stays in the span-based layer the tests drive. One partial per feature. + /// + [ExcludeFromCodeCoverage] + internal static unsafe partial class Exports + { + [UnmanagedCallersOnly(EntryPoint = "fw_abi_version")] + public static int AbiVersion() + { + return NativeStatus.AbiVersion; + } + + [UnmanagedCallersOnly(EntryPoint = "fw_last_error")] + public static int LastError(byte* buffer, int capacity, int* outLength) + { + if (outLength is null || capacity < 0 || (buffer is null && capacity > 0)) + { + return NativeStatus.InvalidArgument; + } + return NativeApi.LastError(new Span(buffer, capacity), out *outLength); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_last_parse_error")] + public static int LastParseError(NativeParseError* outError) + { + if (outError is null || outError->StructSize != sizeof(NativeParseError)) + { + return NativeStatus.InvalidArgument; + } + NativeApi.LastParseError(out outError->LineNumber, out outError->Column); + return NativeStatus.Ok; + } + } + + /// fw_parse_error. + [StructLayout(LayoutKind.Sequential)] + internal struct NativeParseError + { + public int StructSize; + public long LineNumber; + public int Column; + } +} diff --git a/src/FixedWidthParser.Native/FixedWidthParser.Native.csproj b/src/FixedWidthParser.Native/FixedWidthParser.Native.csproj new file mode 100644 index 0000000..d0dfde9 --- /dev/null +++ b/src/FixedWidthParser.Native/FixedWidthParser.Native.csproj @@ -0,0 +1,50 @@ + + + + net10.0 + enable + enable + true + false + true + true + + true + Shared + false + + true + Speed + false + false + true + false + + x86-64-v3 + + + + $(NoWarn);RS0016;RS0026;RS0037;CA1031;S6640 + + + + false + + + + + + + + + + + + diff --git a/src/FixedWidthParser.Native/GrowableNativeBuffer.cs b/src/FixedWidthParser.Native/GrowableNativeBuffer.cs new file mode 100644 index 0000000..858d5cd --- /dev/null +++ b/src/FixedWidthParser.Native/GrowableNativeBuffer.cs @@ -0,0 +1,107 @@ +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace FixedWidthParser.Native +{ + /// + /// One native block that doubles as it fills, so a finished column can be handed over as-is instead + /// of copied. A owns the block, so a buffer dropped on an error path still + /// frees it. + /// + internal sealed unsafe class GrowableNativeBuffer : IDisposable where T : unmanaged + { + private const int InitialLength = 256; +#pragma warning disable SS066 // Disposed in Dispose, or detached by Detach. + private Block? _block; +#pragma warning restore SS066 + private T* _data; + private int _capacity; + + internal int Count { get; private set; } + internal int ByteLength => checked(Count * sizeof(T)); + + internal ref T Last => ref _data[Count - 1]; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void Add(T value) + { + if (Count == _capacity) + { + Grow(Count + 1); + } + _data[Count++] = value; + } + + internal void AddRange(ReadOnlySpan values) + { + if (values.Length > _capacity - Count) + { + Grow(checked(Count + values.Length)); + } + values.CopyTo(new Span(_data + Count, values.Length)); + Count += values.Length; + } + + internal void CopyTo(Span destination) + { + new ReadOnlySpan(_data, ByteLength).CopyTo(destination); + } + + /// Hands the block to the caller, who frees it with . + internal IntPtr Detach() + { + IntPtr block = (IntPtr)_data; + _block?.SetHandleAsInvalid(); + Reset(); + return block; + } + + public void Dispose() + { + _block?.Dispose(); + Reset(); + } + + private void Reset() + { +#pragma warning disable IDISP003 // Callers dispose or detach the block first. + _block = null; +#pragma warning restore IDISP003 + _data = null; + _capacity = 0; + Count = 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void Grow(int minimum) + { + int capacity = Math.Max(minimum, Math.Max(_capacity * 2, InitialLength)); + nint bytes = checked((nint)capacity * sizeof(T)); + _block ??= new Block(); + _data = (T*)_block.Resize(bytes); + _capacity = capacity; + } + + private sealed class Block : SafeHandle + { + internal Block() + : base(IntPtr.Zero, ownsHandle: true) + { + } + + public override bool IsInvalid => handle == IntPtr.Zero; + + internal IntPtr Resize(nint bytes) + { + SetHandle(handle == IntPtr.Zero ? Marshal.AllocHGlobal(bytes) : Marshal.ReAllocHGlobal(handle, bytes)); + return handle; + } + + protected override bool ReleaseHandle() + { + Marshal.FreeHGlobal(handle); + return true; + } + } + } +} diff --git a/src/FixedWidthParser.Native/NativeApi.cs b/src/FixedWidthParser.Native/NativeApi.cs new file mode 100644 index 0000000..560c0dc --- /dev/null +++ b/src/FixedWidthParser.Native/NativeApi.cs @@ -0,0 +1,65 @@ +using System.Runtime.CompilerServices; +using System.Text; + +namespace FixedWidthParser.Native +{ + /// The calling thread's last failure, read back through fw_last_error and fw_last_parse_error. + internal static class NativeApi + { + [ThreadStatic] + private static string? _lastError; + + [ThreadStatic] + private static long _parseLine; + + [ThreadStatic] + private static int _parseColumn; + +#pragma warning disable CA2255 // A native library has no Main; this is its only start-up hook. + [ModuleInitializer] +#pragma warning restore CA2255 + internal static void RegisterCodePages() + { + Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + } + + internal static void SetLastError(string message) + { + _lastError = message; + _parseLine = 0; + _parseColumn = -1; + } + + /// Records a parse failure; is the layout index, -1 for a short line. + internal static void SetParseError(FixedWidthParseException exception, int column) + { + _lastError = exception.Message; + _parseLine = exception.LineNumber; + _parseColumn = column; + } + + internal static void LastParseError(out long line, out int column) + { + line = _parseLine; + column = _lastError is null ? -1 : _parseColumn; + } + + internal static int LastError(Span buffer, out int length) + { + string? message = _lastError; + if (string.IsNullOrEmpty(message)) + { + length = 0; + return NativeStatus.Ok; + } + + length = Encoding.UTF8.GetByteCount(message); + if (buffer.Length < length) + { + return NativeStatus.BufferTooSmall; + } + Encoding.UTF8.GetBytes(message, buffer); + return NativeStatus.Ok; + } + } +} diff --git a/src/FixedWidthParser.Native/NativeHandleTable.cs b/src/FixedWidthParser.Native/NativeHandleTable.cs new file mode 100644 index 0000000..86457fe --- /dev/null +++ b/src/FixedWidthParser.Native/NativeHandleTable.cs @@ -0,0 +1,40 @@ +using System.Collections.Concurrent; + +namespace FixedWidthParser.Native +{ + internal static class NativeHandleTable + { + private static readonly ConcurrentDictionary _live = new(); + private static long _counter; + + internal static nint Register(object handle) + { + while (true) + { + nint id = (nint)Interlocked.Increment(ref _counter); + if (id != 0 && _live.TryAdd(id, handle)) + { + return id; + } + } + } + + internal static T? Resolve(nint id) where T : class + { + return id != 0 && _live.TryGetValue(id, out object? handle) ? handle as T : null; + } + + internal static bool TryUnregister(nint id, out T? handle) where T : class + { + if (id != 0 && _live.TryGetValue(id, out object? existing) && existing is T typed + && _live.TryRemove(new KeyValuePair(id, existing))) + { + handle = typed; + return true; + } + + handle = null; + return false; + } + } +} diff --git a/src/FixedWidthParser.Native/NativeStatus.cs b/src/FixedWidthParser.Native/NativeStatus.cs new file mode 100644 index 0000000..b948032 --- /dev/null +++ b/src/FixedWidthParser.Native/NativeStatus.cs @@ -0,0 +1,15 @@ +namespace FixedWidthParser.Native +{ + internal static class NativeStatus + { + internal const int Ok = 0; + internal const int Eof = -1; + internal const int BufferTooSmall = -2; + internal const int InvalidHandle = -3; + internal const int InvalidArgument = -4; + internal const int Error = -5; + internal const int ParseError = -6; + + internal const int AbiVersion = 1; + } +} diff --git a/src/FixedWidthParser.Native/README.md b/src/FixedWidthParser.Native/README.md new file mode 100644 index 0000000..b6fafd6 --- /dev/null +++ b/src/FixedWidthParser.Native/README.md @@ -0,0 +1,27 @@ +# FixedWidthParser.Native + +NativeAOT shared library exposing FixedWidthParser over a C ABI, so Python, C and C++ can read and write +fixed-width files without a .NET runtime. + +- ABI reference and full contract: `include/fixedwidth.h` +- Python binding: `python/`; C++ wrapper: `include/fixedwidth.hpp` and `cpp/` + +## Build + + dotnet publish src/FixedWidthParser.Native/FixedWidthParser.Native.csproj -c Release -r + +Output lands in `bin/Release/net10.0//publish/FixedWidthParser.Native.{dll,so,dylib}`. On x64 the +library needs AVX2 (`x86-64-v3`). + +## Layout + +| File | Role | +|---|---| +| `*Api.cs`, `Layouts/`, `Reading/`, `Writing/`, `Arrow/` | Span-based implementation. This is what the tests drive. | +| `Exports*.cs` | `[UnmanagedCallersOnly]` pointer wrappers, one partial per feature. Keep logic out of here: it cannot be called from managed code. | +| `NativeHandleTable.cs` | Maps the opaque handles callers see onto layouts and readers. A handle resolves only as its own kind and is never reissued after it is freed. | +| `NativeApi.cs` | The calling thread's last error (`fw_last_error`, `fw_last_parse_error`). | +| `include/fixedwidth.h` | Hand-written C header; keep it in sync with `Exports*.cs`. | + +Every export catches everything and returns a status (`FW_OK` is 0, failures are negative). Every options +struct starts with `struct_size`; `NULL` means all defaults. diff --git a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj index e77ad86..5b3fa45 100644 --- a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj +++ b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj @@ -8,6 +8,7 @@ enable enable false + true Exe true @@ -19,6 +20,7 @@ + @@ -35,6 +37,7 @@ + diff --git a/tests/FixedWidthParser.Tests/Native/NativeApiTests.cs b/tests/FixedWidthParser.Tests/Native/NativeApiTests.cs new file mode 100644 index 0000000..7313238 --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeApiTests.cs @@ -0,0 +1,85 @@ +using System.Text; +using FixedWidthParser.Native; + +namespace FixedWidthParser.Tests.Native +{ + public sealed class NativeApiTests + { + private static string LastError() + { + Span buffer = stackalloc byte[512]; + Assert.Equal(NativeStatus.Ok, NativeApi.LastError(buffer, out int length)); + return Encoding.UTF8.GetString(buffer[..length]); + } + + [Fact] + public void LastError_IsPerThread() + { + NativeApi.SetLastError("on this thread"); + + string? other = null; + var thread = new Thread(() => other = LastError()); + thread.Start(); + thread.Join(); + + Assert.Equal("", other); + Assert.Equal("on this thread", LastError()); + } + + [Fact] + public void LastError_BufferTooSmall_ReportsLength() + { + NativeApi.SetLastError("twelve bytes"); + + Assert.Equal(NativeStatus.BufferTooSmall, NativeApi.LastError(new byte[4], out int length)); + Assert.Equal(12, length); + } + + [Fact] + public void ParseError_CarriesLineAndColumn() + { + var exception = new FixedWidthParseException("bad", 7, "Age", 10, 5, "XX"); + + NativeApi.SetParseError(exception, column: 2); + + NativeApi.LastParseError(out long line, out int column); + Assert.Equal(7, line); + Assert.Equal(2, column); + Assert.Equal("bad", LastError()); + } + + [Fact] + public void SetLastError_ClearsTheParseError() + { + NativeApi.SetParseError(new FixedWidthParseException("bad", 7, null, -1, 0, null), column: -1); + + NativeApi.SetLastError("other"); + + NativeApi.LastParseError(out long line, out int column); + Assert.Equal(0, line); + Assert.Equal(-1, column); + } + + [Fact] + public void HandleTable_IdIsNeverReissued() + { + nint first = NativeHandleTable.Register(new object()); + Assert.True(NativeHandleTable.TryUnregister(first, out _)); + + nint second = NativeHandleTable.Register(new object()); + + Assert.NotEqual(first, second); + Assert.Null(NativeHandleTable.Resolve(first)); + } + + [Fact] + public void HandleTable_ResolvesOnlyItsOwnKind() + { + nint id = NativeHandleTable.Register("a string"); + + Assert.Null(NativeHandleTable.Resolve>(id)); + Assert.Equal("a string", NativeHandleTable.Resolve(id)); + Assert.True(NativeHandleTable.TryUnregister(id, out _)); + } + } +} From 7b8d214c448e7de974a1c77976ba955e21ac0a6a Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 00:34:22 -0300 Subject: [PATCH 35/85] feat(native): build layouts from C column specs fw_layout_create validates fw_column_spec arrays into an immutable layout handle shared by every read and write: positions through the core's ColumnLayoutValidator, types, scale, formats, ASCII trim and padding, overflow, and the struct_size contract. fw_layout_free releases it. --- src/FixedWidthParser.Native/Exports.Layout.cs | 48 ++++++ .../Layouts/NativeColumnRules.cs | 84 ++++++++++ .../Layouts/NativeColumnSpec.cs | 48 ++++++ .../Layouts/NativeLayout.cs | 122 ++++++++++++++ src/FixedWidthParser.Native/NativeOptions.cs | 33 ++++ .../Native/NativeLayoutTests.cs | 153 ++++++++++++++++++ .../Native/NativeTestLayouts.cs | 112 +++++++++++++ 7 files changed, 600 insertions(+) create mode 100644 src/FixedWidthParser.Native/Exports.Layout.cs create mode 100644 src/FixedWidthParser.Native/Layouts/NativeColumnRules.cs create mode 100644 src/FixedWidthParser.Native/Layouts/NativeColumnSpec.cs create mode 100644 src/FixedWidthParser.Native/Layouts/NativeLayout.cs create mode 100644 src/FixedWidthParser.Native/NativeOptions.cs create mode 100644 tests/FixedWidthParser.Tests/Native/NativeLayoutTests.cs create mode 100644 tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs diff --git a/src/FixedWidthParser.Native/Exports.Layout.cs b/src/FixedWidthParser.Native/Exports.Layout.cs new file mode 100644 index 0000000..be29098 --- /dev/null +++ b/src/FixedWidthParser.Native/Exports.Layout.cs @@ -0,0 +1,48 @@ +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Layouts; + +namespace FixedWidthParser.Native +{ + internal static unsafe partial class Exports + { + [UnmanagedCallersOnly(EntryPoint = "fw_layout_create")] + public static int LayoutCreate(NativeColumnSpecRaw* columns, int columnCount, NativeLayoutOptionsRaw* options, nint* outLayout) + { + if (outLayout is null || columns is null || columnCount <= 0) + { + return InvalidArgument("fw_layout_create needs columns, a positive column_count and out_layout."); + } + *outLayout = 0; + if (columns->StructSize != sizeof(NativeColumnSpecRaw)) + { + return InvalidArgument($"fw_column_spec struct_size is {columns->StructSize}; expected {sizeof(NativeColumnSpecRaw)}."); + } + try + { + int status = NativeLayout.TryCreate(new ReadOnlySpan(columns, columnCount), options, out var layout); + if (status == NativeStatus.Ok) + { + *outLayout = NativeHandleTable.Register(layout!); + } + return status; + } + catch (Exception ex) + { + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + + [UnmanagedCallersOnly(EntryPoint = "fw_layout_free")] + public static void LayoutFree(nint layout) + { + NativeHandleTable.TryUnregister(layout, out _); + } + + private static int InvalidArgument(string message) + { + NativeApi.SetLastError(message); + return NativeStatus.InvalidArgument; + } + } +} diff --git a/src/FixedWidthParser.Native/Layouts/NativeColumnRules.cs b/src/FixedWidthParser.Native/Layouts/NativeColumnRules.cs new file mode 100644 index 0000000..338721d --- /dev/null +++ b/src/FixedWidthParser.Native/Layouts/NativeColumnRules.cs @@ -0,0 +1,84 @@ +using FixedWidthParser.Infrastructure; + +namespace FixedWidthParser.Native.Layouts +{ + /// The per-column rules of fw_layout_create, and the defaults the zeroed fields stand for. + internal static class NativeColumnRules + { + internal static string? Check(NativeColumnSpecRaw spec, string? format) + { + if (spec.Type is < NativeType.String or > NativeType.Timestamp) + { + return $"unknown type {spec.Type}."; + } + if (spec.Scale is < 0 or > NativeType.MaxScale) + { + return $"scale {spec.Scale} is outside 0..{NativeType.MaxScale}."; + } + if (spec.Scale != 0 && spec.Type is not (NativeType.Float64 or NativeType.Decimal)) + { + return "scale applies only to FLOAT64 and DECIMAL columns."; + } + if (format is not null && (spec.Type is NativeType.String or NativeType.Decimal || spec.Scale != 0)) + { + return "a format applies only to INT64, an unscaled FLOAT64 and date or time columns."; + } + return CheckOptions(spec); + } + + private static string? CheckOptions(NativeColumnSpecRaw spec) + { + if (!NativeOptions.TryAscii(spec.TrimChar, ' ', out _) || !NativeOptions.TryAscii(spec.Padding, ' ', out _)) + { + return "trim_char and padding must be ASCII characters."; + } + if (spec.TrimMode is < 0 or > 2 || spec.Alignment is < 0 or > 2 || spec.Overflow is < 0 or > 1) + { + return "unknown trim_mode, alignment or overflow."; + } + if (spec.Overflow == 1 && spec.Type != NativeType.String) + { + return "truncate overflow applies only to STRING columns."; + } + return null; + } + + internal static NativeColumn Build(NativeColumnSpecRaw spec, string name, string? format) + { + NativeOptions.TryAscii(spec.TrimChar, ' ', out char trimChar); + NativeOptions.TryAscii(spec.Padding, ' ', out char padding); + return new NativeColumn + { + Name = name, + Start = spec.Start, + Length = spec.Length, + Type = spec.Type, + Nullable = spec.Nullable != 0, + Scale = spec.Scale, + Parse = new ColumnParseOptions(trimChar, TrimModeOf(spec.TrimMode), format), + FormatOptions = new ColumnFormatOptions(AlignmentOf(spec.Alignment, spec.Type), padding, format, + spec.Overflow == 1 ? OverflowBehavior.Truncate : OverflowBehavior.Throw), + }; + } + + private static TrimMode TrimModeOf(int value) + { + return value switch + { + 1 => TrimMode.Trailing, + 2 => TrimMode.Leading, + _ => TrimMode.Both, + }; + } + + private static Alignment AlignmentOf(int value, int type) + { + return value switch + { + 1 => Alignment.Left, + 2 => Alignment.Right, + _ => type == NativeType.String ? Alignment.Left : Alignment.Right, + }; + } + } +} diff --git a/src/FixedWidthParser.Native/Layouts/NativeColumnSpec.cs b/src/FixedWidthParser.Native/Layouts/NativeColumnSpec.cs new file mode 100644 index 0000000..b84d11e --- /dev/null +++ b/src/FixedWidthParser.Native/Layouts/NativeColumnSpec.cs @@ -0,0 +1,48 @@ +using System.Runtime.InteropServices; + +namespace FixedWidthParser.Native.Layouts +{ + /// The FW_T_* column types. + internal static class NativeType + { + internal const int String = 0; + internal const int Int64 = 1; + internal const int Float64 = 2; + internal const int Decimal = 3; + internal const int Date = 4; + internal const int Time = 5; + internal const int Timestamp = 6; + + internal const int MaxScale = 18; + } + + /// fw_column_spec. + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct NativeColumnSpecRaw + { + public int StructSize; + public byte* Name; + public int NameLen; + public int Start; + public int Length; + public int Type; + public int Nullable; + public int Scale; + public byte* Format; + public int FormatLen; + public int TrimChar; + public int TrimMode; + public int Alignment; + public int Padding; + public int Overflow; + } + + /// fw_layout_options. + [StructLayout(LayoutKind.Sequential)] + internal struct NativeLayoutOptionsRaw + { + public int StructSize; + public int LineLength; + public int Filler; + } +} diff --git a/src/FixedWidthParser.Native/Layouts/NativeLayout.cs b/src/FixedWidthParser.Native/Layouts/NativeLayout.cs new file mode 100644 index 0000000..41f4703 --- /dev/null +++ b/src/FixedWidthParser.Native/Layouts/NativeLayout.cs @@ -0,0 +1,122 @@ +using System.Text; +using FixedWidthParser.Attributes; +using FixedWidthParser.Infrastructure; + +namespace FixedWidthParser.Native.Layouts +{ + /// One validated column of a . + internal sealed class NativeColumn + { + internal required string Name { get; init; } + internal int Start { get; init; } + internal int Length { get; init; } + internal int Type { get; init; } + internal bool Nullable { get; init; } + internal int Scale { get; init; } + internal ColumnParseOptions Parse { get; init; } + internal ColumnFormatOptions FormatOptions { get; init; } + } + + /// + /// The C counterpart of FixedWidthLayout<T>: columns described at run time by + /// fw_column_specs, validated once and shared, immutable, by every read and write. + /// + internal sealed class NativeLayout + { + private const int MaxColumns = 16_384; + + private NativeLayout(NativeColumn[] columns, int lineLength, byte filler) + { + Columns = columns; + LineLength = lineLength; + Filler = filler; + } + + internal NativeColumn[] Columns { get; } + internal int LineLength { get; } + internal byte Filler { get; } + + internal static unsafe int TryCreate(ReadOnlySpan specs, NativeLayoutOptionsRaw* options, out NativeLayout? layout) + { + layout = null; + int status = NativeOptions.TryRead(options, "fw_layout_options", out var settings); + if (status != NativeStatus.Ok) + { + return status; + } + string? error = specs.Length is < 1 or > MaxColumns + ? $"A layout needs at least one column and at most {MaxColumns}; got {specs.Length}." + : null; + var columns = new NativeColumn[specs.Length]; + for (int i = 0; i < specs.Length && error is null; i++) + { + error = TryColumn(specs[i], i, out columns[i]); + } + error ??= ValidatePositions(columns); + if (error is not null) + { + return Fail(error); + } + return TryFinish(columns, settings, out layout); + } + + private static int TryFinish(NativeColumn[] columns, NativeLayoutOptionsRaw settings, out NativeLayout? layout) + { + layout = null; + int end = columns.Max(c => c.Start + c.Length); + int lineLength = settings.LineLength == 0 ? end : settings.LineLength; + if (lineLength < end) + { + return Fail($"line_length {settings.LineLength} is shorter than the columns, which end at {end}."); + } + if (!NativeOptions.TryAscii(settings.Filler, ' ', out char filler)) + { + return Fail($"filler {settings.Filler} is not an ASCII character."); + } + layout = new NativeLayout(columns, lineLength, (byte)filler); + return NativeStatus.Ok; + } + + private static string? ValidatePositions(NativeColumn[] columns) + { + var positions = columns.Select(c => (c.Start, c.Length, c.Name)).ToArray(); + try + { + ColumnLayoutValidator.Validate(positions, "the layout"); + return null; + } + catch (InvalidOperationException ex) + { + return ex.Message; + } + } + + private static unsafe string? TryColumn(NativeColumnSpecRaw spec, int index, out NativeColumn column) + { + column = null!; + if (spec.StructSize != sizeof(NativeColumnSpecRaw)) + { + return $"Column {index}: fw_column_spec struct_size is {spec.StructSize}; expected {sizeof(NativeColumnSpecRaw)}."; + } + if (spec.Name is null || spec.NameLen <= 0 || (spec.Format is null && spec.FormatLen != 0) || spec.FormatLen < 0) + { + return $"Column {index} has no name, or a name or format pointer that does not match its length."; + } + string name = Encoding.UTF8.GetString(spec.Name, spec.NameLen); + string? format = spec.FormatLen == 0 ? null : Encoding.UTF8.GetString(spec.Format, spec.FormatLen); + string? error = NativeColumnRules.Check(spec, format); + if (error is not null) + { + return $"Column \"{name}\": {error}"; + } + column = NativeColumnRules.Build(spec, name, format); + return null; + } + + private static int Fail(string message) + { + NativeApi.SetLastError(message); + return NativeStatus.InvalidArgument; + } + } +} diff --git a/src/FixedWidthParser.Native/NativeOptions.cs b/src/FixedWidthParser.Native/NativeOptions.cs new file mode 100644 index 0000000..9610640 --- /dev/null +++ b/src/FixedWidthParser.Native/NativeOptions.cs @@ -0,0 +1,33 @@ +namespace FixedWidthParser.Native +{ + /// + /// Reads a caller's options struct under the struct_size contract: means all + /// defaults (the zeroed struct), and any size other than the current one is refused. + /// + internal static unsafe class NativeOptions + { + internal static int TryRead(T* options, string name, out T value) where T : unmanaged + { + value = default; + if (options is null) + { + return NativeStatus.Ok; + } + int size = *(int*)options; + if (size != sizeof(T)) + { + NativeApi.SetLastError($"{name} struct_size is {size}; expected {sizeof(T)}."); + return NativeStatus.InvalidArgument; + } + value = *options; + return NativeStatus.Ok; + } + + /// An ASCII byte option where 0 means ; for anything outside ASCII. + internal static bool TryAscii(int value, char fallback, out char result) + { + result = value == 0 ? fallback : (char)value; + return value is >= 0 and <= 127; + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeLayoutTests.cs b/tests/FixedWidthParser.Tests/Native/NativeLayoutTests.cs new file mode 100644 index 0000000..781161a --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeLayoutTests.cs @@ -0,0 +1,153 @@ +using FixedWidthParser.Native; +using FixedWidthParser.Native.Layouts; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; + +namespace FixedWidthParser.Tests.Native +{ + public sealed class NativeLayoutTests + { + private static void AssertInvalid(string expectedFragment, params Col[] columns) + { + Assert.Equal(NativeStatus.InvalidArgument, Create(out var layout, columns)); + Assert.Null(layout); + Assert.Contains(expectedFragment, LastError(), StringComparison.Ordinal); + } + + [Fact] + public void Create_ValidCotahistColumns_Succeeds() + { + var layout = Build( + new Col("tipreg", 0, 2, TString), + new Col("data_pregao", 2, 8, TDate) { Format = "yyyyMMdd" }, + new Col("codneg", 12, 12, TString), + new Col("preabe", 56, 13, TFloat64) { Scale = 2 }, + new Col("quatot", 152, 18, TInt64), + new Col("voltot", 170, 18, TDecimal) { Scale = 2, Nullable = true }); + + Assert.Equal(6, layout.Columns.Length); + Assert.Equal("preabe", layout.Columns[3].Name); + Assert.Equal(2, layout.Columns[3].Scale); + Assert.Equal("yyyyMMdd", layout.Columns[1].Parse.Format); + Assert.True(layout.Columns[5].Nullable); + } + + [Fact] + public void Create_Defaults_FollowTheSpec() + { + var layout = Build(new Col("name", 0, 5, TString), new Col("qty", 5, 4, TInt64)); + + Assert.Equal(TrimMode.Both, layout.Columns[0].Parse.TrimMode); + Assert.Equal(' ', layout.Columns[0].Parse.TrimChar); + Assert.Equal(Alignment.Left, layout.Columns[0].FormatOptions.Alignment); + Assert.Equal(Alignment.Right, layout.Columns[1].FormatOptions.Alignment); + Assert.Equal(' ', layout.Columns[1].FormatOptions.Padding); + Assert.Equal(OverflowBehavior.Throw, layout.Columns[0].FormatOptions.Overflow); + Assert.Equal((byte)' ', layout.Filler); + } + + [Fact] + public void Create_Overlap_NamesBothColumns() + { + AssertInvalid("\"a\" [0, 5) and \"b\" [3, 6)", new Col("a", 0, 5, TString), new Col("b", 3, 3, TString)); + } + + [Fact] + public void Create_NegativeStart_Fails() + { + AssertInvalid("negative Start (-1)", new Col("a", -1, 5, TString)); + } + + [Fact] + public void Create_ZeroLength_Fails() + { + AssertInvalid("invalid Length (0)", new Col("a", 0, 0, TString)); + } + + [Fact] + public void Create_NonAsciiPadding_Fails() + { + AssertInvalid("padding", new Col("a", 0, 5, TString) { Padding = 'é' }); + } + + [Fact] + public void Create_ScaleOutOfRange_Fails() + { + AssertInvalid("scale", new Col("a", 0, 20, TFloat64) { Scale = 19 }); + } + + [Fact] + public void Create_ScaleOnInt64_Fails() + { + AssertInvalid("scale", new Col("a", 0, 5, TInt64) { Scale = 2 }); + } + + [Fact] + public void Create_FormatOnString_Fails() + { + AssertInvalid("format", new Col("a", 0, 5, TString) { Format = "x" }); + } + + [Fact] + public void Create_TruncateOnNumber_Fails() + { + AssertInvalid("truncate", new Col("a", 0, 5, TInt64) { Overflow = 1 }); + } + + [Fact] + public void Create_UnknownType_Fails() + { + AssertInvalid("type", new Col("a", 0, 5, 42)); + } + + [Fact] + public void Create_NoName_Fails() + { + AssertInvalid("name", new Col("", 0, 5, TString)); + } + + [Fact] + public void Create_NoColumns_Fails() + { + AssertInvalid("at least one column"); + } + + [Fact] + public void Create_MixedStructSizes_Fails() + { + AssertInvalid("struct_size", new Col("a", 0, 5, TString), new Col("b", 5, 5, TString) { StructSize = 8 }); + } + + [Fact] + public void Options_UnknownStructSize_IsInvalidArgument() + { + var options = Options() with { StructSize = 4 }; + + Assert.Equal(NativeStatus.InvalidArgument, Create(out _, options, new Col("a", 0, 5, TString))); + Assert.Contains("fw_layout_options struct_size is 4", LastError(), StringComparison.Ordinal); + } + + [Fact] + public void LineLength_DefaultsToLastColumnEnd() + { + var layout = Build(new Col("a", 10, 5, TString), new Col("b", 0, 3, TString)); + + Assert.Equal(15, layout.LineLength); + } + + [Fact] + public void LineLength_ShorterThanColumns_Fails() + { + Assert.Equal(NativeStatus.InvalidArgument, Create(out _, Options(lineLength: 4), new Col("a", 0, 5, TString))); + Assert.Contains("line_length", LastError(), StringComparison.Ordinal); + } + + [Fact] + public void LineLength_AndFiller_AreKept() + { + Assert.Equal(NativeStatus.Ok, Create(out var layout, Options(lineLength: 245, filler: '0'), new Col("a", 0, 5, TString))); + + Assert.Equal(245, layout!.LineLength); + Assert.Equal((byte)'0', layout.Filler); + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs b/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs new file mode 100644 index 0000000..153c91f --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs @@ -0,0 +1,112 @@ +using System.Runtime.InteropServices; +using System.Text; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Layouts; + +namespace FixedWidthParser.Tests.Native +{ + /// One column as a C caller would describe it; unset fields stay zero, the C defaults. + internal sealed record Col(string Name, int Start, int Length, int Type) + { + public bool Nullable { get; init; } + public int Scale { get; init; } + public string? Format { get; init; } + public int TrimChar { get; init; } + public int TrimMode { get; init; } + public int Alignment { get; init; } + public int Padding { get; init; } + public int Overflow { get; init; } + public int StructSize { get; init; } = Marshal.SizeOf(); + } + + /// Builds s from s through the same entry point fw_layout_create uses. + internal static unsafe class NativeTestLayouts + { + internal const int TString = 0; + internal const int TInt64 = 1; + internal const int TFloat64 = 2; + internal const int TDecimal = 3; + internal const int TDate = 4; + internal const int TTime = 5; + internal const int TTimestamp = 6; + + internal static int Create(out NativeLayout? layout, params Col[] columns) + { + return Create(out layout, null, columns); + } + + internal static int Create(out NativeLayout? layout, NativeLayoutOptionsRaw? options, params Col[] columns) + { + var allocations = new List(); + try + { + var specs = columns.Select(c => ToRaw(c, allocations)).ToArray(); + if (options is { } value) + { + return NativeLayout.TryCreate(specs, &value, out layout); + } + return NativeLayout.TryCreate(specs, null, out layout); + } + finally + { + allocations.ForEach(Marshal.FreeHGlobal); + } + } + + internal static NativeLayout Build(params Col[] columns) + { + int status = Create(out var layout, columns); + Assert.True(status == NativeStatus.Ok, $"status {status}: {LastError()}"); + return layout!; + } + + internal static NativeLayoutOptionsRaw Options(int lineLength = 0, int filler = 0) + { + return new NativeLayoutOptionsRaw { StructSize = sizeof(NativeLayoutOptionsRaw), LineLength = lineLength, Filler = filler }; + } + + internal static string LastError() + { + var buffer = new byte[4096]; + NativeApi.LastError(buffer, out int length); + return Encoding.UTF8.GetString(buffer, 0, length); + } + + private static NativeColumnSpecRaw ToRaw(Col column, List allocations) + { + var (name, nameLength) = Utf8(column.Name, allocations); + var (format, formatLength) = Utf8(column.Format, allocations); + return new NativeColumnSpecRaw + { + StructSize = column.StructSize, + Name = (byte*)name, + NameLen = nameLength, + Start = column.Start, + Length = column.Length, + Type = column.Type, + Nullable = column.Nullable ? 1 : 0, + Scale = column.Scale, + Format = (byte*)format, + FormatLen = formatLength, + TrimChar = column.TrimChar, + TrimMode = column.TrimMode, + Alignment = column.Alignment, + Padding = column.Padding, + Overflow = column.Overflow, + }; + } + + private static (IntPtr Pointer, int Length) Utf8(string? text, List allocations) + { + if (text is null) + { + return (IntPtr.Zero, 0); + } + byte[] bytes = Encoding.UTF8.GetBytes(text); + IntPtr memory = Marshal.AllocHGlobal(Math.Max(1, bytes.Length)); + allocations.Add(memory); + Marshal.Copy(bytes, 0, memory, bytes.Length); + return (memory, bytes.Length); + } + } +} From 030629c2ca8ca8a704ec8905a8dbccfc740c7ae5 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 00:43:17 -0300 Subject: [PATCH 36/85] feat(native): read fixed-width files into a columnar table fw_read_file and fw_read_memory parse every line straight into native column buffers (int64, float64 with implied decimals, decimal, date, time, timestamp, UTF-8 strings with int32 offsets, validity bitmaps on the first null) through the core's partitioned reader, and publish them as an fw_table freed with fw_free_table. fw_read_options adds header_lines, skip_invalid, encoding, max_line_length and the byte order mark switch; fw_read_result reports lines read and skipped; a parse error sets the line and the layout column. Co-Authored-By: Claude Opus 5.5 --- .../Exports.Reading.cs | 56 ++++ .../GrowableNativeBuffer.cs | 20 ++ .../Reading/ColumnBuilder.cs | 253 +++++++++++++++ .../Reading/NativeTable.cs | 69 ++++ .../Reading/PointerSource.cs | 29 ++ .../Reading/ReadApi.cs | 136 ++++++++ .../Reading/ReadSettings.cs | 84 +++++ .../Reading/TableSink.cs | 96 ++++++ .../FixedWidthReaderOptions.cs | 5 +- .../Native/NativeLayoutTests.cs | 6 +- .../Native/NativeReadTests.cs | 300 ++++++++++++++++++ .../Native/NativeTestLayouts.cs | 2 +- .../Native/NativeTestReads.cs | 135 ++++++++ 13 files changed, 1186 insertions(+), 5 deletions(-) create mode 100644 src/FixedWidthParser.Native/Exports.Reading.cs create mode 100644 src/FixedWidthParser.Native/Reading/ColumnBuilder.cs create mode 100644 src/FixedWidthParser.Native/Reading/NativeTable.cs create mode 100644 src/FixedWidthParser.Native/Reading/PointerSource.cs create mode 100644 src/FixedWidthParser.Native/Reading/ReadApi.cs create mode 100644 src/FixedWidthParser.Native/Reading/ReadSettings.cs create mode 100644 src/FixedWidthParser.Native/Reading/TableSink.cs create mode 100644 tests/FixedWidthParser.Tests/Native/NativeReadTests.cs create mode 100644 tests/FixedWidthParser.Tests/Native/NativeTestReads.cs diff --git a/src/FixedWidthParser.Native/Exports.Reading.cs b/src/FixedWidthParser.Native/Exports.Reading.cs new file mode 100644 index 0000000..ece4cc3 --- /dev/null +++ b/src/FixedWidthParser.Native/Exports.Reading.cs @@ -0,0 +1,56 @@ +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native +{ + internal static unsafe partial class Exports + { + [UnmanagedCallersOnly(EntryPoint = "fw_read_file")] + public static int ReadFile(byte* path, int pathLength, nint layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult) + { + if (path is null || pathLength <= 0 || outTable is null) + { + return InvalidArgument("fw_read_file needs a path and out_table."); + } + *outTable = default; + return WithLayout(layout, l => ReadApi.ReadFile(new ReadOnlySpan(path, pathLength), l, options, outTable, outResult)); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_read_memory")] + public static int ReadMemory(byte* data, long dataLength, nint layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult) + { + if ((data is null && dataLength != 0) || dataLength < 0 || outTable is null) + { + return InvalidArgument("fw_read_memory needs data for a non-zero data_len, and out_table."); + } + *outTable = default; + return WithLayout(layout, l => ReadApi.ReadMemory(data, dataLength, l, options, outTable, outResult)); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_free_table")] + public static void FreeTable(NativeTableRaw* table) + { + NativeTable.Free(table); + } + + private static int WithLayout(nint handle, Func call) + { + var layout = NativeHandleTable.Resolve(handle); + if (layout is null) + { + NativeApi.SetLastError("The layout handle is not a live fw_layout."); + return NativeStatus.InvalidHandle; + } + try + { + return call(layout); + } + catch (Exception ex) + { + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + } +} diff --git a/src/FixedWidthParser.Native/GrowableNativeBuffer.cs b/src/FixedWidthParser.Native/GrowableNativeBuffer.cs index 858d5cd..0ad1d09 100644 --- a/src/FixedWidthParser.Native/GrowableNativeBuffer.cs +++ b/src/FixedWidthParser.Native/GrowableNativeBuffer.cs @@ -22,6 +22,26 @@ internal sealed unsafe class GrowableNativeBuffer : IDisposable where T : unm internal ref T Last => ref _data[Count - 1]; + internal ref T this[int index] => ref _data[index]; + + /// Drops every element from on. + internal void Truncate(int count) + { + Count = count; + } + + /// Appends uninitialized elements and returns them to be filled. + internal Span Append(int count) + { + if (count > _capacity - Count) + { + Grow(checked(Count + count)); + } + var span = new Span(_data + Count, count); + Count += count; + return span; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void Add(T value) { diff --git a/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs new file mode 100644 index 0000000..7c6238f --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs @@ -0,0 +1,253 @@ +using System.Buffers; +using System.Globalization; +using System.Text; +using FixedWidthParser.Infrastructure; +using FixedWidthParser.Native.Layouts; + +namespace FixedWidthParser.Native.Reading +{ + /// + /// One column of one partition, parsed straight into native buffers that are handed to the caller as they + /// are. Values: long for INT64, DECIMAL, TIME and TIMESTAMP, double for FLOAT64, int + /// for DATE and for STRING offsets into UTF-8 bytes. The validity bitmap exists only once a null arrives. + /// + internal sealed unsafe class ColumnBuilder : IDisposable + { + private const int StackChars = 256; + private static readonly int EpochDay = new DateOnly(1970, 1, 1).DayNumber; + private static readonly double[] Pow10 = [.. Enumerable.Range(0, NativeType.MaxScale + 1).Select(p => Math.Pow(10, p))]; + + private readonly NativeColumn _column; + private readonly GrowableNativeBuffer _longs = new(); + private readonly GrowableNativeBuffer _doubles = new(); + private readonly GrowableNativeBuffer _ints = new(); + private readonly GrowableNativeBuffer _bytes = new(); +#pragma warning disable SS066 // Disposed in Dispose, or detached by Publish. + private GrowableNativeBuffer? _validity; +#pragma warning restore SS066 + + internal ColumnBuilder(NativeColumn column) + { + _column = column; + if (column.Type == NativeType.String) + { + _ints.Add(0); + } + } + + internal int Rows { get; private set; } + + internal bool TryAppend(ReadOnlySpan raw, in FixedWidthContext context) + { + var parse = _column.Parse; + if (_column.Nullable && FixedWidthRuntime.IsBlank(raw, in parse)) + { + AppendNull(); + return true; + } + bool parsed = _column.Type switch + { + NativeType.String => AppendString(FixedWidthRuntime.TrimColumn(raw, in parse), in context), + NativeType.Int64 => Add(FixedWidthRuntime.TryParse(raw, in context, in parse, out long integer), integer), + NativeType.Float64 => AppendFloat(raw, in context, in parse), + NativeType.Decimal => Add(TryDecimal(FixedWidthRuntime.TrimColumn(raw, in parse), _column.Scale, out long unscaled), unscaled), + _ => AppendTemporal(raw, in context, in parse), + }; + if (parsed) + { + SetValid(Rows, true); + Rows++; + } + return parsed; + } + + /// Drops every row from on, undoing the columns of a line rejected midway. + internal void Truncate(int rows) + { + Rows = rows; + _longs.Truncate(Math.Min(_longs.Count, rows)); + _doubles.Truncate(Math.Min(_doubles.Count, rows)); + if (_column.Type != NativeType.String) + { + _ints.Truncate(Math.Min(_ints.Count, rows)); + return; + } + _ints.Truncate(rows + 1); + _bytes.Truncate(_ints[rows]); + } + + internal void Publish(NativeColumnRaw* target) + { + target->Type = _column.Type; + target->Scale = _column.Scale; + target->Length = Rows; + target->DataLen = _bytes.Count; + target->Validity = _validity is null ? null : (byte*)_validity.Detach(); + target->Data = _column.Type == NativeType.String ? (byte*)_bytes.Detach() : null; + target->Values = (void*)(_column.Type switch + { + NativeType.Float64 => _doubles.Detach(), + NativeType.String or NativeType.Date => _ints.Detach(), + _ => _longs.Detach(), + }); + } + + public void Dispose() + { + _longs.Dispose(); + _doubles.Dispose(); + _ints.Dispose(); + _bytes.Dispose(); + _validity?.Dispose(); + } + + private bool Add(bool parsed, long value) + { + if (parsed) + { + _longs.Add(value); + } + return parsed; + } + + private bool AppendFloat(ReadOnlySpan raw, in FixedWidthContext context, in ColumnParseOptions parse) + { + if (_column.Scale > 0 && long.TryParse(FixedWidthRuntime.TrimColumn(raw, in parse), NumberStyles.Integer, CultureInfo.InvariantCulture, out long digits)) + { + _doubles.Add(digits / Pow10[_column.Scale]); + return true; + } + if (!FixedWidthRuntime.TryDouble(raw, in context, in parse, out double value)) + { + return false; + } + _doubles.Add(value); + return true; + } + + private static bool TryDecimal(ReadOnlySpan text, int scale, out long unscaled) + { + if (long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out unscaled)) + { + return true; + } + if (!decimal.TryParse(text, NumberStyles.Number, CultureInfo.InvariantCulture, out decimal value)) + { + return false; + } + decimal scaled = value * (decimal)Pow10[scale]; + if (scaled != decimal.Truncate(scaled) || scaled is < long.MinValue or > long.MaxValue) + { + return false; + } + unscaled = (long)scaled; + return true; + } + + private bool AppendTemporal(ReadOnlySpan raw, in FixedWidthContext context, in ColumnParseOptions parse) + { + bool exact = parse.Format is not null; + switch (_column.Type) + { + case NativeType.Date: + bool dateOk = exact ? FixedWidthRuntime.TryDateOnlyExact(raw, in context, in parse, out var date) : FixedWidthRuntime.TryParseDecoded(raw, in context, in parse, out date); + if (dateOk) + { + _ints.Add(date.DayNumber - EpochDay); + } + return dateOk; + case NativeType.Time: + bool timeOk = exact ? FixedWidthRuntime.TryTimeOnlyExact(raw, in context, in parse, out var time) : FixedWidthRuntime.TryParseDecoded(raw, in context, in parse, out time); + return Add(timeOk, time.Ticks / TimeSpan.TicksPerMicrosecond); + default: + bool stampOk = exact ? FixedWidthRuntime.TryDateTimeExact(raw, in context, in parse, out var stamp) : FixedWidthRuntime.TryParseDecoded(raw, in context, in parse, out stamp); + return Add(stampOk, (stamp.Ticks - DateTime.UnixEpoch.Ticks) / TimeSpan.TicksPerMicrosecond); + } + } + + private bool AppendString(ReadOnlySpan text, in FixedWidthContext context) + { + if (!context.IsUtf8) + { + AppendTranscoded(text, context.Encoding); + _ints.Add(_bytes.Count); + return true; + } + EnsureTextRoom(text.Length); + _bytes.AddRange(text); + _ints.Add(_bytes.Count); + return true; + } + + private void AppendTranscoded(ReadOnlySpan text, Encoding encoding) + { + int charCount = encoding.GetCharCount(text); + char[]? rented = charCount > StackChars ? ArrayPool.Shared.Rent(charCount) : null; + try + { + Span chars = rented ?? stackalloc char[StackChars]; + chars = chars[..encoding.GetChars(text, chars)]; + int byteCount = Encoding.UTF8.GetByteCount(chars); + EnsureTextRoom(byteCount); + Encoding.UTF8.GetBytes(chars, _bytes.Append(byteCount)); + } + finally + { + if (rented is not null) + { + ArrayPool.Shared.Return(rented); + } + } + } + + private void EnsureTextRoom(int bytes) + { + if ((long)_bytes.Count + bytes > int.MaxValue) + { + throw new InvalidOperationException( + $"Column '{_column.Name}' holds more than 2 GiB of text; read in batches (fw_reader_open_*)."); + } + } + + private void AppendNull() + { + switch (_column.Type) + { + case NativeType.String: + _ints.Add(_bytes.Count); + break; + case NativeType.Float64: + _doubles.Add(0); + break; + case NativeType.Date: + _ints.Add(0); + break; + default: + _longs.Add(0); + break; + } + SetValid(Rows, false); + Rows++; + } + + private void SetValid(int row, bool valid) + { + if (_validity is null) + { + if (valid) + { + return; + } + _validity = new GrowableNativeBuffer(); + _validity.Append((row >> 3) + 1).Fill(0xFF); + } + while (_validity.Count <= row >> 3) + { + _validity.Add(0xFF); + } + ref byte bits = ref _validity[row >> 3]; + int mask = 1 << (row & 7); + bits = (byte)(valid ? bits | mask : bits & ~mask); + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/NativeTable.cs b/src/FixedWidthParser.Native/Reading/NativeTable.cs new file mode 100644 index 0000000..c1ebb36 --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/NativeTable.cs @@ -0,0 +1,69 @@ +using System.Runtime.InteropServices; + +namespace FixedWidthParser.Native.Reading +{ + /// fw_column: one column of a published table. + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct NativeColumnRaw + { + public int Type; + public int Scale; + public long Length; + public void* Values; + public byte* Validity; + public byte* Data; + public long DataLen; + } + + /// fw_table. + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct NativeTableRaw + { + public int ColumnCount; + public long RowCount; + public NativeColumnRaw* Columns; + } + + /// fw_read_options. + [StructLayout(LayoutKind.Sequential)] + internal struct NativeReadOptionsRaw + { + public int StructSize; + public long HeaderLines; + public int SkipInvalid; + public int DegreeOfParallelism; + public int Encoding; + public int MaxLineLength; + public int SkipByteOrderMark; + } + + /// fw_read_result: an optional output, so its struct_size is checked, not read as options. + [StructLayout(LayoutKind.Sequential)] + internal struct NativeReadResultRaw + { + public int StructSize; + public long LinesRead; + public long LinesSkipped; + } + + internal static unsafe class NativeTable + { + /// Frees every buffer of a table published by the library and zeroes it; safe on a zeroed table. + internal static void Free(NativeTableRaw* table) + { + if (table is null || table->Columns is null) + { + return; + } + for (int i = 0; i < table->ColumnCount; i++) + { + var column = table->Columns[i]; + Marshal.FreeHGlobal((IntPtr)column.Values); + Marshal.FreeHGlobal((IntPtr)column.Validity); + Marshal.FreeHGlobal((IntPtr)column.Data); + } + Marshal.FreeHGlobal((IntPtr)table->Columns); + *table = default; + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/PointerSource.cs b/src/FixedWidthParser.Native/Reading/PointerSource.cs new file mode 100644 index 0000000..b851978 --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/PointerSource.cs @@ -0,0 +1,29 @@ +using FixedWidthParser.Readers; + +namespace FixedWidthParser.Native.Reading +{ + /// The caller's memory as a random-access source. The memory must stay valid while it is read. + internal sealed unsafe class PointerSource : IRandomAccessSource + { + private readonly byte* _data; + + internal PointerSource(byte* data, long length) + { + _data = data; + Length = length; + } + + public long Length { get; } + + public int Read(Span destination, long offset) + { + if (offset >= Length) + { + return 0; + } + int count = (int)Math.Min(destination.Length, Length - offset); + new ReadOnlySpan(_data + offset, count).CopyTo(destination); + return count; + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/ReadApi.cs b/src/FixedWidthParser.Native/Reading/ReadApi.cs new file mode 100644 index 0000000..d1ceaf0 --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/ReadApi.cs @@ -0,0 +1,136 @@ +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Text; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Readers; + +namespace FixedWidthParser.Native.Reading +{ + /// fw_read_file and fw_read_memory: a whole source into one columnar table. + internal static unsafe class ReadApi + { + internal static int ReadFile(ReadOnlySpan path, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult) + { + try + { + using var handle = File.OpenHandle(Encoding.UTF8.GetString(path), FileMode.Open, FileAccess.Read, FileShare.Read); + return Read(new FileSource(handle), layout, options, outTable, outResult); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException) + { + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + + internal static int ReadMemory(byte* data, long length, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult) + { + return Read(new PointerSource(data, length), layout, options, outTable, outResult); + } + + private static int Read(IRandomAccessSource source, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult) + { + *outTable = default; + int status = CheckResult(outResult); + if (status != NativeStatus.Ok) + { + return status; + } + status = ReadSettings.TryCreate(options, out var settings); + if (status != NativeStatus.Ok) + { + return status; + } + try + { + long dataStart = HeaderEnd(source, settings!.HeaderLines, out long headerLines); + var skipped = new StrongBox(headerLines); + var readerOptions = settings.ReaderOptions(skipped, degreeOfParallelism: 1); +#pragma warning disable VSTHRD002 // An export is synchronous and runs with no synchronization context. + using var table = ParallelAggregator.RunAsync(source, dataStart, () => new TableSink(layout), readerOptions, CancellationToken.None) + .GetAwaiter().GetResult(); +#pragma warning restore VSTHRD002 + table.Publish(outTable); + WriteResult(outResult, table.Rows, skipped.Value); + return NativeStatus.Ok; + } + catch (FixedWidthParseException ex) + { + NativeApi.SetParseError(ex, Array.FindIndex(layout.Columns, c => string.Equals(c.Name, ex.ColumnName, StringComparison.Ordinal))); + return NativeStatus.ParseError; + } + catch (Exception ex) + { + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + + private static int CheckResult(NativeReadResultRaw* result) + { + if (result is not null && result->StructSize != sizeof(NativeReadResultRaw)) + { + NativeApi.SetLastError($"fw_read_result struct_size is {result->StructSize}; expected {sizeof(NativeReadResultRaw)}."); + return NativeStatus.InvalidArgument; + } + return NativeStatus.Ok; + } + + private static void WriteResult(NativeReadResultRaw* result, long rows, long skipped) + { + if (result is null) + { + return; + } + result->LinesRead = rows; + result->LinesSkipped = skipped; + } + + /// The offset just past the first physical lines, and how many lines that was. + internal static long HeaderEnd(IRandomAccessSource source, long count, out long skipped) + { + skipped = 0; + if (count == 0) + { + return 0; + } + byte[] buffer = ArrayPool.Shared.Rent(64 * 1024); + try + { + long offset = 0; + int read; + while ((read = source.Read(buffer, offset)) > 0) + { + var block = buffer.AsSpan(0, read); + for (int at = block.IndexOf((byte)'\n'); at >= 0; at = NextBreak(block, at)) + { + if (++skipped == count) + { + return offset + at + 1; + } + } + offset += read; + } + skipped += offset > 0 && LastByte(source, offset) != (byte)'\n' ? 1 : 0; + return offset; + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + private static int NextBreak(ReadOnlySpan block, int at) + { + int next = block[(at + 1)..].IndexOf((byte)'\n'); + return next < 0 ? -1 : at + 1 + next; + } + + private static byte LastByte(IRandomAccessSource source, long length) + { + Span last = stackalloc byte[1]; + source.Read(last, length - 1); + return last[0]; + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/ReadSettings.cs b/src/FixedWidthParser.Native/Reading/ReadSettings.cs new file mode 100644 index 0000000..bdb8b27 --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/ReadSettings.cs @@ -0,0 +1,84 @@ +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Text; + +namespace FixedWidthParser.Native.Reading +{ + /// A validated fw_read_options, turned into the core's reader options. + internal sealed class ReadSettings + { + private const int OptDefault = 0; + private const int OptTrue = 2; + + private ReadSettings(NativeReadOptionsRaw raw, Encoding? encoding) + { + HeaderLines = raw.HeaderLines; + SkipInvalid = raw.SkipInvalid != 0; + DegreeOfParallelism = raw.DegreeOfParallelism; + MaxLineLength = raw.MaxLineLength == 0 ? 1024 * 1024 : raw.MaxLineLength; + ByteOrderMark = raw.SkipByteOrderMark == OptDefault ? null : raw.SkipByteOrderMark == OptTrue; + Encoding = encoding; + } + + internal long HeaderLines { get; } + internal bool SkipInvalid { get; } + internal int DegreeOfParallelism { get; } + internal int MaxLineLength { get; } + internal bool? ByteOrderMark { get; } + internal Encoding? Encoding { get; } + + internal static unsafe int TryCreate(NativeReadOptionsRaw* options, out ReadSettings? settings) + { + settings = null; + int status = NativeOptions.TryRead(options, "fw_read_options", out var raw); + if (status != NativeStatus.Ok) + { + return status; + } + if (raw.HeaderLines < 0 || raw.DegreeOfParallelism < 0 || raw.MaxLineLength < 0 || raw.SkipByteOrderMark is < 0 or > OptTrue) + { + NativeApi.SetLastError("fw_read_options: header_lines, degree_of_parallelism and max_line_length must not be negative, and skip_byte_order_mark must be an FW_OPT_* value."); + return NativeStatus.InvalidArgument; + } + if (!TryEncoding(raw.Encoding, out var encoding)) + { + NativeApi.SetLastError($"fw_read_options: encoding {raw.Encoding} is not a code page this library knows."); + return NativeStatus.InvalidArgument; + } + settings = new ReadSettings(raw, encoding); + return NativeStatus.Ok; + } + + /// The core's options; a skipped line is counted in . + internal FixedWidthReaderOptions ReaderOptions(StrongBox skipped, int degreeOfParallelism) + { + return FixedWidthReaderOptions.Validate(new FixedWidthReaderOptions + { + FormatProvider = CultureInfo.InvariantCulture, + Encoding = Encoding, + MaxLineLength = MaxLineLength, + DegreeOfParallelism = degreeOfParallelism, + ByteOrderMark = ByteOrderMark, + OnParseError = SkipInvalid ? _ => Interlocked.Increment(ref skipped.Value) > 0 : null, + }); + } + + internal static bool TryEncoding(int codePage, out Encoding? encoding) + { + encoding = null; + if (codePage == 0) + { + return true; + } + try + { + encoding = Encoding.GetEncoding(codePage); + return true; + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException) + { + return false; + } + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/TableSink.cs b/src/FixedWidthParser.Native/Reading/TableSink.cs new file mode 100644 index 0000000..6b0555c --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/TableSink.cs @@ -0,0 +1,96 @@ +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Readers; + +namespace FixedWidthParser.Native.Reading +{ + /// One partition's columns: each accepted line appends one value to every . + internal struct TableSink : ILineSink, IDisposable + { + private readonly NativeLayout _layout; + private readonly ColumnBuilder[] _columns; + private readonly int _requiredLength; + private int _failedColumn; + + internal TableSink(NativeLayout layout) + { + _layout = layout; + _columns = [.. layout.Columns.Select(c => new ColumnBuilder(c))]; + _requiredLength = layout.Columns.Max(c => c.Start + c.Length); + _failedColumn = -1; + Rows = 0; + } + + internal int Rows { get; private set; } + + public bool TryAdd(ReadOnlySpan line, in FixedWidthContext context) + { + if (line.Length < _requiredLength) + { + _failedColumn = -1; + return false; + } + var columns = _layout.Columns; + for (int i = 0; i < _columns.Length; i++) + { + if (!_columns[i].TryAppend(line.Slice(columns[i].Start, columns[i].Length), in context)) + { + Reject(i); + return false; + } + } + Rows++; + return true; + } + + public readonly FixedWidthParseException Diagnose(ReadOnlySpan line, long lineNumber, in FixedWidthContext context) + { + if (_failedColumn < 0) + { + return new FixedWidthParseException( + $"Line {lineNumber} could not be parsed: it is {line.Length} bytes long and the layout needs {_requiredLength}.", + lineNumber, columnName: null, columnStart: -1, columnLength: 0, rawValue: null); + } + var column = _layout.Columns[_failedColumn]; + string raw = context.Encoding.GetString(line.Slice(column.Start, column.Length)); + return new FixedWidthParseException( + $"Line {lineNumber} could not be parsed: column '{column.Name}' (start {column.Start}, length {column.Length}) holds \"{raw}\".", + lineNumber, column.Name, column.Start, column.Length, raw); + } + + public readonly void Merge(TableSink following) + { + throw new NotSupportedException("Partitioned reads arrive in a later step."); + } + + /// Hands every column's buffers to ; the caller frees them with . + internal readonly unsafe void Publish(NativeTableRaw* table) + { + var columns = (NativeColumnRaw*)Marshal.AllocHGlobal(_columns.Length * sizeof(NativeColumnRaw)); + for (int i = 0; i < _columns.Length; i++) + { + _columns[i].Publish(columns + i); + } + table->ColumnCount = _columns.Length; + table->RowCount = Rows; + table->Columns = columns; + } + + public readonly void Dispose() + { + foreach (var column in _columns) + { + column.Dispose(); + } + } + + private void Reject(int failed) + { + _failedColumn = failed; + foreach (ref readonly var column in _columns.AsSpan(0, failed)) + { + column.Truncate(Rows); + } + } + } +} diff --git a/src/FixedWidthParser/FixedWidthReaderOptions.cs b/src/FixedWidthParser/FixedWidthReaderOptions.cs index 9db4697..48bdb5a 100644 --- a/src/FixedWidthParser/FixedWidthReaderOptions.cs +++ b/src/FixedWidthParser/FixedWidthReaderOptions.cs @@ -64,7 +64,10 @@ internal int Workers internal FixedWidthContext Context => new() { FormatProvider = FormatProvider!, Encoding = Encoding!, StringPool = StringPool }; - internal bool SkipsByteOrderMark => Encoding is null || Encoding.CodePage == Encoding.UTF8.CodePage; + // Overrides whether a leading UTF-8 byte order mark is skipped; null decides from the encoding. + internal bool? ByteOrderMark { get; init; } + + internal bool SkipsByteOrderMark => ByteOrderMark ?? (Encoding is null || Encoding.CodePage == Encoding.UTF8.CodePage); internal static FixedWidthReaderOptions Validate(FixedWidthReaderOptions? options) { diff --git a/tests/FixedWidthParser.Tests/Native/NativeLayoutTests.cs b/tests/FixedWidthParser.Tests/Native/NativeLayoutTests.cs index 781161a..b2108e1 100644 --- a/tests/FixedWidthParser.Tests/Native/NativeLayoutTests.cs +++ b/tests/FixedWidthParser.Tests/Native/NativeLayoutTests.cs @@ -120,7 +120,7 @@ public void Create_MixedStructSizes_Fails() [Fact] public void Options_UnknownStructSize_IsInvalidArgument() { - var options = Options() with { StructSize = 4 }; + var options = LayoutOptions() with { StructSize = 4 }; Assert.Equal(NativeStatus.InvalidArgument, Create(out _, options, new Col("a", 0, 5, TString))); Assert.Contains("fw_layout_options struct_size is 4", LastError(), StringComparison.Ordinal); @@ -137,14 +137,14 @@ public void LineLength_DefaultsToLastColumnEnd() [Fact] public void LineLength_ShorterThanColumns_Fails() { - Assert.Equal(NativeStatus.InvalidArgument, Create(out _, Options(lineLength: 4), new Col("a", 0, 5, TString))); + Assert.Equal(NativeStatus.InvalidArgument, Create(out _, LayoutOptions(lineLength: 4), new Col("a", 0, 5, TString))); Assert.Contains("line_length", LastError(), StringComparison.Ordinal); } [Fact] public void LineLength_AndFiller_AreKept() { - Assert.Equal(NativeStatus.Ok, Create(out var layout, Options(lineLength: 245, filler: '0'), new Col("a", 0, 5, TString))); + Assert.Equal(NativeStatus.Ok, Create(out var layout, LayoutOptions(lineLength: 245, filler: '0'), new Col("a", 0, 5, TString))); Assert.Equal(245, layout!.LineLength); Assert.Equal((byte)'0', layout.Filler); diff --git a/tests/FixedWidthParser.Tests/Native/NativeReadTests.cs b/tests/FixedWidthParser.Tests/Native/NativeReadTests.cs new file mode 100644 index 0000000..a3e7f61 --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeReadTests.cs @@ -0,0 +1,300 @@ +using System.Text; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; +using static FixedWidthParser.Tests.Native.NativeTestReads; + +namespace FixedWidthParser.Tests.Native +{ + public sealed unsafe class NativeReadTests + { + private static readonly DateOnly Epoch = new(1970, 1, 1); + + private static int Days(int year, int month, int day) + { + return new DateOnly(year, month, day).DayNumber - Epoch.DayNumber; + } + + private static ReadOutcome[] Ok(string content, NativeLayout layout, NativeReadOptionsRaw? options = null, Encoding? encoding = null) + { + var outcomes = Both(content, layout, options, encoding); + foreach (var outcome in outcomes) + { + Assert.True(outcome.Status == NativeStatus.Ok, $"status {outcome.Status}: {outcome.Error}"); + } + return outcomes; + } + + [Fact] + public void Read_EveryType_ProducesColumns() + { + var layout = Build( + new Col("name", 0, 6, TString), + new Col("qty", 6, 5, TInt64), + new Col("price", 11, 8, TFloat64), + new Col("amount", 19, 8, TDecimal) { Scale = 2 }, + new Col("day", 27, 8, TDate) { Format = "yyyyMMdd" }, + new Col("time", 35, 6, TTime) { Format = "HHmmss" }, + new Col("stamp", 41, 14, TTimestamp) { Format = "yyyyMMddHHmmss" }); + const string Content = + "Ann " + "00042" + " 12.500" + "00012345" + "20231024" + "133005" + "20231024133000" + "\n" + + "Bob " + " -7" + " -0.5" + "-0000001" + "19700101" + "000000" + "19700101000000" + "\n"; + + foreach (var outcome in Ok(Content, layout)) + { + Assert.Equal(2, outcome.RowCount); + Assert.Equal(["Ann", "Bob"], outcome.Column(0)); + Assert.Equal([42L, -7L], outcome.Column(1)); + Assert.Equal([12.5, -0.5], outcome.Column(2)); + Assert.Equal([12345L, -1L], outcome.Column(3)); + Assert.Equal([Days(2023, 10, 24), 0], outcome.Column(4)); + Assert.Equal([new TimeOnly(13, 30, 5).Ticks / 10, 0L], outcome.Column(5)); + Assert.Equal([(new DateTime(2023, 10, 24, 13, 30, 0) - DateTime.UnixEpoch).Ticks / 10, 0L], outcome.Column(6)); + Assert.Equal(2, outcome.Columns[3].Scale); + Assert.Equal(2, outcome.LinesRead); + } + } + + [Fact] + public void Read_ScaledFloat_DividesByPowerOfTen() + { + var layout = Build(new Col("preabe", 0, 13, TFloat64) { Scale = 2 }); + + foreach (var outcome in Ok("0000000123456\n0000000000000\n", layout)) + { + Assert.Equal([1234.56, 0.0], outcome.Column(0)); + } + } + + [Fact] + public void Read_ScaledNegative_KeepsSign() + { + var layout = Build(new Col("v", 0, 11, TFloat64) { Scale = 2 }); + + foreach (var outcome in Ok("-0000012345\n", layout)) + { + Assert.Equal([-123.45], outcome.Column(0)); + } + } + + [Fact] + public void Read_ScaledWithDecimalPoint_ParsesTheText() + { + var layout = Build(new Col("v", 0, 8, TFloat64) { Scale = 2 }, new Col("d", 8, 8, TDecimal) { Scale = 2 }); + + foreach (var outcome in Ok(" 123.45 67.8 \n", layout)) + { + Assert.Equal([123.45], outcome.Column(0)); + Assert.Equal([6780L], outcome.Column(1)); + } + } + + [Fact] + public void Read_DecimalWithTooManyFractionalDigits_IsParseError() + { + var layout = Build(new Col("d", 0, 8, TDecimal) { Scale = 2 }); + + Assert.All(Both(" 1.234\n", layout), o => Assert.Equal(NativeStatus.ParseError, o.Status)); + } + + [Fact] + public void Read_Decimal_KeepsUnscaled() + { + var layout = Build(new Col("voltot", 0, 18, TDecimal) { Scale = 2 }); + + foreach (var outcome in Ok("000000000000123456\n", layout)) + { + Assert.Equal([123456L], outcome.Column(0)); + } + } + + [Fact] + public void Read_Nullable_BlankIsNull() + { + var layout = Build(new Col("a", 0, 4, TInt64) { Nullable = true }, new Col("s", 4, 3, TString) { Nullable = true }); + + foreach (var outcome in Ok("0001abc\n \n0003xyz\n", layout)) + { + Assert.Equal([1L, null, 3L], outcome.Column(0)); + Assert.Equal(["abc", null, "xyz"], outcome.Column(1)); + } + } + + [Fact] + public void Read_NoNulls_HasNoValidity() + { + var layout = Build(new Col("a", 0, 4, TInt64) { Nullable = true }); + + foreach (var outcome in Ok("0001\n0002\n", layout)) + { + Assert.True(outcome.Columns[0].Validity is null); + } + } + + [Fact] + public void Read_NotNullableBlank_IsParseError() + { + var layout = Build(new Col("a", 0, 4, TInt64)); + + foreach (var outcome in Both("0001\n \n", layout)) + { + Assert.Equal(NativeStatus.ParseError, outcome.Status); + Assert.Contains("Line 2", outcome.Error, StringComparison.Ordinal); + } + } + + [Fact] + public void Read_NotNullableBlankString_IsEmpty() + { + var layout = Build(new Col("s", 0, 4, TString), new Col("n", 4, 1, TInt64)); + + foreach (var outcome in Ok(" 1\n", layout)) + { + Assert.Equal([""], outcome.Column(0)); + } + } + + [Fact] + public void Read_HeaderLines_AreNotParsed() + { + var layout = Build(new Col("n", 0, 3, TInt64)); + + foreach (var outcome in Ok("HEADER\nXX\n001\n002\n", layout, Options(headerLines: 2))) + { + Assert.Equal([1L, 2L], outcome.Column(0)); + Assert.Equal(2, outcome.LinesSkipped); + } + } + + [Fact] + public void Read_HeaderLinesBeyondTheFile_ReadNothing() + { + var layout = Build(new Col("n", 0, 3, TInt64)); + + foreach (var outcome in Ok("HDR\n", layout, Options(headerLines: 5))) + { + Assert.Equal(0, outcome.RowCount); + Assert.Equal(1, outcome.LinesSkipped); + } + } + + [Fact] + public void Read_SkipInvalid_SkipsTrailerAndCounts() + { + var layout = Build(new Col("tipreg", 0, 2, TString), new Col("price", 2, 5, TFloat64) { Scale = 2 }); + const string Content = "00COTAHIST\n0100123\n0100456\n99TRAILER\n"; + + foreach (var outcome in Ok(Content, layout, Options(skipInvalid: true))) + { + Assert.Equal(["01", "01"], outcome.Column(0)); + Assert.Equal([1.23, 4.56], outcome.Column(1)); + Assert.Equal(2, outcome.LinesSkipped); + Assert.Equal(2, outcome.LinesRead); + } + } + + [Fact] + public void Read_RejectedLineMidway_LeavesNoPartialValues() + { + var layout = Build(new Col("s", 0, 3, TString), new Col("n", 3, 2, TInt64), new Col("t", 5, 2, TString)); + + foreach (var outcome in Ok("abc01xy\ndefXXzz\nghi03uv\n", layout, Options(skipInvalid: true))) + { + Assert.Equal(["abc", "ghi"], outcome.Column(0)); + Assert.Equal([1L, 3L], outcome.Column(1)); + Assert.Equal(["xy", "uv"], outcome.Column(2)); + } + } + + [Fact] + public void Read_ParseError_ReportsLineAndColumn() + { + var layout = Build(new Col("name", 0, 3, TString), new Col("age", 3, 2, TInt64)); + + foreach (var outcome in Both("Ann30\nBobXX\n", layout)) + { + Assert.Equal(NativeStatus.ParseError, outcome.Status); + Assert.Contains("column 'age'", outcome.Error, StringComparison.Ordinal); + Assert.Contains("\"XX\"", outcome.Error, StringComparison.Ordinal); + NativeApi.LastParseError(out long line, out int column); + Assert.Equal(2, line); + Assert.Equal(1, column); + } + } + + [Fact] + public void Read_ShortLine_IsParseErrorWithColumnMinusOne() + { + var layout = Build(new Col("a", 0, 3, TString), new Col("b", 3, 4, TString)); + + foreach (var outcome in Both("abc\n", layout)) + { + Assert.Equal(NativeStatus.ParseError, outcome.Status); + Assert.Contains("it is 3 bytes long and the layout needs 7", outcome.Error, StringComparison.Ordinal); + NativeApi.LastParseError(out _, out int column); + Assert.Equal(-1, column); + } + } + + [Fact] + public void Read_Latin1Text_IsTranscodedToUtf8() + { + var layout = Build(new Col("name", 0, 6, TString), new Col("n", 6, 2, TInt64)); + var latin1 = Encoding.Latin1; + + foreach (var outcome in Ok("AÇÃO 12\n", layout, Options(encoding: 28591), latin1)) + { + Assert.Equal(["AÇÃO"], outcome.Column(0)); + Assert.Equal([12L], outcome.Column(1)); + } + } + + [Fact] + public void Read_EmptyFile_ReturnsZeroRows() + { + var layout = Build(new Col("a", 0, 3, TString)); + + foreach (var outcome in Ok("", layout)) + { + Assert.Equal(0, outcome.RowCount); + Assert.Equal(1, outcome.Columns.Length); + } + } + + [Fact] + public void Read_NullOptions_UsesDefaults() + { + var layout = Build(new Col("a", 0, 3, TString)); + + foreach (var outcome in Ok("abc\r\ndef", layout)) + { + Assert.Equal(["abc", "def"], outcome.Column(0)); + } + } + + [Fact] + public void Read_BadOptionsStructSize_IsInvalidArgument() + { + var layout = Build(new Col("a", 0, 3, TString)); + + var outcome = FromMemory("abc"u8.ToArray(), layout, Options() with { StructSize = 12 }); + + Assert.Equal(NativeStatus.InvalidArgument, outcome.Status); + Assert.Contains("fw_read_options struct_size is 12", outcome.Error, StringComparison.Ordinal); + } + + [Fact] + public void Read_MissingFile_IsError() + { + var layout = Build(new Col("a", 0, 3, TString)); + byte[] path = Encoding.UTF8.GetBytes(Path.Combine(Path.GetTempPath(), "fwp-missing-" + Guid.NewGuid())); + + unsafe + { + NativeTableRaw table = default; + Assert.Equal(NativeStatus.Error, ReadApi.ReadFile(path, layout, null, &table, null)); + } + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs b/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs index 153c91f..a8a988b 100644 --- a/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs +++ b/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs @@ -60,7 +60,7 @@ internal static NativeLayout Build(params Col[] columns) return layout!; } - internal static NativeLayoutOptionsRaw Options(int lineLength = 0, int filler = 0) + internal static NativeLayoutOptionsRaw LayoutOptions(int lineLength = 0, int filler = 0) { return new NativeLayoutOptionsRaw { StructSize = sizeof(NativeLayoutOptionsRaw), LineLength = lineLength, Filler = filler }; } diff --git a/tests/FixedWidthParser.Tests/Native/NativeTestReads.cs b/tests/FixedWidthParser.Tests/Native/NativeTestReads.cs new file mode 100644 index 0000000..4f67c11 --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeTestReads.cs @@ -0,0 +1,135 @@ +using System.Runtime.InteropServices; +using System.Text; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Tests.Native +{ + /// A published fw_table copied into managed arrays, then freed. + internal sealed class ReadOutcome + { + public int Status { get; init; } + public long RowCount { get; init; } + public NativeColumnRaw[] Columns { get; init; } = []; + public List Values { get; init; } = []; + public long LinesRead { get; init; } + public long LinesSkipped { get; init; } + public string Error { get; init; } = ""; + + public object?[] Column(int index) + { + return Values[index]; + } + } + + internal static unsafe class NativeTestReads + { + internal static NativeReadOptionsRaw Options(long headerLines = 0, bool skipInvalid = false, int degreeOfParallelism = 1, int encoding = 0) + { + return new NativeReadOptionsRaw + { + StructSize = sizeof(NativeReadOptionsRaw), + HeaderLines = headerLines, + SkipInvalid = skipInvalid ? 1 : 0, + DegreeOfParallelism = degreeOfParallelism, + Encoding = encoding, + }; + } + + internal static ReadOutcome FromMemory(byte[] content, NativeLayout layout, NativeReadOptionsRaw? options = null) + { + NativeReadOptionsRaw value = options ?? default; + NativeTableRaw table = default; + var result = new NativeReadResultRaw { StructSize = sizeof(NativeReadResultRaw) }; + fixed (byte* data = content) + { + int status = ReadApi.ReadMemory(data, content.Length, layout, options is null ? null : &value, &table, &result); + return Collect(status, &table, result); + } + } + + internal static ReadOutcome FromFile(byte[] content, NativeLayout layout, NativeReadOptionsRaw? options = null) + { + string path = Path.GetTempFileName(); + try + { + File.WriteAllBytes(path, content); + NativeReadOptionsRaw value = options ?? default; + NativeTableRaw table = default; + var result = new NativeReadResultRaw { StructSize = sizeof(NativeReadResultRaw) }; + int status = ReadApi.ReadFile(Encoding.UTF8.GetBytes(path), layout, options is null ? null : &value, &table, &result); + return Collect(status, &table, result); + } + finally + { + File.Delete(path); + } + } + + internal static ReadOutcome[] Both(string content, NativeLayout layout, NativeReadOptionsRaw? options = null, Encoding? encoding = null) + { + byte[] bytes = (encoding ?? Encoding.UTF8).GetBytes(content); + return [FromMemory(bytes, layout, options), FromFile(bytes, layout, options)]; + } + + internal static ReadOutcome Collect(int status, NativeTableRaw* table, NativeReadResultRaw result) + { + try + { + return new ReadOutcome + { + Status = status, + RowCount = table->RowCount, + Columns = [.. Enumerable.Range(0, table->ColumnCount).Select(i => Column(table, i))], + Values = [.. Enumerable.Range(0, table->ColumnCount).Select(i => Copy(Column(table, i)))], + LinesRead = result.LinesRead, + LinesSkipped = result.LinesSkipped, + Error = status == NativeStatus.Ok ? "" : NativeTestLayouts.LastError(), + }; + } + finally + { + NativeTable.Free(table); + } + } + + private static NativeColumnRaw Column(NativeTableRaw* table, int index) + { + return table->Columns[index]; + } + + internal static object?[] Copy(NativeColumnRaw column) + { + var values = new object?[column.Length]; + for (int row = 0; row < column.Length; row++) + { + values[row] = IsNull(column, row) ? null : Value(column, row); + } + return values; + } + + private static bool IsNull(NativeColumnRaw column, int row) + { + return column.Validity is not null && (column.Validity[row >> 3] & (1 << (row & 7))) == 0; + } + + private static string Text(NativeColumnRaw column, int row) + { + int* offsets = (int*)column.Values; + int length = offsets[row + 1] - offsets[row]; + return length == 0 ? "" : Encoding.UTF8.GetString(column.Data + offsets[row], length); + } + + private static object Value(NativeColumnRaw column, int row) + { + return column.Type switch + { + NativeType.String => Text(column, row), + NativeType.Float64 => ((double*)column.Values)[row], + NativeType.Date => ((int*)column.Values)[row], + _ => ((long*)column.Values)[row], + }; + } + } +} From 903894d9058a1a3fcdc159c57ebd259a44c03c2e Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 01:09:40 -0300 Subject: [PATCH 37/85] feat(native): read in parallel by byte range degree_of_parallelism now splits fw_read_file and fw_read_memory into byte-range partitions. Each partition fills its own column builders, a merge adopts the following partition without copying, and publishing concatenates every column once, in file order, so the table and the error line numbers equal a sequential read. --- .../GrowableNativeBuffer.cs | 2 + .../Reading/ColumnBuilder.cs | 116 ++++++++++++++++++ .../Reading/ReadApi.cs | 15 +-- .../Reading/ReadSettings.cs | 3 +- .../Reading/TableSink.cs | 31 ++++- .../FixedWidthParser.Tests.csproj | 2 +- .../Native/NativeParallelReadTests.cs | 96 +++++++++++++++ .../Native/NativeReadTests.cs | 2 +- .../Native/NativeTestReads.cs | 4 +- 9 files changed, 253 insertions(+), 18 deletions(-) create mode 100644 tests/FixedWidthParser.Tests/Native/NativeParallelReadTests.cs diff --git a/src/FixedWidthParser.Native/GrowableNativeBuffer.cs b/src/FixedWidthParser.Native/GrowableNativeBuffer.cs index 0ad1d09..5d93c58 100644 --- a/src/FixedWidthParser.Native/GrowableNativeBuffer.cs +++ b/src/FixedWidthParser.Native/GrowableNativeBuffer.cs @@ -24,6 +24,8 @@ internal sealed unsafe class GrowableNativeBuffer : IDisposable where T : unm internal ref T this[int index] => ref _data[index]; + internal ReadOnlySpan Span => new(_data, Count); + /// Drops every element from on. internal void Truncate(int count) { diff --git a/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs index 7c6238f..9d05ff6 100644 --- a/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs +++ b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs @@ -1,5 +1,6 @@ using System.Buffers; using System.Globalization; +using System.Runtime.InteropServices; using System.Text; using FixedWidthParser.Infrastructure; using FixedWidthParser.Native.Layouts; @@ -92,6 +93,121 @@ internal void Publish(NativeColumnRaw* target) }); } + /// Publishes the same column of several partitions, in order, as one column: the one copy a partitioned read makes. + internal static void Concat(ReadOnlySpan parts, NativeColumnRaw* target) + { + if (parts.Length == 1) + { + parts[0].Publish(target); + return; + } + long rows = 0; + foreach (ref readonly var part in parts) + { + rows += part.Rows; + } + var column = parts[0]._column; + target->Type = column.Type; + target->Scale = column.Scale; + target->Length = rows; + target->Validity = ConcatValidity(parts, rows); + target->Data = null; + target->DataLen = 0; + if (column.Type == NativeType.String) + { + ConcatStrings(parts, rows, target); + return; + } + target->Values = ConcatValues(parts); + } + + private ReadOnlySpan ValueBytes => _column.Type switch + { + NativeType.Float64 => MemoryMarshal.AsBytes(_doubles.Span), + NativeType.Date => MemoryMarshal.AsBytes(_ints.Span), + _ => MemoryMarshal.AsBytes(_longs.Span), + }; + + private static void* ConcatValues(ReadOnlySpan parts) + { + long total = 0; + foreach (ref readonly var part in parts) + { + total += part.ValueBytes.Length; + } + byte* values = (byte*)Marshal.AllocHGlobal((nint)total); + long at = 0; + foreach (ref readonly var part in parts) + { + var bytes = part.ValueBytes; + bytes.CopyTo(new Span(values + at, bytes.Length)); + at += bytes.Length; + } + return values; + } + + private static void ConcatStrings(ReadOnlySpan parts, long rows, NativeColumnRaw* target) + { + long total = 0; + foreach (ref readonly var part in parts) + { + total += part._bytes.Count; + } + if (total > int.MaxValue) + { + throw new InvalidOperationException( + $"Column '{parts[0]._column.Name}' holds more than 2 GiB of text; read in batches (fw_reader_open_*)."); + } + int* offsets = (int*)Marshal.AllocHGlobal((nint)((rows + 1) * sizeof(int))); + byte* data = (byte*)Marshal.AllocHGlobal((nint)total); + offsets[0] = 0; + long row = 1; + int baseOffset = 0; + foreach (ref readonly var part in parts) + { + var partOffsets = part._ints.Span; + foreach (ref readonly int offset in partOffsets[1..]) + { + offsets[row++] = baseOffset + offset; + } + part._bytes.Span.CopyTo(new Span(data + baseOffset, part._bytes.Count)); + baseOffset += part._bytes.Count; + } + target->Values = offsets; + target->Data = data; + target->DataLen = total; + } + + private static byte* ConcatValidity(ReadOnlySpan parts, long rows) + { + bool any = false; + foreach (ref readonly var part in parts) + { + any |= part._validity is not null; + } + if (!any) + { + return null; + } + long length = (rows + 7) / 8; + byte* bits = (byte*)Marshal.AllocHGlobal((nint)length); + new Span(bits, (int)length).Fill(0xFF); + long row = 0; + foreach (ref readonly var part in parts) + { + var partBits = part._validity is null ? default : part._validity.Span; + for (int r = 0; r < part.Rows && !partBits.IsEmpty; r++) + { + if ((partBits[r >> 3] & (1 << (r & 7))) == 0) + { + bits[(row + r) >> 3] &= (byte)~(1 << (int)((row + r) & 7)); + } + } + row += part.Rows; + } + return bits; + } + public void Dispose() { _longs.Dispose(); diff --git a/src/FixedWidthParser.Native/Reading/ReadApi.cs b/src/FixedWidthParser.Native/Reading/ReadApi.cs index d1ceaf0..aefa7b2 100644 --- a/src/FixedWidthParser.Native/Reading/ReadApi.cs +++ b/src/FixedWidthParser.Native/Reading/ReadApi.cs @@ -9,12 +9,12 @@ namespace FixedWidthParser.Native.Reading /// fw_read_file and fw_read_memory: a whole source into one columnar table. internal static unsafe class ReadApi { - internal static int ReadFile(ReadOnlySpan path, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult) + internal static int ReadFile(ReadOnlySpan path, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult, int chunkSize = 0) { try { using var handle = File.OpenHandle(Encoding.UTF8.GetString(path), FileMode.Open, FileAccess.Read, FileShare.Read); - return Read(new FileSource(handle), layout, options, outTable, outResult); + return Read(new FileSource(handle), layout, options, outTable, outResult, chunkSize); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException) { @@ -23,12 +23,13 @@ internal static int ReadFile(ReadOnlySpan path, NativeLayout layout, Nativ } } - internal static int ReadMemory(byte* data, long length, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult) + // chunkSize: bytes per partition, 0 sizes them from the source; tests set it to cut every boundary. + internal static int ReadMemory(byte* data, long length, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult, int chunkSize = 0) { - return Read(new PointerSource(data, length), layout, options, outTable, outResult); + return Read(new PointerSource(data, length), layout, options, outTable, outResult, chunkSize); } - private static int Read(IRandomAccessSource source, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult) + private static int Read(IRandomAccessSource source, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult, int chunkSize) { *outTable = default; int status = CheckResult(outResult); @@ -45,13 +46,13 @@ private static int Read(IRandomAccessSource source, NativeLayout layout, NativeR { long dataStart = HeaderEnd(source, settings!.HeaderLines, out long headerLines); var skipped = new StrongBox(headerLines); - var readerOptions = settings.ReaderOptions(skipped, degreeOfParallelism: 1); + var readerOptions = settings.ReaderOptions(skipped, settings.DegreeOfParallelism, chunkSize); #pragma warning disable VSTHRD002 // An export is synchronous and runs with no synchronization context. using var table = ParallelAggregator.RunAsync(source, dataStart, () => new TableSink(layout), readerOptions, CancellationToken.None) .GetAwaiter().GetResult(); #pragma warning restore VSTHRD002 table.Publish(outTable); - WriteResult(outResult, table.Rows, skipped.Value); + WriteResult(outResult, table.TotalRows, skipped.Value); return NativeStatus.Ok; } catch (FixedWidthParseException ex) diff --git a/src/FixedWidthParser.Native/Reading/ReadSettings.cs b/src/FixedWidthParser.Native/Reading/ReadSettings.cs index bdb8b27..29501a0 100644 --- a/src/FixedWidthParser.Native/Reading/ReadSettings.cs +++ b/src/FixedWidthParser.Native/Reading/ReadSettings.cs @@ -50,7 +50,7 @@ internal static unsafe int TryCreate(NativeReadOptionsRaw* options, out ReadSett } /// The core's options; a skipped line is counted in . - internal FixedWidthReaderOptions ReaderOptions(StrongBox skipped, int degreeOfParallelism) + internal FixedWidthReaderOptions ReaderOptions(StrongBox skipped, int degreeOfParallelism, int chunkSize) { return FixedWidthReaderOptions.Validate(new FixedWidthReaderOptions { @@ -58,6 +58,7 @@ internal FixedWidthReaderOptions ReaderOptions(StrongBox skipped, int degr Encoding = Encoding, MaxLineLength = MaxLineLength, DegreeOfParallelism = degreeOfParallelism, + ChunkSize = chunkSize, ByteOrderMark = ByteOrderMark, OnParseError = SkipInvalid ? _ => Interlocked.Increment(ref skipped.Value) > 0 : null, }); diff --git a/src/FixedWidthParser.Native/Reading/TableSink.cs b/src/FixedWidthParser.Native/Reading/TableSink.cs index 6b0555c..18b6b7f 100644 --- a/src/FixedWidthParser.Native/Reading/TableSink.cs +++ b/src/FixedWidthParser.Native/Reading/TableSink.cs @@ -11,6 +11,7 @@ internal struct TableSink : ILineSink, IDisposable private readonly ColumnBuilder[] _columns; private readonly int _requiredLength; private int _failedColumn; + private List? _following; internal TableSink(NativeLayout layout) { @@ -18,11 +19,16 @@ internal TableSink(NativeLayout layout) _columns = [.. layout.Columns.Select(c => new ColumnBuilder(c))]; _requiredLength = layout.Columns.Max(c => c.Start + c.Length); _failedColumn = -1; + _following = null; Rows = 0; } + /// The rows of this partition only. internal int Rows { get; private set; } + /// The rows of this partition and every partition merged into it. + internal readonly long TotalRows => Rows + (_following?.Sum(f => (long)f.Rows) ?? 0); + public bool TryAdd(ReadOnlySpan line, in FixedWidthContext context) { if (line.Length < _requiredLength) @@ -58,29 +64,42 @@ public readonly FixedWidthParseException Diagnose(ReadOnlySpan line, long lineNumber, column.Name, column.Start, column.Length, raw); } - public readonly void Merge(TableSink following) + /// Adopts the following partition without copying; concatenates once. + public void Merge(TableSink following) { - throw new NotSupportedException("Partitioned reads arrive in a later step."); + (_following ??= []).Add(following); } + /// This partition's columns, then every merged partition's, in file order. + internal readonly IEnumerable Chunks => _following is null ? [_columns] : [_columns, .. _following.Select(f => f._columns)]; + /// Hands every column's buffers to ; the caller frees them with . internal readonly unsafe void Publish(NativeTableRaw* table) { + var chunks = Chunks.ToArray(); var columns = (NativeColumnRaw*)Marshal.AllocHGlobal(_columns.Length * sizeof(NativeColumnRaw)); + var parts = new ColumnBuilder[chunks.Length]; for (int i = 0; i < _columns.Length; i++) { - _columns[i].Publish(columns + i); + for (int c = 0; c < chunks.Length; c++) + { + parts[c] = chunks[c][i]; + } + ColumnBuilder.Concat(parts, columns + i); } table->ColumnCount = _columns.Length; - table->RowCount = Rows; + table->RowCount = TotalRows; table->Columns = columns; } public readonly void Dispose() { - foreach (var column in _columns) + foreach (var chunk in Chunks) { - column.Dispose(); + foreach (var column in chunk) + { + column.Dispose(); + } } } diff --git a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj index 5b3fa45..b7aed9a 100644 --- a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj +++ b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj @@ -12,7 +12,7 @@ Exe true - $(NoWarn);HLQ005;CA1707;CA1815;S3241;CA1051 + $(NoWarn);HLQ005;CA1707;CA1815;S3241;CA1051;S6640 diff --git a/tests/FixedWidthParser.Tests/Native/NativeParallelReadTests.cs b/tests/FixedWidthParser.Tests/Native/NativeParallelReadTests.cs new file mode 100644 index 0000000..125cfa2 --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeParallelReadTests.cs @@ -0,0 +1,96 @@ +using System.Text; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Layouts; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; +using static FixedWidthParser.Tests.Native.NativeTestReads; + +namespace FixedWidthParser.Tests.Native +{ + public sealed class NativeParallelReadTests + { + private static NativeLayout Layout() + { + return Build( + new Col("code", 0, 4, TString), + new Col("qty", 4, 4, TInt64) { Nullable = true }, + new Col("price", 8, 6, TFloat64) { Scale = 2 }); + } + + // A BOM, CRLF and LF breaks, empty lines, invalid lines, nulls and no final line break. + private static byte[] Content(bool withBom) + { + var text = new StringBuilder(withBom ? "" : ""); + text.Append("HEADER ONE\r\nHEADER TWO\n"); + for (int i = 1; i <= 200; i++) + { + string qty = i % 7 == 0 ? " " : (i % 10000).ToString("D4", System.Globalization.CultureInfo.InvariantCulture); + string price = i % 13 == 0 ? "XXXXXX" : (i * 3).ToString("D6", System.Globalization.CultureInfo.InvariantCulture); + text.Append(i % 3 == 0 ? "CD" : "AB").Append(System.Globalization.CultureInfo.InvariantCulture, $"{i % 100:D2}").Append(qty).Append(price); + text.Append(i % 2 == 0 ? "\r\n" : "\n"); + if (i % 17 == 0) + { + text.Append('\n'); + } + } + text.Append("LAST0001000123"); + return Encoding.UTF8.GetBytes(text.ToString()); + } + + public static TheoryData Partitions() + { + var data = new TheoryData(); + foreach (int chunkSize in new[] { 1, 7, 64, 4096 }) + { + foreach (int degree in new[] { 1, 4 }) + { + data.Add(chunkSize, degree, false); + data.Add(chunkSize, degree, true); + } + } + return data; + } + + [Theory] + [MemberData(nameof(Partitions))] + public void ParallelRead_EqualsSequential(int chunkSize, int degree, bool withBom) + { + var layout = Layout(); + byte[] content = Content(withBom); + + var sequential = FromMemory(content, layout, Options(headerLines: 2, skipInvalid: true, degreeOfParallelism: 1)); + var parallel = FromMemory(content, layout, Options(headerLines: 2, skipInvalid: true, degreeOfParallelism: degree), chunkSize); + + Assert.Equal(NativeStatus.Ok, sequential.Status); + Assert.True(parallel.Status == NativeStatus.Ok, parallel.Error); + Assert.Equal(sequential.RowCount, parallel.RowCount); + for (int column = 0; column < 3; column++) + { + Assert.Equal(sequential.Column(column), parallel.Column(column)); + } + Assert.Equal(sequential.LinesRead, parallel.LinesRead); + Assert.Equal(sequential.LinesSkipped, parallel.LinesSkipped); + Assert.Equal(200 - (200 / 13) + 1, parallel.RowCount); + } + + [Theory] + [InlineData(1)] + [InlineData(7)] + [InlineData(64)] + public void ParallelRead_ErrorLineNumber_EqualsSequential(int chunkSize) + { + var layout = Layout(); + byte[] content = Content(withBom: true); + + var sequential = FromMemory(content, layout, Options(headerLines: 2, degreeOfParallelism: 1)); + NativeApi.LastParseError(out long sequentialLine, out int sequentialColumn); + var parallel = FromMemory(content, layout, Options(headerLines: 2, degreeOfParallelism: 4), chunkSize); + NativeApi.LastParseError(out long parallelLine, out int parallelColumn); + + Assert.Equal(NativeStatus.ParseError, sequential.Status); + Assert.Equal(NativeStatus.ParseError, parallel.Status); + Assert.Equal(sequentialLine, parallelLine); + Assert.Equal(sequentialColumn, parallelColumn); + Assert.Equal(2, parallelColumn); + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeReadTests.cs b/tests/FixedWidthParser.Tests/Native/NativeReadTests.cs index a3e7f61..c262a8e 100644 --- a/tests/FixedWidthParser.Tests/Native/NativeReadTests.cs +++ b/tests/FixedWidthParser.Tests/Native/NativeReadTests.cs @@ -50,7 +50,7 @@ public void Read_EveryType_ProducesColumns() Assert.Equal([12345L, -1L], outcome.Column(3)); Assert.Equal([Days(2023, 10, 24), 0], outcome.Column(4)); Assert.Equal([new TimeOnly(13, 30, 5).Ticks / 10, 0L], outcome.Column(5)); - Assert.Equal([(new DateTime(2023, 10, 24, 13, 30, 0) - DateTime.UnixEpoch).Ticks / 10, 0L], outcome.Column(6)); + Assert.Equal([(new DateTime(2023, 10, 24, 13, 30, 0, DateTimeKind.Utc) - DateTime.UnixEpoch).Ticks / 10, 0L], outcome.Column(6)); Assert.Equal(2, outcome.Columns[3].Scale); Assert.Equal(2, outcome.LinesRead); } diff --git a/tests/FixedWidthParser.Tests/Native/NativeTestReads.cs b/tests/FixedWidthParser.Tests/Native/NativeTestReads.cs index 4f67c11..75d2ab3 100644 --- a/tests/FixedWidthParser.Tests/Native/NativeTestReads.cs +++ b/tests/FixedWidthParser.Tests/Native/NativeTestReads.cs @@ -37,14 +37,14 @@ internal static NativeReadOptionsRaw Options(long headerLines = 0, bool skipInva }; } - internal static ReadOutcome FromMemory(byte[] content, NativeLayout layout, NativeReadOptionsRaw? options = null) + internal static ReadOutcome FromMemory(byte[] content, NativeLayout layout, NativeReadOptionsRaw? options = null, int chunkSize = 0) { NativeReadOptionsRaw value = options ?? default; NativeTableRaw table = default; var result = new NativeReadResultRaw { StructSize = sizeof(NativeReadResultRaw) }; fixed (byte* data = content) { - int status = ReadApi.ReadMemory(data, content.Length, layout, options is null ? null : &value, &table, &result); + int status = ReadApi.ReadMemory(data, content.Length, layout, options is null ? null : &value, &table, &result, chunkSize); return Collect(status, &table, result); } } From e1aee57382aa814261b3228130ae178604e20558 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 01:13:29 -0300 Subject: [PATCH 38/85] feat(native): read in batches fw_reader_open_file and fw_reader_open_memory open a sequential reader that fw_reader_next hands out as tables of at most batch_rows rows, returning FW_EOF at the end and after a failure; fw_reader_close releases it. fw_reader_options embeds fw_read_options; header lines, skipping and error line numbers behave as in a whole read. --- src/FixedWidthParser.Native/Exports.Reader.cs | 63 +++++++ .../Layouts/NativeLayout.cs | 6 + .../Reading/NativeTableReader.cs | 112 ++++++++++++ .../Reading/ReadApi.cs | 2 +- .../Reading/ReaderApi.cs | 103 ++++++++++++ .../Native/NativeReaderTests.cs | 159 ++++++++++++++++++ 6 files changed, 444 insertions(+), 1 deletion(-) create mode 100644 src/FixedWidthParser.Native/Exports.Reader.cs create mode 100644 src/FixedWidthParser.Native/Reading/NativeTableReader.cs create mode 100644 src/FixedWidthParser.Native/Reading/ReaderApi.cs create mode 100644 tests/FixedWidthParser.Tests/Native/NativeReaderTests.cs diff --git a/src/FixedWidthParser.Native/Exports.Reader.cs b/src/FixedWidthParser.Native/Exports.Reader.cs new file mode 100644 index 0000000..92fdfd3 --- /dev/null +++ b/src/FixedWidthParser.Native/Exports.Reader.cs @@ -0,0 +1,63 @@ +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native +{ + internal static unsafe partial class Exports + { + [UnmanagedCallersOnly(EntryPoint = "fw_reader_open_file")] + public static int ReaderOpenFile(byte* path, int pathLength, nint layout, NativeReaderOptionsRaw* options, nint* outReader) + { + if (path is null || pathLength <= 0 || outReader is null) + { + return InvalidArgument("fw_reader_open_file needs a path and out_reader."); + } + *outReader = 0; + return WithLayout(layout, l => + { + int status = ReaderApi.OpenFile(new ReadOnlySpan(path, pathLength), l, options, out nint reader); + *outReader = reader; + return status; + }); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_reader_open_memory")] + public static int ReaderOpenMemory(byte* data, long dataLength, nint layout, NativeReaderOptionsRaw* options, nint* outReader) + { + if ((data is null && dataLength != 0) || dataLength < 0 || outReader is null) + { + return InvalidArgument("fw_reader_open_memory needs data for a non-zero data_len, and out_reader."); + } + *outReader = 0; + return WithLayout(layout, l => + { + int status = ReaderApi.OpenMemory(data, dataLength, l, options, out nint reader); + *outReader = reader; + return status; + }); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_reader_next")] + public static int ReaderNext(nint reader, NativeTableRaw* outTable) + { + if (outTable is null) + { + return InvalidArgument("fw_reader_next needs out_table."); + } + return ReaderApi.Next(reader, outTable); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_reader_close")] + public static void ReaderClose(nint reader) + { + try + { + ReaderApi.Close(reader); + } + catch (Exception ex) + { + NativeApi.SetLastError(ex.Message); + } + } + } +} diff --git a/src/FixedWidthParser.Native/Layouts/NativeLayout.cs b/src/FixedWidthParser.Native/Layouts/NativeLayout.cs index 41f4703..fdf0bb6 100644 --- a/src/FixedWidthParser.Native/Layouts/NativeLayout.cs +++ b/src/FixedWidthParser.Native/Layouts/NativeLayout.cs @@ -36,6 +36,12 @@ private NativeLayout(NativeColumn[] columns, int lineLength, byte filler) internal int LineLength { get; } internal byte Filler { get; } + /// The index of the first column named , or -1. + internal int IndexOf(string? name) + { + return Array.FindIndex(Columns, c => string.Equals(c.Name, name, StringComparison.Ordinal)); + } + internal static unsafe int TryCreate(ReadOnlySpan specs, NativeLayoutOptionsRaw* options, out NativeLayout? layout) { layout = null; diff --git a/src/FixedWidthParser.Native/Reading/NativeTableReader.cs b/src/FixedWidthParser.Native/Reading/NativeTableReader.cs new file mode 100644 index 0000000..a6e0c40 --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/NativeTableReader.cs @@ -0,0 +1,112 @@ +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Readers; + +namespace FixedWidthParser.Native.Reading +{ + /// fw_reader_options. + [StructLayout(LayoutKind.Sequential)] + internal struct NativeReaderOptionsRaw + { + public int StructSize; + public NativeReadOptionsRaw Read; + public long BatchRows; + } + + /// + /// A sequential read handed out a table of at most batch_rows rows at a time, behind + /// fw_reader_open_*. Used by one thread at a time; it keeps its layout alive after + /// fw_layout_free. + /// + internal sealed unsafe class NativeTableReader : IDisposable + { + private const long DefaultBatchRows = 65_536; + + private readonly Stream _stream; + private readonly NativeLayout _layout; + private readonly FixedWidthReaderOptions _options; + private readonly FixedWidthContext _context; + private readonly long _batchRows; + private LineBufferState _lines; + private bool _done; + + internal NativeTableReader(Stream stream, long dataStart, long headerLines, NativeLayout layout, ReadSettings settings, long batchRows) + { + _stream = stream; + _stream.Position = dataStart; + _layout = layout; + _options = settings.ReaderOptions(new StrongBox(), degreeOfParallelism: 1, chunkSize: 0); + _context = _options.Context; + _batchRows = batchRows == 0 ? DefaultBatchRows : batchRows; + _lines = default; + _lines.Rent(_options, dataStart == 0 && _options.SkipsByteOrderMark); + _lines.LineNumber = headerLines; + } + + /// Publishes the next batch into , or returns ; a failure ends the read. + internal int Next(NativeTableRaw* table) + { + if (_done) + { + return NativeStatus.Eof; + } + var sink = new TableSink(_layout); + try + { +#pragma warning disable IDISP003 // Fill appends to the sink; it never replaces it. + Fill(ref sink); +#pragma warning restore IDISP003 + if (sink.Rows == 0) + { + return NativeStatus.Eof; + } + sink.Publish(table); + return NativeStatus.Ok; + } + catch + { + _done = true; + throw; + } + finally + { + sink.Dispose(); + } + } + + private void Fill(ref TableSink sink) + { + while (sink.Rows < _batchRows) + { + var status = _lines.TryGetLine(out var line); + if (status == LineStatus.End) + { + _done = true; + return; + } + if (status == LineStatus.NeedData) + { + _lines.Compact(); + _lines.GrowIfFull(); + _lines.Advance(_stream.Read(_lines.Buffer.AsSpan(_lines.End))); + continue; + } + if (!sink.TryAdd(line, in _context)) + { + ParseErrors.Reject(sink.Diagnose(line, _lines.LineNumber, in _context), _options.OnParseError); + } + } + } + + internal NativeLayout Layout => _layout; + + public void Dispose() + { + _lines.Return(); +#pragma warning disable IDISP007 // The reader owns the stream it was opened over. + _stream.Dispose(); +#pragma warning restore IDISP007 + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/ReadApi.cs b/src/FixedWidthParser.Native/Reading/ReadApi.cs index aefa7b2..a07fd33 100644 --- a/src/FixedWidthParser.Native/Reading/ReadApi.cs +++ b/src/FixedWidthParser.Native/Reading/ReadApi.cs @@ -57,7 +57,7 @@ private static int Read(IRandomAccessSource source, NativeLayout layout, NativeR } catch (FixedWidthParseException ex) { - NativeApi.SetParseError(ex, Array.FindIndex(layout.Columns, c => string.Equals(c.Name, ex.ColumnName, StringComparison.Ordinal))); + NativeApi.SetParseError(ex, layout.IndexOf(ex.ColumnName)); return NativeStatus.ParseError; } catch (Exception ex) diff --git a/src/FixedWidthParser.Native/Reading/ReaderApi.cs b/src/FixedWidthParser.Native/Reading/ReaderApi.cs new file mode 100644 index 0000000..422ee55 --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/ReaderApi.cs @@ -0,0 +1,103 @@ +using System.Text; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Readers; + +namespace FixedWidthParser.Native.Reading +{ + /// fw_reader_*: batches of a sequential read, behind a handle. + internal static unsafe class ReaderApi + { + internal static int OpenFile(ReadOnlySpan path, NativeLayout layout, NativeReaderOptionsRaw* options, out nint reader) + { + reader = 0; + int status = TrySettings(options, out var settings, out long batchRows); + if (status != NativeStatus.Ok) + { + return status; + } + FileStream? stream = null; + try + { + stream = new FileStream(Encoding.UTF8.GetString(path), FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1, FileOptions.SequentialScan); + long dataStart = ReadApi.HeaderEnd(new FileSource(stream.SafeFileHandle), settings!.HeaderLines, out long headerLines); + reader = NativeHandleTable.Register(new NativeTableReader(stream, dataStart, headerLines, layout, settings, batchRows)); + return NativeStatus.Ok; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException) + { + stream?.Dispose(); + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + + /// Reads in place: the caller keeps it alive until the reader is closed. + internal static int OpenMemory(byte* data, long length, NativeLayout layout, NativeReaderOptionsRaw* options, out nint reader) + { + reader = 0; + int status = TrySettings(options, out var settings, out long batchRows); + if (status != NativeStatus.Ok) + { + return status; + } + long dataStart = ReadApi.HeaderEnd(new PointerSource(data, length), settings!.HeaderLines, out long headerLines); +#pragma warning disable CA2000 // The reader owns and disposes the stream. + var stream = new UnmanagedMemoryStream(data, length); +#pragma warning restore CA2000 + reader = NativeHandleTable.Register(new NativeTableReader(stream, dataStart, headerLines, layout, settings, batchRows)); + return NativeStatus.Ok; + } + + internal static int Next(nint handle, NativeTableRaw* table) + { + *table = default; + var reader = NativeHandleTable.Resolve(handle); + if (reader is null) + { + NativeApi.SetLastError("The reader handle is not a live fw_reader."); + return NativeStatus.InvalidHandle; + } + try + { + return reader.Next(table); + } + catch (FixedWidthParseException ex) + { + NativeApi.SetParseError(ex, reader.Layout.IndexOf(ex.ColumnName)); + return NativeStatus.ParseError; + } + catch (Exception ex) + { + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + + internal static void Close(nint handle) + { + if (NativeHandleTable.TryUnregister(handle, out var reader)) + { + reader!.Dispose(); + } + } + + private static int TrySettings(NativeReaderOptionsRaw* options, out ReadSettings? settings, out long batchRows) + { + settings = null; + batchRows = 0; + int status = NativeOptions.TryRead(options, "fw_reader_options", out var raw); + if (status != NativeStatus.Ok) + { + return status; + } + if (raw.BatchRows < 0 || (raw.Read.StructSize != 0 && raw.Read.StructSize != sizeof(NativeReadOptionsRaw))) + { + NativeApi.SetLastError("fw_reader_options: batch_rows must not be negative, and read.struct_size must be 0 or sizeof(fw_read_options)."); + return NativeStatus.InvalidArgument; + } + batchRows = raw.BatchRows; + var read = raw.Read with { StructSize = sizeof(NativeReadOptionsRaw) }; + return ReadSettings.TryCreate(&read, out settings); + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeReaderTests.cs b/tests/FixedWidthParser.Tests/Native/NativeReaderTests.cs new file mode 100644 index 0000000..ae1ddd6 --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeReaderTests.cs @@ -0,0 +1,159 @@ +using System.Text; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; +using static FixedWidthParser.Tests.Native.NativeTestReads; + +namespace FixedWidthParser.Tests.Native +{ + public sealed unsafe class NativeReaderTests + { + private static readonly byte[] Content = Encoding.UTF8.GetBytes( + "HEADER\n" + string.Concat(Enumerable.Range(1, 10).Select(i => i == 5 ? "ZZXX\r\n" : $"AB{i:D2}\n")) + "AB11"); + + private static NativeLayout Layout() + { + return Build(new Col("code", 0, 2, TString), new Col("n", 2, 2, TInt64)); + } + + private static NativeReaderOptionsRaw ReaderOptions(long batchRows, bool skipInvalid = true) + { + return new NativeReaderOptionsRaw + { + StructSize = sizeof(NativeReaderOptionsRaw), + Read = Options(headerLines: 1, skipInvalid: skipInvalid), + BatchRows = batchRows, + }; + } + + private static nint Open(NativeLayout layout, NativeReaderOptionsRaw options, byte* data) + { + Assert.Equal(NativeStatus.Ok, ReaderApi.OpenMemory(data, Content.Length, layout, &options, out nint reader)); + return reader; + } + + private static List Drain(nint reader) + { + var batches = new List(); + while (true) + { + NativeTableRaw table = default; + int status = ReaderApi.Next(reader, &table); + if (status == NativeStatus.Eof) + { + return batches; + } + Assert.True(status == NativeStatus.Ok, LastError()); + batches.Add(Collect(status, &table, default)); + } + } + + [Theory] + [InlineData(1)] + [InlineData(3)] + [InlineData(1000)] + public void Reader_BatchesConcatenate_ToTheWholeRead(long batchRows) + { + var layout = Layout(); + var whole = FromMemory(Content, layout, Options(headerLines: 1, skipInvalid: true)); + + fixed (byte* data = Content) + { + nint reader = Open(layout, ReaderOptions(batchRows), data); + var batches = Drain(reader); + ReaderApi.Close(reader); + + Assert.All(batches, b => Assert.InRange(b.RowCount, 1, batchRows)); + Assert.Equal(whole.Column(0), batches.SelectMany(b => b.Column(0))); + Assert.Equal(whole.Column(1), batches.SelectMany(b => b.Column(1))); + Assert.Equal(10, batches.Sum(b => b.RowCount)); + } + } + + [Fact] + public void Reader_File_EqualsMemory() + { + var layout = Layout(); + string path = Path.GetTempFileName(); + try + { + File.WriteAllBytes(path, Content); + var options = ReaderOptions(4); + Assert.Equal(NativeStatus.Ok, ReaderApi.OpenFile(Encoding.UTF8.GetBytes(path), layout, &options, out nint reader)); + var batches = Drain(reader); + ReaderApi.Close(reader); + + Assert.Equal([4L, 4L, 2L], batches.Select(b => b.RowCount)); + Assert.Equal(FromMemory(Content, layout, Options(headerLines: 1, skipInvalid: true)).Column(1), batches.SelectMany(b => b.Column(1))); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void Reader_AfterEof_ReturnsEof() + { + fixed (byte* data = Content) + { + nint reader = Open(Layout(), ReaderOptions(100), data); + Drain(reader); + NativeTableRaw table = default; + + Assert.Equal(NativeStatus.Eof, ReaderApi.Next(reader, &table)); + Assert.Equal(0, table.ColumnCount); + ReaderApi.Close(reader); + } + } + + [Fact] + public void Reader_ParseError_StopsWithLineNumber() + { + fixed (byte* data = Content) + { + nint reader = Open(Layout(), ReaderOptions(2, skipInvalid: false), data); + NativeTableRaw table = default; + + Assert.Equal(NativeStatus.Ok, ReaderApi.Next(reader, &table)); + NativeTable.Free(&table); + Assert.Equal(NativeStatus.Ok, ReaderApi.Next(reader, &table)); + NativeTable.Free(&table); + Assert.Equal(NativeStatus.ParseError, ReaderApi.Next(reader, &table)); + NativeApi.LastParseError(out long line, out int column); + Assert.Equal(6, line); + Assert.Equal(1, column); + Assert.Equal(NativeStatus.Eof, ReaderApi.Next(reader, &table)); + ReaderApi.Close(reader); + } + } + + [Fact] + public void Reader_Closed_IsInvalidHandle() + { + fixed (byte* data = Content) + { + nint reader = Open(Layout(), ReaderOptions(2), data); + ReaderApi.Close(reader); + NativeTableRaw table = default; + + Assert.Equal(NativeStatus.InvalidHandle, ReaderApi.Next(reader, &table)); + } + } + + [Fact] + public void Reader_ZeroedOptions_UseDefaults() + { + var options = new NativeReaderOptionsRaw { StructSize = sizeof(NativeReaderOptionsRaw) }; + fixed (byte* data = "AB01\nAB02\n"u8) + { + Assert.Equal(NativeStatus.Ok, ReaderApi.OpenMemory(data, 10, Layout(), &options, out nint reader)); + var batches = Drain(reader); + ReaderApi.Close(reader); + + Assert.Equal([2L], batches.Select(b => b.RowCount)); + } + } + } +} From 7608ee7f0637596f0bf28d4dce24470e03ea5c8f Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 01:20:19 -0300 Subject: [PATCH 39/85] feat(native): export reads through the Arrow C Data Interface fw_read_arrow_file/memory publish a struct array, one child per layout column (utf8, int64, float64, decimal128(18, scale), date32, time64[us], timestamp[us]), moving the column buffers into the arrays without a copy; a parallel read is one concatenated array. fw_read_arrow_stream_* yield the batched reader's tables as an ArrowArrayStream. --- .../Arrow/ArrowApi.Stream.cs | 157 ++++++++++ src/FixedWidthParser.Native/Arrow/ArrowApi.cs | 273 ++++++++++++++++++ .../Arrow/NativeArrow.cs | 48 +++ src/FixedWidthParser.Native/Exports.Arrow.cs | 91 ++++++ src/FixedWidthParser.Native/NativeApi.cs | 5 + .../Reading/ReaderApi.cs | 35 ++- .../Native/NativeArrowTests.cs | 221 ++++++++++++++ 7 files changed, 824 insertions(+), 6 deletions(-) create mode 100644 src/FixedWidthParser.Native/Arrow/ArrowApi.Stream.cs create mode 100644 src/FixedWidthParser.Native/Arrow/ArrowApi.cs create mode 100644 src/FixedWidthParser.Native/Arrow/NativeArrow.cs create mode 100644 src/FixedWidthParser.Native/Exports.Arrow.cs create mode 100644 tests/FixedWidthParser.Tests/Native/NativeArrowTests.cs diff --git a/src/FixedWidthParser.Native/Arrow/ArrowApi.Stream.cs b/src/FixedWidthParser.Native/Arrow/ArrowApi.Stream.cs new file mode 100644 index 0000000..9d15ed2 --- /dev/null +++ b/src/FixedWidthParser.Native/Arrow/ArrowApi.Stream.cs @@ -0,0 +1,157 @@ +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native.Arrow +{ + // The stream's GCHandle owns its state, and the state owns the reader, until the release callback runs. +#pragma warning disable CA2000, IDISP001, IDISP004, IDISP007 + internal static unsafe partial class ArrowApi + { + private const int ArrowErrno = 5; + + /// One ArrowArrayStream: a batched reader and the message its last failure left. + internal sealed class ArrowStreamState(NativeTableReader reader) : IDisposable + { + internal NativeTableReader Reader { get; } = reader; + + internal IntPtr LastErrorUtf8 { get; private set; } + + internal void SetError(string message) + { + FreeIfSet(LastErrorUtf8); + LastErrorUtf8 = AllocUtf8Z(message); + } + + public void Dispose() + { + Reader.Dispose(); + FreeIfSet(LastErrorUtf8); + LastErrorUtf8 = IntPtr.Zero; + } + } + + internal static int OpenStreamFile(ReadOnlySpan path, NativeLayout layout, NativeReaderOptionsRaw* options, ArrowArrayStream* outStream) + { + *outStream = default; + int status = ReaderApi.CreateFile(path, layout, options, out var reader); + return status == NativeStatus.Ok ? Publish(reader!, outStream) : status; + } + + /// Reads lazily: the caller keeps it alive until the stream is released. + internal static int OpenStreamMemory(byte* data, long length, NativeLayout layout, NativeReaderOptionsRaw* options, ArrowArrayStream* outStream) + { + *outStream = default; + int status = ReaderApi.CreateMemory(data, length, layout, options, out var reader); + return status == NativeStatus.Ok ? Publish(reader!, outStream) : status; + } + + private static int Publish(NativeTableReader reader, ArrowArrayStream* outStream) + { + *outStream = new ArrowArrayStream + { + GetSchema = (IntPtr)(delegate* unmanaged)&Exports.ArrowStreamGetSchema, + GetNext = (IntPtr)(delegate* unmanaged)&Exports.ArrowStreamGetNext, + GetLastError = (IntPtr)(delegate* unmanaged)&Exports.ArrowStreamGetLastError, + Release = (IntPtr)(delegate* unmanaged)&Exports.ArrowStreamRelease, + PrivateData = GCHandle.ToIntPtr(GCHandle.Alloc(new ArrowStreamState(reader))), + }; + return NativeStatus.Ok; + } + + private static ArrowStreamState? StateOf(ArrowArrayStream* stream) + { + if (stream is null || stream->PrivateData == IntPtr.Zero) + { + return null; + } + return GCHandle.FromIntPtr(stream->PrivateData).Target as ArrowStreamState; + } + + internal static int StreamGetSchema(ArrowArrayStream* stream, ArrowSchema* outSchema) + { + var state = StateOf(stream); + if (state is null || outSchema is null) + { + return ArrowErrno; + } + try + { + *outSchema = BuildSchema(state.Reader.Layout); + return 0; + } + catch (Exception ex) + { + state.SetError(ex.Message); + *outSchema = default; + return ArrowErrno; + } + } + + internal static int StreamGetNext(ArrowArrayStream* stream, ArrowArray* outArray) + { + var state = StateOf(stream); + if (state is null || outArray is null) + { + return ArrowErrno; + } + *outArray = default; + NativeTableRaw table = default; + int status = ReaderApi.NextBatch(state.Reader, &table); + if (status == NativeStatus.Eof) + { + return 0; + } + if (status != NativeStatus.Ok) + { + state.SetError(NativeApi.LastErrorText()); + return ArrowErrno; + } + try + { + *outArray = BuildArray(&table); + return 0; + } + catch (Exception ex) + { + state.SetError(ex.Message); + *outArray = default; + return ArrowErrno; + } + finally + { + NativeTable.Free(&table); + } + } + + internal static IntPtr StreamGetLastError(ArrowArrayStream* stream) + { + return StateOf(stream)?.LastErrorUtf8 ?? IntPtr.Zero; + } + + internal static void StreamRelease(ArrowArrayStream* stream) + { + if (stream is null || stream->Release == IntPtr.Zero) + { + return; + } + IntPtr privateData = stream->PrivateData; + stream->PrivateData = IntPtr.Zero; + stream->Release = IntPtr.Zero; + if (privateData == IntPtr.Zero) + { + return; + } + var handle = GCHandle.FromIntPtr(privateData); + try + { + (handle.Target as ArrowStreamState)?.Dispose(); + } + finally + { + handle.Free(); + } + } + } +#pragma warning restore CA2000, IDISP001, IDISP004, IDISP007 +} diff --git a/src/FixedWidthParser.Native/Arrow/ArrowApi.cs b/src/FixedWidthParser.Native/Arrow/ArrowApi.cs new file mode 100644 index 0000000..3c71ba8 --- /dev/null +++ b/src/FixedWidthParser.Native/Arrow/ArrowApi.cs @@ -0,0 +1,273 @@ +using System.Globalization; +using System.Numerics; +using System.Runtime.InteropServices; +using System.Text; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native.Arrow +{ + /// + /// fw_read_arrow_*: a columnar read handed over through the Arrow C Data Interface as a struct + /// array, one child per column. The column buffers move into the arrays without a copy; only DECIMAL + /// widens its int64 values to Arrow's 128-bit decimals. + /// + internal static unsafe partial class ArrowApi + { + private const int DecimalBytes = 16; + + internal static int ReadFile(ReadOnlySpan path, NativeLayout layout, NativeReadOptionsRaw* options, ArrowArray* outArray, ArrowSchema* outSchema, NativeReadResultRaw* outResult) + { + NativeTableRaw table = default; + int status = ReadApi.ReadFile(path, layout, options, &table, outResult); + return Export(status, layout, &table, outArray, outSchema); + } + + internal static int ReadMemory(byte* data, long length, NativeLayout layout, NativeReadOptionsRaw* options, ArrowArray* outArray, ArrowSchema* outSchema, NativeReadResultRaw* outResult) + { + NativeTableRaw table = default; + int status = ReadApi.ReadMemory(data, length, layout, options, &table, outResult); + return Export(status, layout, &table, outArray, outSchema); + } + + private static int Export(int status, NativeLayout layout, NativeTableRaw* table, ArrowArray* outArray, ArrowSchema* outSchema) + { + *outArray = default; + *outSchema = default; + if (status != NativeStatus.Ok) + { + return status; + } + try + { + *outSchema = BuildSchema(layout); + *outArray = BuildArray(table); + return NativeStatus.Ok; + } + catch (Exception ex) + { + NativeApi.SetLastError(ex.Message); + ReleaseArrowSchema((IntPtr)outSchema); + ReleaseArrowArray((IntPtr)outArray); + *outArray = default; + *outSchema = default; + return NativeStatus.Error; + } + finally + { + NativeTable.Free(table); + } + } + + internal static void ReleaseArrowSchema(IntPtr schemaPtr) + { + var schema = (ArrowSchema*)schemaPtr; + if (schema is null || schema->Release == IntPtr.Zero) + { + return; + } + ReleaseChildren(schema->Children, schema->NChildren, &ReleaseArrowSchema); + FreeIfSet(schema->Format); + FreeIfSet(schema->Name); + schema->Release = IntPtr.Zero; + } + + internal static void ReleaseArrowArray(IntPtr arrayPtr) + { + var array = (ArrowArray*)arrayPtr; + if (array is null || array->Release == IntPtr.Zero) + { + return; + } + ReleaseChildren(array->Children, array->NChildren, &ReleaseArrowArray); + var buffers = (IntPtr*)array->Buffers; + for (long i = 0; i < array->NBuffers; i++) + { + FreeIfSet(buffers[i]); + } + FreeIfSet(array->Buffers); + array->Release = IntPtr.Zero; + } + + private static void ReleaseChildren(IntPtr children, long count, delegate* release) + { + var pointers = (IntPtr*)children; + for (long i = 0; i < count; i++) + { + release(pointers[i]); + Marshal.FreeHGlobal(pointers[i]); + } + FreeIfSet(children); + } + + private static void FreeIfSet(IntPtr pointer) + { + if (pointer != IntPtr.Zero) + { + Marshal.FreeHGlobal(pointer); + } + } + + internal static ArrowSchema BuildSchema(NativeLayout layout) + { + var columns = layout.Columns; + var children = (IntPtr*)Marshal.AllocHGlobal(columns.Length * IntPtr.Size); + var schema = new ArrowSchema + { + Format = AllocUtf8Z("+s"), + NChildren = 0, + Children = (IntPtr)children, + Release = SchemaReleaseCallback, + }; + try + { + foreach (var column in columns) + { + var child = (ArrowSchema*)Marshal.AllocHGlobal(sizeof(ArrowSchema)); + *child = new ArrowSchema + { + Format = AllocUtf8Z(FormatCode(column)), + Name = AllocUtf8Z(column.Name), + Flags = column.Nullable ? ArrowFlags.Nullable : 0, + Release = SchemaReleaseCallback, + }; + children[schema.NChildren++] = (IntPtr)child; + } + return schema; + } + catch + { + ReleaseArrowSchema((IntPtr)(&schema)); + throw; + } + } + + /// Moves every column of into a struct array, leaving the table's buffers empty. + internal static ArrowArray BuildArray(NativeTableRaw* table) + { + var children = (IntPtr*)Marshal.AllocHGlobal(Math.Max(1, table->ColumnCount) * IntPtr.Size); + var buffers = (IntPtr*)Marshal.AllocHGlobal(IntPtr.Size); + buffers[0] = IntPtr.Zero; + var array = new ArrowArray + { + Length = table->RowCount, + NBuffers = 1, + Buffers = (IntPtr)buffers, + Children = (IntPtr)children, + Release = ArrayReleaseCallback, + }; + try + { + for (int i = 0; i < table->ColumnCount; i++) + { + var child = (ArrowArray*)Marshal.AllocHGlobal(sizeof(ArrowArray)); + *child = default; + children[array.NChildren++] = (IntPtr)child; + *child = BuildChild(table->Columns + i); + } + return array; + } + catch + { + ReleaseArrowArray((IntPtr)(&array)); + throw; + } + } + + private static ArrowArray BuildChild(NativeColumnRaw* column) + { + bool isString = column->Type == NativeType.String; + int bufferCount = isString ? 3 : 2; + var buffers = (IntPtr*)Marshal.AllocHGlobal(bufferCount * IntPtr.Size); + buffers[0] = Take(ref column->Validity); + buffers[1] = column->Type == NativeType.Decimal ? WidenDecimals(column) : OrAllocate((IntPtr)column->Values, 8); + column->Values = null; + if (isString) + { + buffers[2] = OrAllocate((IntPtr)column->Data, 1); + column->Data = null; + } + return new ArrowArray + { + Length = column->Length, + NullCount = CountUnset(buffers[0], column->Length), + NBuffers = bufferCount, + Buffers = (IntPtr)buffers, + Release = ArrayReleaseCallback, + }; + } + + private static IntPtr Take(ref byte* pointer) + { + IntPtr taken = (IntPtr)pointer; + pointer = null; + return taken; + } + + // A buffer of size 0 may be NULL under the C Data Interface, but some importers want a pointer. + private static IntPtr OrAllocate(IntPtr buffer, int minimum) + { + return buffer == IntPtr.Zero ? Marshal.AllocHGlobal(minimum) : buffer; + } + + private static IntPtr WidenDecimals(NativeColumnRaw* column) + { + var values = new ReadOnlySpan(column->Values, (int)column->Length); + var wide = (long*)Marshal.AllocHGlobal((int)Math.Max(1, column->Length) * DecimalBytes); + for (int i = 0; i < values.Length; i++) + { + wide[2 * i] = values[i]; + wide[(2 * i) + 1] = values[i] < 0 ? -1 : 0; + } + Marshal.FreeHGlobal((IntPtr)column->Values); + column->Values = null; + return (IntPtr)wide; + } + + private static long CountUnset(IntPtr validity, long length) + { + if (validity == IntPtr.Zero) + { + return 0; + } + ReadOnlySpan bits = new((void*)validity, (int)((length + 7) / 8)); + long set = 0; + for (long row = 0; row < length; row += 64) + { + int bytes = (int)Math.Min(8, bits.Length - (row / 8)); + ulong packed = 0; + bits.Slice((int)(row / 8), bytes).CopyTo(MemoryMarshal.AsBytes(new Span(ref packed))); + packed &= length - row >= 64 ? ulong.MaxValue : (1UL << (int)(length - row)) - 1; + set += BitOperations.PopCount(packed); + } + return length - set; + } + + private static string FormatCode(NativeColumn column) + { + return column.Type switch + { + NativeType.String => "u", + NativeType.Int64 => "l", + NativeType.Float64 => "g", + NativeType.Decimal => string.Create(CultureInfo.InvariantCulture, $"d:18,{column.Scale}"), + NativeType.Date => "tdD", + NativeType.Time => "ttu", + _ => "tsu:", + }; + } + + private static IntPtr AllocUtf8Z(string value) + { + int length = Encoding.UTF8.GetByteCount(value); + var block = (byte*)Marshal.AllocHGlobal(length + 1); + Encoding.UTF8.GetBytes(value, new Span(block, length)); + block[length] = 0; + return (IntPtr)block; + } + + private static IntPtr SchemaReleaseCallback => (IntPtr)(delegate* unmanaged)&Exports.ReleaseArrowSchemaCallback; + + private static IntPtr ArrayReleaseCallback => (IntPtr)(delegate* unmanaged)&Exports.ReleaseArrowArrayCallback; + } +} diff --git a/src/FixedWidthParser.Native/Arrow/NativeArrow.cs b/src/FixedWidthParser.Native/Arrow/NativeArrow.cs new file mode 100644 index 0000000..6dc0c82 --- /dev/null +++ b/src/FixedWidthParser.Native/Arrow/NativeArrow.cs @@ -0,0 +1,48 @@ +using System.Runtime.InteropServices; + +namespace FixedWidthParser.Native.Arrow +{ + internal static class ArrowFlags + { + internal const long Nullable = 2; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct ArrowSchema + { + public IntPtr Format; + public IntPtr Name; + public IntPtr Metadata; + public long Flags; + public long NChildren; + public IntPtr Children; + public IntPtr Dictionary; + public IntPtr Release; + public IntPtr PrivateData; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct ArrowArray + { + public long Length; + public long NullCount; + public long Offset; + public long NBuffers; + public long NChildren; + public IntPtr Buffers; + public IntPtr Children; + public IntPtr Dictionary; + public IntPtr Release; + public IntPtr PrivateData; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct ArrowArrayStream + { + public IntPtr GetSchema; + public IntPtr GetNext; + public IntPtr GetLastError; + public IntPtr Release; + public IntPtr PrivateData; + } +} diff --git a/src/FixedWidthParser.Native/Exports.Arrow.cs b/src/FixedWidthParser.Native/Exports.Arrow.cs new file mode 100644 index 0000000..1fbbf1f --- /dev/null +++ b/src/FixedWidthParser.Native/Exports.Arrow.cs @@ -0,0 +1,91 @@ +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Arrow; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native +{ + internal static unsafe partial class Exports + { + [UnmanagedCallersOnly(EntryPoint = "fw_read_arrow_file")] + public static int ReadArrowFile(byte* path, int pathLength, nint layout, NativeReadOptionsRaw* options, ArrowArray* outArray, ArrowSchema* outSchema, NativeReadResultRaw* outResult) + { + if (path is null || pathLength <= 0 || outArray is null || outSchema is null) + { + return InvalidArgument("fw_read_arrow_file needs a path, out_array and out_schema."); + } + *outArray = default; + *outSchema = default; + return WithLayout(layout, l => ArrowApi.ReadFile(new ReadOnlySpan(path, pathLength), l, options, outArray, outSchema, outResult)); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_read_arrow_memory")] + public static int ReadArrowMemory(byte* data, long dataLength, nint layout, NativeReadOptionsRaw* options, ArrowArray* outArray, ArrowSchema* outSchema, NativeReadResultRaw* outResult) + { + if ((data is null && dataLength != 0) || dataLength < 0 || outArray is null || outSchema is null) + { + return InvalidArgument("fw_read_arrow_memory needs data for a non-zero data_len, out_array and out_schema."); + } + *outArray = default; + *outSchema = default; + return WithLayout(layout, l => ArrowApi.ReadMemory(data, dataLength, l, options, outArray, outSchema, outResult)); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_read_arrow_stream_file")] + public static int ReadArrowStreamFile(byte* path, int pathLength, nint layout, NativeReaderOptionsRaw* options, ArrowArrayStream* outStream) + { + if (path is null || pathLength <= 0 || outStream is null) + { + return InvalidArgument("fw_read_arrow_stream_file needs a path and out_stream."); + } + *outStream = default; + return WithLayout(layout, l => ArrowApi.OpenStreamFile(new ReadOnlySpan(path, pathLength), l, options, outStream)); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_read_arrow_stream_memory")] + public static int ReadArrowStreamMemory(byte* data, long dataLength, nint layout, NativeReaderOptionsRaw* options, ArrowArrayStream* outStream) + { + if ((data is null && dataLength != 0) || dataLength < 0 || outStream is null) + { + return InvalidArgument("fw_read_arrow_stream_memory needs data for a non-zero data_len, and out_stream."); + } + *outStream = default; + return WithLayout(layout, l => ArrowApi.OpenStreamMemory(data, dataLength, l, options, outStream)); + } + + [UnmanagedCallersOnly] + internal static void ReleaseArrowSchemaCallback(ArrowSchema* schema) + { + ArrowApi.ReleaseArrowSchema((IntPtr)schema); + } + + [UnmanagedCallersOnly] + internal static void ReleaseArrowArrayCallback(ArrowArray* array) + { + ArrowApi.ReleaseArrowArray((IntPtr)array); + } + + [UnmanagedCallersOnly] + internal static int ArrowStreamGetSchema(ArrowArrayStream* stream, ArrowSchema* outSchema) + { + return ArrowApi.StreamGetSchema(stream, outSchema); + } + + [UnmanagedCallersOnly] + internal static int ArrowStreamGetNext(ArrowArrayStream* stream, ArrowArray* outArray) + { + return ArrowApi.StreamGetNext(stream, outArray); + } + + [UnmanagedCallersOnly] + internal static IntPtr ArrowStreamGetLastError(ArrowArrayStream* stream) + { + return ArrowApi.StreamGetLastError(stream); + } + + [UnmanagedCallersOnly] + internal static void ArrowStreamRelease(ArrowArrayStream* stream) + { + ArrowApi.StreamRelease(stream); + } + } +} diff --git a/src/FixedWidthParser.Native/NativeApi.cs b/src/FixedWidthParser.Native/NativeApi.cs index 560c0dc..d0cf51a 100644 --- a/src/FixedWidthParser.Native/NativeApi.cs +++ b/src/FixedWidthParser.Native/NativeApi.cs @@ -44,6 +44,11 @@ internal static void LastParseError(out long line, out int column) column = _lastError is null ? -1 : _parseColumn; } + internal static string LastErrorText() + { + return _lastError ?? ""; + } + internal static int LastError(Span buffer, out int length) { string? message = _lastError; diff --git a/src/FixedWidthParser.Native/Reading/ReaderApi.cs b/src/FixedWidthParser.Native/Reading/ReaderApi.cs index 422ee55..8ea05d4 100644 --- a/src/FixedWidthParser.Native/Reading/ReaderApi.cs +++ b/src/FixedWidthParser.Native/Reading/ReaderApi.cs @@ -7,9 +7,27 @@ namespace FixedWidthParser.Native.Reading /// fw_reader_*: batches of a sequential read, behind a handle. internal static unsafe class ReaderApi { +#pragma warning disable CA2000, IDISP001 // The handle table owns a reader until fw_reader_close. internal static int OpenFile(ReadOnlySpan path, NativeLayout layout, NativeReaderOptionsRaw* options, out nint reader) { - reader = 0; + int status = CreateFile(path, layout, options, out var created); + reader = status == NativeStatus.Ok ? NativeHandleTable.Register(created!) : 0; + return status; + } + + /// Reads in place: the caller keeps it alive until the reader is closed. + internal static int OpenMemory(byte* data, long length, NativeLayout layout, NativeReaderOptionsRaw* options, out nint reader) + { + int status = CreateMemory(data, length, layout, options, out var created); + reader = status == NativeStatus.Ok ? NativeHandleTable.Register(created!) : 0; + return status; + } + +#pragma warning restore CA2000, IDISP001 + + internal static int CreateFile(ReadOnlySpan path, NativeLayout layout, NativeReaderOptionsRaw* options, out NativeTableReader? reader) + { + reader = null; int status = TrySettings(options, out var settings, out long batchRows); if (status != NativeStatus.Ok) { @@ -20,7 +38,7 @@ internal static int OpenFile(ReadOnlySpan path, NativeLayout layout, Nativ { stream = new FileStream(Encoding.UTF8.GetString(path), FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1, FileOptions.SequentialScan); long dataStart = ReadApi.HeaderEnd(new FileSource(stream.SafeFileHandle), settings!.HeaderLines, out long headerLines); - reader = NativeHandleTable.Register(new NativeTableReader(stream, dataStart, headerLines, layout, settings, batchRows)); + reader = new NativeTableReader(stream, dataStart, headerLines, layout, settings, batchRows); return NativeStatus.Ok; } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException) @@ -31,10 +49,9 @@ internal static int OpenFile(ReadOnlySpan path, NativeLayout layout, Nativ } } - /// Reads in place: the caller keeps it alive until the reader is closed. - internal static int OpenMemory(byte* data, long length, NativeLayout layout, NativeReaderOptionsRaw* options, out nint reader) + internal static int CreateMemory(byte* data, long length, NativeLayout layout, NativeReaderOptionsRaw* options, out NativeTableReader? reader) { - reader = 0; + reader = null; int status = TrySettings(options, out var settings, out long batchRows); if (status != NativeStatus.Ok) { @@ -44,7 +61,7 @@ internal static int OpenMemory(byte* data, long length, NativeLayout layout, Nat #pragma warning disable CA2000 // The reader owns and disposes the stream. var stream = new UnmanagedMemoryStream(data, length); #pragma warning restore CA2000 - reader = NativeHandleTable.Register(new NativeTableReader(stream, dataStart, headerLines, layout, settings, batchRows)); + reader = new NativeTableReader(stream, dataStart, headerLines, layout, settings, batchRows); return NativeStatus.Ok; } @@ -57,6 +74,12 @@ internal static int Next(nint handle, NativeTableRaw* table) NativeApi.SetLastError("The reader handle is not a live fw_reader."); return NativeStatus.InvalidHandle; } + return NextBatch(reader, table); + } + + /// The next batch of , with every failure turned into a status and the last error. + internal static int NextBatch(NativeTableReader reader, NativeTableRaw* table) + { try { return reader.Next(table); diff --git a/tests/FixedWidthParser.Tests/Native/NativeArrowTests.cs b/tests/FixedWidthParser.Tests/Native/NativeArrowTests.cs new file mode 100644 index 0000000..90a951e --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeArrowTests.cs @@ -0,0 +1,221 @@ +using System.Text; +using Apache.Arrow; +using Apache.Arrow.C; +using Apache.Arrow.Ipc; +using Apache.Arrow.Types; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Arrow; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; +using static FixedWidthParser.Tests.Native.NativeTestReads; + +namespace FixedWidthParser.Tests.Native +{ + public sealed class NativeArrowTests + { + private const string Content = + "Ann " + "00042" + " 12.500" + "00012345" + "20231024" + "133005" + "20231024133000" + "\n" + + " " + " " + " -0.5" + "-0000001" + "19700101" + "000000" + "19700101000000" + "\n" + + "Cid " + "00007" + " 3.250" + "00000100" + "20000229" + "235959" + "20000229235959" + "\n"; + + private static readonly int ReaderOptionsSize = System.Runtime.InteropServices.Marshal.SizeOf(); + + private static NativeLayout Layout() + { + return Build( + new Col("name", 0, 6, TString) { Nullable = true }, + new Col("qty", 6, 5, TInt64) { Nullable = true }, + new Col("price", 11, 8, TFloat64), + new Col("amount", 19, 8, TDecimal) { Scale = 2 }, + new Col("day", 27, 8, TDate) { Format = "yyyyMMdd" }, + new Col("time", 35, 6, TTime) { Format = "HHmmss" }, + new Col("stamp", 41, 14, TTimestamp) { Format = "yyyyMMddHHmmss" }); + } + + private static unsafe RecordBatch ReadBatch(string content, NativeLayout layout, NativeReadOptionsRaw? options = null) + { + byte[] bytes = Encoding.UTF8.GetBytes(content); + ArrowArray array = default; + ArrowSchema schema = default; + NativeReadOptionsRaw value = options ?? default; + fixed (byte* data = bytes) + { + int status = ArrowApi.ReadMemory(data, bytes.Length, layout, options is null ? null : &value, &array, &schema, null); + Assert.True(status == NativeStatus.Ok, LastError()); + } + var imported = CArrowSchemaImporter.ImportSchema((CArrowSchema*)&schema); + return CArrowArrayImporter.ImportRecordBatch((CArrowArray*)&array, imported); + } + + private static object?[] Values(IArrowArray column) + { + return column switch + { + StringArray s => [.. Enumerable.Range(0, s.Length).Select(i => s.IsNull(i) ? null : s.GetString(i))], + Int64Array l => [.. Enumerable.Range(0, l.Length).Select(i => (object?)l.GetValue(i))], + DoubleArray d => [.. Enumerable.Range(0, d.Length).Select(i => (object?)d.GetValue(i))], + Decimal128Array m => [.. Enumerable.Range(0, m.Length).Select(i => (object?)m.GetValue(i))], + Date32Array t => [.. Enumerable.Range(0, t.Length).Select(i => (object?)t.GetValue(i))], + Time64Array t => [.. Enumerable.Range(0, t.Length).Select(i => (object?)t.GetValue(i))], + TimestampArray t => [.. Enumerable.Range(0, t.Length).Select(i => (object?)t.GetValue(i))], + _ => throw new InvalidOperationException(column.GetType().Name), + }; + } + + [Fact] + public void Arrow_EveryType_MatchesColumnarRead() + { + var layout = Layout(); + var columnar = FromMemory(Encoding.UTF8.GetBytes(Content), layout); + + using var batch = ReadBatch(Content, layout); + + Assert.Equal(3, batch.Length); + Assert.Equal(["name", "qty", "price", "amount", "day", "time", "stamp"], batch.Schema.FieldsList.Select(f => f.Name)); + Assert.Equal(columnar.Column(0), Values(batch.Column(0))); + Assert.Equal(columnar.Column(1), Values(batch.Column(1))); + Assert.Equal(columnar.Column(2), Values(batch.Column(2))); + Assert.Equal([123.45m, -0.01m, 1.00m], Values(batch.Column(3))); + Assert.Equal(columnar.Column(4), Values(batch.Column(4))); + Assert.Equal(columnar.Column(5), Values(batch.Column(5))); + Assert.Equal(columnar.Column(6), Values(batch.Column(6))); + } + + [Fact] + public void Arrow_Nulls_HaveValidity() + { + using var batch = ReadBatch(Content, Layout()); + + Assert.Equal(1, batch.Column(0).NullCount); + Assert.True(batch.Column(1).IsNull(1)); + Assert.Equal(0, batch.Column(2).NullCount); + Assert.True(batch.Schema.GetFieldByName("name").IsNullable); + Assert.False(batch.Schema.GetFieldByName("price").IsNullable); + } + + [Fact] + public void Arrow_Decimal_HasScale() + { + using var batch = ReadBatch(Content, Layout()); + + var type = Assert.IsType(batch.Schema.GetFieldByName("amount").DataType); + Assert.Equal(18, type.Precision); + Assert.Equal(2, type.Scale); + Assert.IsType(batch.Schema.GetFieldByName("stamp").DataType); + Assert.Equal(TimeUnit.Microsecond, ((TimestampType)batch.Schema.GetFieldByName("stamp").DataType).Unit); + } + + [Fact] + public void Arrow_Parallel_EqualsSequential() + { + var layout = Layout(); + string content = string.Concat(Enumerable.Repeat(Content, 50)); + + using var sequential = ReadBatch(content, layout, Options(degreeOfParallelism: 1)); + using var parallel = ReadBatch(content, layout, Options(degreeOfParallelism: 4)); + + for (int i = 0; i < 7; i++) + { + Assert.Equal(Values(sequential.Column(i)), Values(parallel.Column(i))); + } + } + + [Fact] + public async Task ArrowStream_BatchesConcatenate_ToTheWholeRead() + { + var layout = Layout(); + string content = string.Concat(Enumerable.Repeat(Content, 5)); + using var whole = ReadBatch(content, layout); + byte[] bytes = Encoding.UTF8.GetBytes(content); + var options = new NativeReaderOptionsRaw { StructSize = ReaderOptionsSize, BatchRows = 4 }; + + var batches = new List(); + IArrowArrayStream stream = OpenStream(bytes, layout, options); + try + { + while (await stream.ReadNextRecordBatchAsync() is { } batch) + { + batches.Add(batch); + } + } + finally + { + stream.Dispose(); + } + + Assert.Equal([4, 4, 4, 3], batches.Select(b => b.Length)); + for (int i = 0; i < 7; i++) + { + Assert.Equal(Values(whole.Column(i)), batches.SelectMany(b => Values(b.Column(i)))); + } + batches.ForEach(b => b.Dispose()); + } + + [Fact] + public async Task ArrowStream_ParseError_IsReported() + { + var layout = Build(new Col("n", 0, 2, TInt64)); + byte[] bytes = "01\nXX\n"u8.ToArray(); + + IArrowArrayStream stream = OpenStream(bytes, layout, new NativeReaderOptionsRaw { StructSize = ReaderOptionsSize, BatchRows = 1 }); + try + { + using var first = await stream.ReadNextRecordBatchAsync(); + var ex = await Assert.ThrowsAnyAsync(async () => await stream.ReadNextRecordBatchAsync()); + Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); + } + finally + { + stream.Dispose(); + } + } + + [Fact] + public unsafe void Arrow_ParseError_IsReported() + { + var layout = Build(new Col("n", 0, 2, TInt64)); + ArrowArray array = default; + ArrowSchema schema = default; + fixed (byte* data = "XX\n"u8) + { + Assert.Equal(NativeStatus.ParseError, ArrowApi.ReadMemory(data, 3, layout, null, &array, &schema, null)); + } + Assert.Equal(IntPtr.Zero, array.Release); + Assert.Equal(IntPtr.Zero, schema.Release); + } + + [Fact] + public unsafe void Arrow_Release_IsIdempotent() + { + byte[] bytes = Encoding.UTF8.GetBytes(Content); + ArrowArray array = default; + ArrowSchema schema = default; + fixed (byte* data = bytes) + { + Assert.Equal(NativeStatus.Ok, ArrowApi.ReadMemory(data, bytes.Length, Layout(), null, &array, &schema, null)); + } + + ArrowApi.ReleaseArrowArray((IntPtr)(&array)); + ArrowApi.ReleaseArrowArray((IntPtr)(&array)); + ArrowApi.ReleaseArrowSchema((IntPtr)(&schema)); + ArrowApi.ReleaseArrowSchema((IntPtr)(&schema)); + + Assert.Equal(IntPtr.Zero, array.Release); + Assert.Equal(IntPtr.Zero, schema.Release); + } + + private static unsafe IArrowArrayStream OpenStream(byte[] bytes, NativeLayout layout, NativeReaderOptionsRaw options) + { + var stream = (CArrowArrayStream*)System.Runtime.InteropServices.NativeMemory.AllocZeroed((nuint)sizeof(CArrowArrayStream)); + fixed (byte* data = bytes) + { + // The stream reads the memory lazily, so it is copied to memory that outlives this call. + byte* copy = (byte*)System.Runtime.InteropServices.NativeMemory.Alloc((nuint)bytes.Length); + new ReadOnlySpan(data, bytes.Length).CopyTo(new Span(copy, bytes.Length)); + Assert.Equal(NativeStatus.Ok, ArrowApi.OpenStreamMemory(copy, bytes.Length, layout, &options, (ArrowArrayStream*)stream)); + } + return CArrowArrayStreamImporter.ImportArrayStream(stream); + } + } +} From 57029c1891bc38c5a734292c33eefa58d82ef8d6 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 01:24:53 -0300 Subject: [PATCH 40/85] feat(native): write columnar tables fw_write_file and fw_write_memory format an fw_table through a layout: strings from UTF-8 (cut at a character boundary on truncate, transcoded for other encodings), integers, scaled floats as digits, decimals, dates and times (ISO 8601 without a format), nulls as the trim character, gaps and line_length padding as the filler. Batches of batch_rows lines format in parallel into their known offsets, so the output equals a serial write. fw_free_buffer releases memory output. --- src/FixedWidthParser.Native/Exports.Writer.cs | 36 +++ .../Layouts/NativeColumnRules.cs | 14 +- .../Writing/TableFormatter.cs | 166 +++++++++++ .../Writing/WriteApi.cs | 144 +++++++++ .../Writing/WriteSettings.cs | 72 +++++ .../Native/NativeTestLayouts.cs | 7 + .../Native/NativeWriteTests.cs | 274 ++++++++++++++++++ 7 files changed, 712 insertions(+), 1 deletion(-) create mode 100644 src/FixedWidthParser.Native/Exports.Writer.cs create mode 100644 src/FixedWidthParser.Native/Writing/TableFormatter.cs create mode 100644 src/FixedWidthParser.Native/Writing/WriteApi.cs create mode 100644 src/FixedWidthParser.Native/Writing/WriteSettings.cs create mode 100644 tests/FixedWidthParser.Tests/Native/NativeWriteTests.cs diff --git a/src/FixedWidthParser.Native/Exports.Writer.cs b/src/FixedWidthParser.Native/Exports.Writer.cs new file mode 100644 index 0000000..0073bdf --- /dev/null +++ b/src/FixedWidthParser.Native/Exports.Writer.cs @@ -0,0 +1,36 @@ +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Reading; +using FixedWidthParser.Native.Writing; + +namespace FixedWidthParser.Native +{ + internal static unsafe partial class Exports + { + [UnmanagedCallersOnly(EntryPoint = "fw_write_file")] + public static int WriteFile(byte* path, int pathLength, NativeTableRaw* table, nint layout, NativeWriteOptionsRaw* options) + { + if (path is null || pathLength <= 0 || table is null) + { + return InvalidArgument("fw_write_file needs a path and a table."); + } + return WithLayout(layout, l => WriteApi.WriteFile(new ReadOnlySpan(path, pathLength), table, l, options)); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_write_memory")] + public static int WriteMemory(NativeTableRaw* table, nint layout, NativeWriteOptionsRaw* options, NativeBufferRaw* outBuffer) + { + if (table is null || outBuffer is null) + { + return InvalidArgument("fw_write_memory needs a table and out_buffer."); + } + *outBuffer = default; + return WithLayout(layout, l => WriteApi.WriteMemory(table, l, options, outBuffer)); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_free_buffer")] + public static void FreeBuffer(NativeBufferRaw* buffer) + { + WriteApi.FreeBuffer(buffer); + } + } +} diff --git a/src/FixedWidthParser.Native/Layouts/NativeColumnRules.cs b/src/FixedWidthParser.Native/Layouts/NativeColumnRules.cs index 338721d..edd34af 100644 --- a/src/FixedWidthParser.Native/Layouts/NativeColumnRules.cs +++ b/src/FixedWidthParser.Native/Layouts/NativeColumnRules.cs @@ -56,11 +56,23 @@ internal static NativeColumn Build(NativeColumnSpecRaw spec, string name, string Nullable = spec.Nullable != 0, Scale = spec.Scale, Parse = new ColumnParseOptions(trimChar, TrimModeOf(spec.TrimMode), format), - FormatOptions = new ColumnFormatOptions(AlignmentOf(spec.Alignment, spec.Type), padding, format, + FormatOptions = new ColumnFormatOptions(AlignmentOf(spec.Alignment, spec.Type), padding, format ?? IsoFormat(spec.Type), spec.Overflow == 1 ? OverflowBehavior.Truncate : OverflowBehavior.Throw), }; } + // Without a format, dates and times are written as ISO 8601 to the second; reading without one accepts it back. + private static string? IsoFormat(int type) + { + return type switch + { + NativeType.Date => "yyyy-MM-dd", + NativeType.Time => "HH:mm:ss", + NativeType.Timestamp => "yyyy-MM-ddTHH:mm:ss", + _ => null, + }; + } + private static TrimMode TrimModeOf(int value) { return value switch diff --git a/src/FixedWidthParser.Native/Writing/TableFormatter.cs b/src/FixedWidthParser.Native/Writing/TableFormatter.cs new file mode 100644 index 0000000..dd19646 --- /dev/null +++ b/src/FixedWidthParser.Native/Writing/TableFormatter.cs @@ -0,0 +1,166 @@ +using System.Globalization; +using System.Text; +using FixedWidthParser.Infrastructure; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native.Writing +{ + /// + /// Formats the rows of a caller's fw_table through a layout, one line at a time into a slice the + /// caller owns. Read-only over the table, so several threads may format different rows at once. + /// + internal sealed unsafe class TableFormatter + { + private static readonly int EpochDay = new DateOnly(1970, 1, 1).DayNumber; + private static readonly double[] Pow10 = [.. Enumerable.Range(0, NativeType.MaxScale + 1).Select(p => Math.Pow(10, p))]; + + private readonly NativeColumn[] _columns; + private readonly NativeColumnRaw* _data; + private readonly byte _filler; + private readonly FixedWidthContext _context; + + private TableFormatter(NativeLayout layout, NativeTableRaw* table, Encoding? encoding) + { + _columns = layout.Columns; + _data = table->Columns; + _filler = layout.Filler; + _context = new FixedWidthContext { FormatProvider = CultureInfo.InvariantCulture, Encoding = encoding! }; + Rows = table->RowCount; + } + + internal long Rows { get; } + + internal static int TryCreate(NativeTableRaw* table, NativeLayout layout, Encoding? encoding, out TableFormatter? formatter) + { + formatter = null; + string? error = Check(table, layout); + if (error is not null) + { + NativeApi.SetLastError(error); + return NativeStatus.InvalidArgument; + } + formatter = new TableFormatter(layout, table, encoding); + return NativeStatus.Ok; + } + + private static string? Check(NativeTableRaw* table, NativeLayout layout) + { + var columns = layout.Columns; + if (table->ColumnCount != columns.Length || table->RowCount < 0 || (table->Columns is null && table->ColumnCount > 0)) + { + return $"The table has {table->ColumnCount} columns and the layout {columns.Length}; they must match by position."; + } + for (int i = 0; i < columns.Length; i++) + { + var raw = table->Columns[i]; + var column = columns[i]; + if (raw.Type != column.Type || (column.Type == NativeType.Decimal && raw.Scale != column.Scale)) + { + return $"Column '{column.Name}' has type {raw.Type} (scale {raw.Scale}) in the table and {column.Type} (scale {column.Scale}) in the layout."; + } + if (raw.Length != table->RowCount || (raw.Values is null && raw.Length > 0) || (raw.Data is null && raw.DataLen > 0)) + { + return $"Column '{column.Name}' has {raw.Length} values for {table->RowCount} rows, or a buffer that does not match its length."; + } + } + return null; + } + + /// Writes row into , whose length is the layout's line length. + internal void FormatRow(long row, Span line) + { + line.Fill(_filler); + for (int i = 0; i < _columns.Length; i++) + { + var column = _columns[i]; + var slice = line.Slice(column.Start, column.Length); + var raw = _data + i; + if (raw->Validity is not null && (raw->Validity[row >> 3] & (1 << (int)(row & 7))) == 0) + { + slice.Fill((byte)column.Parse.TrimChar); + continue; + } + FormatValue(column, raw, row, slice); + } + } + + private void FormatValue(NativeColumn column, NativeColumnRaw* raw, long row, Span slice) + { + var options = column.FormatOptions; + switch (column.Type) + { + case NativeType.String: + var offsets = (int*)raw->Values; + FormatText(column, new ReadOnlySpan(raw->Data + offsets[row], offsets[row + 1] - offsets[row]), slice); + return; + case NativeType.Float64: + FormatFloat(column, ((double*)raw->Values)[row], slice); + return; + case NativeType.Date: + FixedWidthRuntime.FormatUtf8Value(DateOnly.FromDayNumber(EpochDay + ((int*)raw->Values)[row]), slice, in _context, options, column.Name); + return; + case NativeType.Time: + FixedWidthRuntime.FormatUtf8Value(new TimeOnly(((long*)raw->Values)[row] * TimeSpan.TicksPerMicrosecond), slice, in _context, options, column.Name); + return; + case NativeType.Timestamp: + var ticks = DateTime.UnixEpoch.Ticks + (((long*)raw->Values)[row] * TimeSpan.TicksPerMicrosecond); + FixedWidthRuntime.FormatUtf8Value(new DateTime(ticks, DateTimeKind.Unspecified), slice, in _context, options, column.Name); + return; + default: + FixedWidthRuntime.FormatUtf8Value(((long*)raw->Values)[row], slice, in _context, options, column.Name); + return; + } + } + + private void FormatFloat(NativeColumn column, double value, Span slice) + { + if (column.Scale == 0) + { + FixedWidthRuntime.FormatUtf8Value(value, slice, in _context, column.FormatOptions, column.Name); + return; + } + double scaled = Math.Round(value * Pow10[column.Scale]); + if (!(Math.Abs(scaled) < 9e18)) + { + throw new InvalidOperationException($"Value {value} of column \"{column.Name}\" does not fit a scaled 64-bit integer."); + } + FixedWidthRuntime.FormatUtf8Value((long)scaled, slice, in _context, column.FormatOptions, column.Name); + } + + private void FormatText(NativeColumn column, ReadOnlySpan utf8, Span slice) + { + var options = column.FormatOptions; + if (!_context.IsUtf8) + { + FixedWidthRuntime.FormatString(Encoding.UTF8.GetString(utf8), slice, in _context, options, column.Name); + return; + } + if (utf8.Length > slice.Length && options.Overflow == OverflowBehavior.Truncate) + { + utf8 = WholeCharacters(utf8, slice.Length, options.Alignment == Alignment.Right); + } + options.WriteInto(utf8, slice, column.Name); + } + + /// At most bytes of , from the aligned side, never splitting a character. + private static ReadOnlySpan WholeCharacters(ReadOnlySpan utf8, int width, bool keepEnd) + { + if (keepEnd) + { + int start = utf8.Length - width; + while (start < utf8.Length && (utf8[start] & 0xC0) == 0x80) + { + start++; + } + return utf8[start..]; + } + int end = width; + while (end > 0 && (utf8[end] & 0xC0) == 0x80) + { + end--; + } + return utf8[..end]; + } + } +} diff --git a/src/FixedWidthParser.Native/Writing/WriteApi.cs b/src/FixedWidthParser.Native/Writing/WriteApi.cs new file mode 100644 index 0000000..39c88ee --- /dev/null +++ b/src/FixedWidthParser.Native/Writing/WriteApi.cs @@ -0,0 +1,144 @@ +using System.Buffers; +using System.Runtime.InteropServices; +using System.Text; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native.Writing +{ + /// + /// fw_write_file and fw_write_memory. Every line has the same length, so the output size and + /// each batch's offset are known up front: batches format in parallel straight into place, and the bytes + /// equal a serial write. + /// + internal static unsafe class WriteApi + { + internal static int WriteMemory(NativeTableRaw* table, NativeLayout layout, NativeWriteOptionsRaw* options, NativeBufferRaw* outBuffer) + { + *outBuffer = default; + int status = Prepare(table, layout, options, out var formatter, out var settings, out long lineBytes); + if (status != NativeStatus.Ok) + { + return status; + } + long total = formatter!.Rows * lineBytes; + var data = (byte*)Marshal.AllocHGlobal((nint)Math.Max(1, total)); + status = Run(formatter, settings!, lineBytes, (first, count) => + { + FormatBatch(formatter, settings!, layout, first, new Span(data + (first * lineBytes), (int)(count * lineBytes))); + }); + if (status != NativeStatus.Ok) + { + Marshal.FreeHGlobal((IntPtr)data); + return status; + } + outBuffer->Data = data; + outBuffer->Length = total; + return NativeStatus.Ok; + } + + internal static int WriteFile(ReadOnlySpan path, NativeTableRaw* table, NativeLayout layout, NativeWriteOptionsRaw* options) + { + int status = Prepare(table, layout, options, out var formatter, out var settings, out long lineBytes); + if (status != NativeStatus.Ok) + { + return status; + } + try + { + long total = formatter!.Rows * lineBytes; + using var handle = File.OpenHandle(Encoding.UTF8.GetString(path), FileMode.Create, FileAccess.Write, FileShare.None, FileOptions.None, total); + RandomAccess.SetLength(handle, total); + return Run(formatter, settings!, lineBytes, (first, count) => WriteBatch(handle, formatter, settings!, layout, first, count, lineBytes)); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException) + { + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + + internal static void FreeBuffer(NativeBufferRaw* buffer) + { + if (buffer is null) + { + return; + } + Marshal.FreeHGlobal((IntPtr)buffer->Data); + *buffer = default; + } + + private static int Prepare(NativeTableRaw* table, NativeLayout layout, NativeWriteOptionsRaw* options, + out TableFormatter? formatter, out WriteSettings? settings, out long lineBytes) + { + formatter = null; + lineBytes = 0; + int status = WriteSettings.TryCreate(options, out settings); + if (status != NativeStatus.Ok) + { + return status; + } + lineBytes = layout.LineLength + settings!.NewLine.Length; + return TableFormatter.TryCreate(table, layout, settings.Encoding, out formatter); + } + + private static void FormatBatch(TableFormatter formatter, WriteSettings settings, NativeLayout layout, long first, Span destination) + { + int lineLength = layout.LineLength; + int lineBytes = lineLength + settings.NewLine.Length; + for (int at = 0; at < destination.Length; at += lineBytes) + { + formatter.FormatRow(first + (at / lineBytes), destination.Slice(at, lineLength)); + settings.NewLine.CopyTo(destination.Slice(at + lineLength)); + } + } + + private static void WriteBatch(Microsoft.Win32.SafeHandles.SafeFileHandle handle, TableFormatter formatter, WriteSettings settings, NativeLayout layout, + long first, long count, long lineBytes) + { + int length = (int)(count * lineBytes); + byte[] buffer = ArrayPool.Shared.Rent(length); + try + { + FormatBatch(formatter, settings, layout, first, buffer.AsSpan(0, length)); + RandomAccess.Write(handle, buffer.AsSpan(0, length), first * lineBytes); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + /// Runs over every batch of rows, in parallel, and turns a failure into a status. + private static int Run(TableFormatter formatter, WriteSettings settings, long lineBytes, Action batch) + { + long rows = formatter.Rows; + long batchRows = Math.Max(1, Math.Min(settings.BatchRows, int.MaxValue / lineBytes)); + long batches = (rows + batchRows - 1) / batchRows; + try + { + if (settings.DegreeOfParallelism == 1) + { + for (long b = 0; b < batches; b++) + { + batch(b * batchRows, Math.Min(batchRows, rows - (b * batchRows))); + } + return NativeStatus.Ok; + } + var parallel = new ParallelOptions { MaxDegreeOfParallelism = settings.DegreeOfParallelism }; + Parallel.For(0, batches, parallel, b => batch(b * batchRows, Math.Min(batchRows, rows - (b * batchRows)))); + return NativeStatus.Ok; + } + catch (AggregateException ex) + { + NativeApi.SetLastError(ex.InnerExceptions[0].Message); + return NativeStatus.Error; + } + catch (Exception ex) + { + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + } +} diff --git a/src/FixedWidthParser.Native/Writing/WriteSettings.cs b/src/FixedWidthParser.Native/Writing/WriteSettings.cs new file mode 100644 index 0000000..e7a137f --- /dev/null +++ b/src/FixedWidthParser.Native/Writing/WriteSettings.cs @@ -0,0 +1,72 @@ +using System.Runtime.InteropServices; +using System.Text; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native.Writing +{ + /// fw_write_options. + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct NativeWriteOptionsRaw + { + public int StructSize; + public int Encoding; + public byte* NewLine; + public int NewLineLen; + public int DegreeOfParallelism; + public long BatchRows; + } + + /// fw_buffer: bytes the library allocated, released with fw_free_buffer. + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct NativeBufferRaw + { + public byte* Data; + public long Length; + } + + /// A validated fw_write_options. + internal sealed class WriteSettings + { + private const long DefaultBatchRows = 8192; + + private WriteSettings(Encoding? encoding, byte[] newLine, int degreeOfParallelism, long batchRows) + { + Encoding = encoding; + NewLine = newLine; + DegreeOfParallelism = degreeOfParallelism == 0 ? Environment.ProcessorCount : degreeOfParallelism; + BatchRows = batchRows == 0 ? DefaultBatchRows : batchRows; + } + + internal Encoding? Encoding { get; } + internal byte[] NewLine { get; } + internal int DegreeOfParallelism { get; } + internal long BatchRows { get; } + + internal static unsafe int TryCreate(NativeWriteOptionsRaw* options, out WriteSettings? settings) + { + settings = null; + int status = NativeOptions.TryRead(options, "fw_write_options", out var raw); + if (status != NativeStatus.Ok) + { + return status; + } + if (raw.DegreeOfParallelism < 0 || raw.BatchRows < 0 || raw.NewLineLen < 0 || (raw.NewLine is null && raw.NewLineLen > 0)) + { + return Fail("fw_write_options: degree_of_parallelism, batch_rows and newline_len must not be negative, and a newline_len needs a newline."); + } + if (!ReadSettings.TryEncoding(raw.Encoding, out var encoding)) + { + return Fail($"fw_write_options: encoding {raw.Encoding} is not a code page this library knows."); + } + byte[] newLine = raw.NewLine is null ? [(byte)'\n'] : new ReadOnlySpan(raw.NewLine, raw.NewLineLen).ToArray(); + settings = new WriteSettings(encoding, newLine, raw.DegreeOfParallelism, raw.BatchRows); + return NativeStatus.Ok; + } + + private static int Fail(string message) + { + NativeApi.SetLastError(message); + return NativeStatus.InvalidArgument; + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs b/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs index a8a988b..f0e6697 100644 --- a/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs +++ b/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs @@ -60,6 +60,13 @@ internal static NativeLayout Build(params Col[] columns) return layout!; } + internal static NativeLayout Build(NativeLayoutOptionsRaw options, params Col[] columns) + { + int status = Create(out var layout, options, columns); + Assert.True(status == NativeStatus.Ok, $"status {status}: {LastError()}"); + return layout!; + } + internal static NativeLayoutOptionsRaw LayoutOptions(int lineLength = 0, int filler = 0) { return new NativeLayoutOptionsRaw { StructSize = sizeof(NativeLayoutOptionsRaw), LineLength = lineLength, Filler = filler }; diff --git a/tests/FixedWidthParser.Tests/Native/NativeWriteTests.cs b/tests/FixedWidthParser.Tests/Native/NativeWriteTests.cs new file mode 100644 index 0000000..43a5685 --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeWriteTests.cs @@ -0,0 +1,274 @@ +using System.Runtime.InteropServices; +using System.Text; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; +using FixedWidthParser.Native.Writing; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; +using static FixedWidthParser.Tests.Native.NativeTestReads; + +namespace FixedWidthParser.Tests.Native +{ + public sealed unsafe class NativeWriteTests + { + private const string EveryType = + "Ann " + "00042" + " 12.500" + "00012345" + "20231024" + "133005" + "20231024133000" + "\n" + + " " + " " + " -0.5" + "-0000001" + "19700101" + "000000" + "19700101000000" + "\n"; + + private static NativeLayout EveryTypeLayout() + { + return Build( + new Col("name", 0, 6, TString) { Nullable = true }, + new Col("qty", 6, 5, TInt64) { Nullable = true }, + new Col("price", 11, 8, TFloat64), + new Col("amount", 19, 8, TDecimal) { Scale = 2 }, + new Col("day", 27, 8, TDate) { Format = "yyyyMMdd" }, + new Col("time", 35, 6, TTime) { Format = "HHmmss" }, + new Col("stamp", 41, 14, TTimestamp) { Format = "yyyyMMddHHmmss" }); + } + + private static NativeWriteOptionsRaw WriteOptions(int degree = 1, long batchRows = 0, int encoding = 0) + { + return new NativeWriteOptionsRaw { StructSize = sizeof(NativeWriteOptionsRaw), DegreeOfParallelism = degree, BatchRows = batchRows, Encoding = encoding }; + } + + /// Reads into a table the caller frees. + private static NativeTableRaw ReadTable(byte[] content, NativeLayout layout, NativeReadOptionsRaw? options = null) + { + NativeTableRaw table = default; + NativeReadOptionsRaw value = options ?? default; + fixed (byte* data = content) + { + int status = ReadApi.ReadMemory(data, content.Length, layout, options is null ? null : &value, &table, null); + Assert.True(status == NativeStatus.Ok, LastError()); + } + return table; + } + + private static (int Status, byte[] Bytes, string Error) WriteMemory(NativeTableRaw* table, NativeLayout layout, NativeWriteOptionsRaw? options = null) + { + NativeWriteOptionsRaw value = options ?? default; + NativeBufferRaw buffer = default; + int status = WriteApi.WriteMemory(table, layout, options is null ? null : &value, &buffer); + byte[] bytes = buffer.Data is null ? [] : new ReadOnlySpan(buffer.Data, (int)buffer.Length).ToArray(); + WriteApi.FreeBuffer(&buffer); + return (status, bytes, status == NativeStatus.Ok ? "" : LastError()); + } + + private static byte[] RoundTrip(string content, NativeLayout readLayout, NativeLayout writeLayout, NativeWriteOptionsRaw? options = null) + { + var table = ReadTable(Encoding.UTF8.GetBytes(content), readLayout); + try + { + var (status, bytes, error) = WriteMemory(&table, writeLayout, options); + Assert.True(status == NativeStatus.Ok, error); + return bytes; + } + finally + { + NativeTable.Free(&table); + } + } + + [Fact] + public void Write_EveryType_RoundTripsThroughRead() + { + var layout = EveryTypeLayout(); + var original = FromMemory(Encoding.UTF8.GetBytes(EveryType), layout); + + byte[] written = RoundTrip(EveryType, layout, layout); + var reread = FromMemory(written, layout); + + Assert.Equal(original.RowCount, reread.RowCount); + for (int i = 0; i < 7; i++) + { + Assert.Equal(original.Column(i), reread.Column(i)); + } + Assert.Equal(2 * 56, written.Length); + } + + [Fact] + public void Write_ScaledZeroPadded_RoundTrips() + { + var layout = Build(new Col("preabe", 0, 13, TFloat64) { Scale = 2, Padding = '0' }, new Col("voltot", 13, 18, TDecimal) { Scale = 2, Padding = '0' }); + const string Content = + "0000000123456" + "000000000000123456" + "\n" + + "-000000012345" + "-00000000000000001" + "\n" + + "0000000000000" + "000000000000000000" + "\n"; + + Assert.Equal(Content, Encoding.UTF8.GetString(RoundTrip(Content, layout, layout))); + } + + [Fact] + public void Write_Null_FillsWithTheTrimCharacter() + { + var layout = Build(new Col("n", 0, 4, TInt64) { Nullable = true, TrimChar = '0', Padding = '0' }, new Col("s", 4, 3, TString) { Nullable = true }); + + Assert.Equal("0012abc\n0000 \n", Encoding.UTF8.GetString(RoundTrip("0012abc\n0000 \n", layout, layout))); + } + + [Fact] + public void Write_StringTruncate_KeepsWholeCharacters() + { + var read = Build(new Col("s", 0, 7, TString)); + var write = Build(new Col("s", 0, 2, TString) { Overflow = 1 }); + + Assert.Equal("A \n", Encoding.UTF8.GetString(RoundTrip("Açores\n", read, write))); + } + + [Fact] + public void Write_StringOverflowThrow_IsError() + { + var read = Build(new Col("s", 0, 6, TString)); + var write = Build(new Col("s", 0, 3, TString)); + var table = ReadTable("ABCDEF\n"u8.ToArray(), read); + try + { + var (status, _, error) = WriteMemory(&table, write); + + Assert.Equal(NativeStatus.Error, status); + Assert.Contains("exceeds the width 3", error, StringComparison.Ordinal); + } + finally + { + NativeTable.Free(&table); + } + } + + [Fact] + public void Write_NumberOverflow_IsError() + { + var read = Build(new Col("n", 0, 5, TInt64)); + var write = Build(new Col("n", 0, 2, TInt64)); + var table = ReadTable("12345\n"u8.ToArray(), read); + try + { + Assert.Equal(NativeStatus.Error, WriteMemory(&table, write).Status); + } + finally + { + NativeTable.Free(&table); + } + } + + [Fact] + public void Write_TypeMismatch_IsInvalidArgument() + { + var read = Build(new Col("n", 0, 5, TInt64)); + var table = ReadTable("12345\n"u8.ToArray(), read); + try + { + var (status, _, error) = WriteMemory(&table, Build(new Col("n", 0, 5, TString))); + + Assert.Equal(NativeStatus.InvalidArgument, status); + Assert.Contains("'n'", error, StringComparison.Ordinal); + Assert.Equal(NativeStatus.InvalidArgument, WriteMemory(&table, Build(new Col("a", 0, 2, TInt64), new Col("b", 2, 2, TInt64))).Status); + } + finally + { + NativeTable.Free(&table); + } + } + + public static TheoryData Batches() + { + return new TheoryData { { 1, 1 }, { 1, 3 }, { 4, 1 }, { 4, 3 }, { 4, 8192 }, { 0, 0 } }; + } + + [Theory] + [MemberData(nameof(Batches))] + public void Write_Parallel_IsByteIdenticalToSerial(int degree, long batchRows) + { + var layout = EveryTypeLayout(); + string content = string.Concat(Enumerable.Repeat(EveryType, 40)); + + byte[] serial = RoundTrip(content, layout, layout, WriteOptions(degree: 1)); + byte[] parallel = RoundTrip(content, layout, layout, WriteOptions(degree, batchRows)); + + Assert.Equal(serial, parallel); + } + + [Fact] + public void Write_File_EqualsMemory() + { + var layout = EveryTypeLayout(); + var table = ReadTable(Encoding.UTF8.GetBytes(EveryType), layout); + string path = Path.GetTempFileName(); + try + { + var options = WriteOptions(degree: 4, batchRows: 1); + Assert.Equal(NativeStatus.Ok, WriteApi.WriteFile(Encoding.UTF8.GetBytes(path), &table, layout, &options)); + + Assert.Equal(WriteMemory(&table, layout).Bytes, File.ReadAllBytes(path)); + } + finally + { + NativeTable.Free(&table); + File.Delete(path); + } + } + + [Fact] + public void Write_Latin1_EncodesText() + { + var layout = Build(new Col("s", 0, 6, TString)); + + byte[] written = RoundTrip("AÇÃO \n", layout, layout, WriteOptions(encoding: 28591)); + + Assert.Equal(Encoding.Latin1.GetBytes("AÇÃO \n"), written); + } + + [Fact] + public void Write_NewLineAndFiller_AreHonored() + { + var read = Build(new Col("a", 0, 2, TString), new Col("b", 2, 2, TString)); + var write = Build(LayoutOptions(lineLength: 8, filler: '.'), new Col("a", 0, 2, TString), new Col("b", 4, 2, TString)); + byte[] newline = "\r\n"u8.ToArray(); + fixed (byte* nl = newline) + { + var options = WriteOptions() with { NewLine = nl, NewLineLen = 2 }; + + Assert.Equal("ab..cd..\r\n", Encoding.UTF8.GetString(RoundTrip("abcd\n", read, write, options))); + } + } + + [Fact] + public void Write_TemporalWithoutFormat_IsIso() + { + var read = Build(new Col("d", 0, 8, TDate) { Format = "yyyyMMdd" }); + var write = Build(new Col("d", 0, 10, TDate)); + + Assert.Equal("2023-10-24\n", Encoding.UTF8.GetString(RoundTrip("20231024\n", read, write))); + } + + [Fact] + public void Write_EmptyTable_IsEmpty() + { + var layout = Build(new Col("a", 0, 2, TString)); + + Assert.Empty(RoundTrip("", layout, layout)); + } + + [Fact] + public void Write_BadOptionsStructSize_IsInvalidArgument() + { + var layout = Build(new Col("a", 0, 2, TString)); + var table = ReadTable("ab\n"u8.ToArray(), layout); + try + { + Assert.Equal(NativeStatus.InvalidArgument, WriteMemory(&table, layout, WriteOptions() with { StructSize = 4 }).Status); + } + finally + { + NativeTable.Free(&table); + } + } + + [Fact] + public void BufferStruct_HasTheHeaderLayout() + { + Assert.Equal(16, Marshal.SizeOf()); + Assert.Equal(32, Marshal.SizeOf()); + } + } +} From 7629e848a976c3ea9cb64530de3e2c37a2e4721b Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 01:29:48 -0300 Subject: [PATCH 41/85] feat(native): C header and smoke test include/fixedwidth.h and fixedwidth_arrow.h declare every export with its ownership, threading and struct_size contract. The NativeSmoke C program loads the published library dynamically, pins every struct's offsets and size with static asserts, and exercises layouts, reads, batches, Arrow, writes, parse errors and argument misuse. --- .gitignore | 3 + .../include/fixedwidth.h | 209 ++++++++++ .../include/fixedwidth_arrow.h | 88 +++++ .../CMakeLists.txt | 39 ++ tests/FixedWidthParser.NativeSmoke/README.md | 15 + tests/FixedWidthParser.NativeSmoke/smoke.c | 364 ++++++++++++++++++ 6 files changed, 718 insertions(+) create mode 100644 src/FixedWidthParser.Native/include/fixedwidth.h create mode 100644 src/FixedWidthParser.Native/include/fixedwidth_arrow.h create mode 100644 tests/FixedWidthParser.NativeSmoke/CMakeLists.txt create mode 100644 tests/FixedWidthParser.NativeSmoke/README.md create mode 100644 tests/FixedWidthParser.NativeSmoke/smoke.c diff --git a/.gitignore b/.gitignore index 0808c4a..a98beaf 100644 --- a/.gitignore +++ b/.gitignore @@ -480,3 +480,6 @@ $RECYCLE.BIN/ # Vim temporary swap files *.swp + +# CMake build trees (native smoke and C++ tests) +build/ diff --git a/src/FixedWidthParser.Native/include/fixedwidth.h b/src/FixedWidthParser.Native/include/fixedwidth.h new file mode 100644 index 0000000..c7d93ef --- /dev/null +++ b/src/FixedWidthParser.Native/include/fixedwidth.h @@ -0,0 +1,209 @@ +/* FixedWidthParser C ABI: read and write fixed-width files into a columnar table. + * + * Conventions: + * - Every function returns an int32_t status: FW_OK (0) or a negative FW_* code. On failure, + * fw_last_error() gives the calling thread's message; after FW_PARSE_ERROR, fw_last_parse_error() + * also gives the 1-based line number and the layout column. + * - Strings crossing the ABI are UTF-8 pointer + length, never NUL-terminated. + * - Every options struct starts with int32_t struct_size, which the caller sets to sizeof(the struct). + * A NULL options pointer, or a zeroed struct with struct_size set, means every default. Any other + * struct_size is refused with FW_INVALID_ARGUMENT. New fields only ever arrive at the end. + * - Handles (fw_layout, fw_reader) are opaque. A freed handle stays invalid (FW_INVALID_HANDLE); it + * is never reissued. + * - Threads: a layout is immutable and may be used by several threads at once. A reader is used by + * one thread at a time. Reads and writes may run concurrently on different tables. + * - Column positions and lengths are bytes. Text is decoded with the read encoding and handed out as + * UTF-8; numbers and dates use the invariant culture. + * + * Call fw_abi_version() first and refuse to continue when it differs from FW_ABI_VERSION. */ +#ifndef FIXEDWIDTH_H +#define FIXEDWIDTH_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define FW_ABI_VERSION 1 + +#define FW_OK 0 +#define FW_EOF -1 +#define FW_BUFFER_TOO_SMALL -2 +#define FW_INVALID_HANDLE -3 +#define FW_INVALID_ARGUMENT -4 +#define FW_ERROR -5 +#define FW_PARSE_ERROR -6 + +/* Column types and their fw_column values. */ +#define FW_T_STRING 0 /* int32 offsets (length + 1) into data, UTF-8 */ +#define FW_T_INT64 1 /* int64 */ +#define FW_T_FLOAT64 2 /* double */ +#define FW_T_DECIMAL 3 /* int64 unscaled value: 10^-scale per unit; at most 18 digits */ +#define FW_T_DATE 4 /* int32 days since 1970-01-01 */ +#define FW_T_TIME 5 /* int64 microseconds since midnight */ +#define FW_T_TIMESTAMP 6 /* int64 microseconds since 1970-01-01T00:00:00, no time zone */ + +#define FW_TRIM_BOTH 0 +#define FW_TRIM_TRAILING 1 +#define FW_TRIM_LEADING 2 + +#define FW_ALIGN_DEFAULT 0 /* left for STRING, right for everything else */ +#define FW_ALIGN_LEFT 1 +#define FW_ALIGN_RIGHT 2 + +#define FW_OVERFLOW_THROW 0 /* a value wider than its column fails the write */ +#define FW_OVERFLOW_TRUNCATE 1 /* STRING only: keep the aligned side, whole characters */ + +#define FW_OPT_DEFAULT 0 +#define FW_OPT_FALSE 1 +#define FW_OPT_TRUE 2 + +int32_t fw_abi_version(void); + +/* Copies the calling thread's last error message into buffer (not NUL-terminated) and its length into + * *out_len. FW_BUFFER_TOO_SMALL when capacity is short; *out_len is then the length needed. */ +int32_t fw_last_error(uint8_t* buffer, int32_t capacity, int32_t* out_len); + +typedef struct fw_parse_error { + int32_t struct_size; + int64_t line_number; /* 1-based, counting every line of the source as a sequential read does */ + int32_t column; /* index in the layout; -1 when the line is shorter than the layout */ +} fw_parse_error; + +/* The parts of the calling thread's last FW_PARSE_ERROR. */ +int32_t fw_last_parse_error(fw_parse_error* out_error); + +/* ---- Layouts ---------------------------------------------------------------------------------- */ + +typedef struct fw_layout fw_layout; + +typedef struct fw_column_spec { + int32_t struct_size; /* sizeof(fw_column_spec); the same in every element */ + const uint8_t* name; /* UTF-8, required */ + int32_t name_len; + int32_t start; /* zero-based byte offset */ + int32_t length; /* bytes, at least 1 */ + int32_t type; /* FW_T_* */ + int32_t nullable; /* a blank column (only trim_char and spaces) reads as null */ + int32_t scale; /* 0..18: FLOAT64 divides digits-only text by 10^scale; DECIMAL records it */ + const uint8_t* format; /* exact date/time format, or a number format on write; NULL for none */ + int32_t format_len; + int32_t trim_char; /* ASCII; 0 = space. A null is written as this character. */ + int32_t trim_mode; /* FW_TRIM_* */ + int32_t alignment; /* FW_ALIGN_*, on write */ + int32_t padding; /* ASCII; 0 = space, on write. '0' keeps a sign leftmost. */ + int32_t overflow; /* FW_OVERFLOW_*, on write */ +} fw_column_spec; + +typedef struct fw_layout_options { + int32_t struct_size; + int32_t line_length; /* 0 = the end of the last column; longer pads lines with filler on write */ + int32_t filler; /* ASCII written in gaps between columns; 0 = space */ +} fw_layout_options; + +/* Validates the columns (positions, overlaps, types, scale, formats, ASCII options) and creates an + * immutable layout. The specs and their strings may be freed after the call. Free with fw_layout_free; + * readers and Arrow streams opened from it keep working after that. */ +int32_t fw_layout_create(const fw_column_spec* columns, int32_t column_count, + const fw_layout_options* options, fw_layout** out_layout); +void fw_layout_free(fw_layout* layout); + +/* ---- Reading ---------------------------------------------------------------------------------- */ + +typedef struct fw_read_options { + int32_t struct_size; + int64_t header_lines; /* lines skipped first, never parsed; counted by line breaks */ + int32_t skip_invalid; /* a line that fails to parse is skipped instead of an error */ + int32_t degree_of_parallelism; /* 0 = one thread per processor, 1 = sequential */ + int32_t encoding; /* code page of the text; 0 = UTF-8, 28591 = Latin-1, 1252, ... */ + int32_t max_line_length; /* bytes; 0 = 1 MiB. A longer line fails with FW_ERROR. */ + int32_t skip_byte_order_mark; /* FW_OPT_*; by default only a UTF-8 read skips a leading BOM */ +} fw_read_options; + +typedef struct fw_column { + int32_t type; /* FW_T_* */ + int32_t scale; + int64_t length; /* rows */ + const void* values; /* see FW_T_*; may be NULL when length is 0 */ + const uint8_t* validity; /* LSB-first bitmap, 1 = valid; NULL when nothing is null */ + const uint8_t* data; /* STRING bytes; NULL otherwise or when empty */ + int64_t data_len; +} fw_column; + +typedef struct fw_table { + int32_t column_count; + int64_t row_count; + fw_column* columns; /* in layout order */ +} fw_table; + +typedef struct fw_read_result { + int32_t struct_size; + int64_t lines_read; /* rows in the table */ + int64_t lines_skipped; /* header lines plus invalid lines skipped; empty lines are not counted */ +} fw_read_result; + +/* Reads a whole file or buffer into *out_table, which the caller frees with fw_free_table. out_result + * may be NULL. A read with degree_of_parallelism other than 1 splits the source into byte ranges and + * gives the same table and error line numbers as a sequential read. fw_read_memory does not keep + * data past the call. A string column holding more than 2 GiB of text fails: read in batches. */ +int32_t fw_read_file(const uint8_t* path, int32_t path_len, const fw_layout* layout, + const fw_read_options* options, fw_table* out_table, fw_read_result* out_result); +int32_t fw_read_memory(const uint8_t* data, int64_t data_len, const fw_layout* layout, + const fw_read_options* options, fw_table* out_table, fw_read_result* out_result); + +/* Frees a table the library published and zeroes it. Safe on a zeroed table. */ +void fw_free_table(fw_table* table); + +typedef struct fw_reader fw_reader; + +typedef struct fw_reader_options { + int32_t struct_size; + fw_read_options read; /* read.struct_size 0 means its defaults; degree_of_parallelism is ignored */ + int64_t batch_rows; /* rows per table; 0 = 65,536 */ +} fw_reader_options; + +/* A sequential reader handing out tables of at most batch_rows rows. fw_reader_open_memory reads data + * in place: keep it alive until fw_reader_close. */ +int32_t fw_reader_open_file(const uint8_t* path, int32_t path_len, const fw_layout* layout, + const fw_reader_options* options, fw_reader** out_reader); +int32_t fw_reader_open_memory(const uint8_t* data, int64_t data_len, const fw_layout* layout, + const fw_reader_options* options, fw_reader** out_reader); + +/* FW_OK with a table the caller frees with fw_free_table, FW_EOF at the end. After a failure the + * reader is finished: later calls return FW_EOF. */ +int32_t fw_reader_next(fw_reader* reader, fw_table* out_table); +void fw_reader_close(fw_reader* reader); + +/* ---- Writing ---------------------------------------------------------------------------------- */ + +typedef struct fw_write_options { + int32_t struct_size; + int32_t encoding; /* code page for text; 0 = UTF-8 */ + const uint8_t* newline; /* NULL = "\n" */ + int32_t newline_len; + int32_t degree_of_parallelism; /* 0 = one thread per processor, 1 = serial */ + int64_t batch_rows; /* lines formatted per task; 0 = 8,192 */ +} fw_write_options; + +typedef struct fw_buffer { + uint8_t* data; + int64_t length; +} fw_buffer; + +/* Writes every row of table through layout. Columns match the layout by position and type (and scale + * for DECIMAL), else FW_INVALID_ARGUMENT. A null writes as its column filled with trim_char; a date or + * time without a format writes as ISO 8601 to the second. The output is byte-identical whatever + * degree_of_parallelism is. The table is read from several threads and must not change during the + * call. fw_write_memory's buffer is freed with fw_free_buffer. */ +int32_t fw_write_file(const uint8_t* path, int32_t path_len, const fw_table* table, + const fw_layout* layout, const fw_write_options* options); +int32_t fw_write_memory(const fw_table* table, const fw_layout* layout, + const fw_write_options* options, fw_buffer* out_buffer); +void fw_free_buffer(fw_buffer* buffer); + +#ifdef __cplusplus +} +#endif + +#endif /* FIXEDWIDTH_H */ diff --git a/src/FixedWidthParser.Native/include/fixedwidth_arrow.h b/src/FixedWidthParser.Native/include/fixedwidth_arrow.h new file mode 100644 index 0000000..aa8c46a --- /dev/null +++ b/src/FixedWidthParser.Native/include/fixedwidth_arrow.h @@ -0,0 +1,88 @@ +/* Optional Arrow C Data Interface export for FixedWidthParser. Separate from fixedwidth.h so a caller + * that does not want Arrow never pulls these declarations in. + * + * struct ArrowSchema, ArrowArray and ArrowArrayStream are the Arrow C Data and C Stream Interface + * structs, reproduced from the Apache Arrow specification (Apache License 2.0). */ +#ifndef FIXEDWIDTH_ARROW_H +#define FIXEDWIDTH_ARROW_H + +#include "fixedwidth.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef ARROW_C_DATA_INTERFACE +#define ARROW_C_DATA_INTERFACE + +#define ARROW_FLAG_DICTIONARY_ORDERED 1 +#define ARROW_FLAG_NULLABLE 2 +#define ARROW_FLAG_MAP_KEYS_SORTED 4 + +struct ArrowSchema { + const char* format; + const char* name; + const char* metadata; + int64_t flags; + int64_t n_children; + struct ArrowSchema** children; + struct ArrowSchema* dictionary; + + void (*release)(struct ArrowSchema*); + void* private_data; +}; + +struct ArrowArray { + int64_t length; + int64_t null_count; + int64_t offset; + int64_t n_buffers; + int64_t n_children; + const void** buffers; + struct ArrowArray** children; + struct ArrowArray* dictionary; + + void (*release)(struct ArrowArray*); + void* private_data; +}; + +#endif /* ARROW_C_DATA_INTERFACE */ + +#ifndef ARROW_C_STREAM_INTERFACE +#define ARROW_C_STREAM_INTERFACE + +struct ArrowArrayStream { + int (*get_schema)(struct ArrowArrayStream*, struct ArrowSchema* out); + int (*get_next)(struct ArrowArrayStream*, struct ArrowArray* out); + const char* (*get_last_error)(struct ArrowArrayStream*); + void (*release)(struct ArrowArrayStream*); + void* private_data; +}; + +#endif /* ARROW_C_STREAM_INTERFACE */ + +/* A struct array ("+s"), one child per layout column, named from the layout and flagged nullable when + * the column is: STRING "u", INT64 "l", FLOAT64 "g", DECIMAL "d:18,", DATE "tdD", + * TIME "ttu", TIMESTAMP "tsu:". The column buffers move into the arrays without a copy, except + * DECIMAL, widened to 128 bits. A parallel read (degree_of_parallelism other than 1) is concatenated + * into one array. Release both with their release callbacks. */ +int32_t fw_read_arrow_file(const uint8_t* path, int32_t path_len, const fw_layout* layout, + const fw_read_options* options, + struct ArrowArray* out_array, struct ArrowSchema* out_schema, fw_read_result* out_result); +int32_t fw_read_arrow_memory(const uint8_t* data, int64_t data_len, const fw_layout* layout, + const fw_read_options* options, + struct ArrowArray* out_array, struct ArrowSchema* out_schema, fw_read_result* out_result); + +/* The batches of a sequential reader (fw_reader_options) as an ArrowArrayStream; a failure makes + * get_next return a non-zero errno and get_last_error describe it. fw_read_arrow_stream_memory reads + * data lazily: keep it alive until the stream is released. */ +int32_t fw_read_arrow_stream_file(const uint8_t* path, int32_t path_len, const fw_layout* layout, + const fw_reader_options* options, struct ArrowArrayStream* out_stream); +int32_t fw_read_arrow_stream_memory(const uint8_t* data, int64_t data_len, const fw_layout* layout, + const fw_reader_options* options, struct ArrowArrayStream* out_stream); + +#ifdef __cplusplus +} +#endif + +#endif /* FIXEDWIDTH_ARROW_H */ diff --git a/tests/FixedWidthParser.NativeSmoke/CMakeLists.txt b/tests/FixedWidthParser.NativeSmoke/CMakeLists.txt new file mode 100644 index 0000000..8e7b8de --- /dev/null +++ b/tests/FixedWidthParser.NativeSmoke/CMakeLists.txt @@ -0,0 +1,39 @@ +# Builds and runs smoke.c against the published FixedWidthParser.Native shared library. The library is +# loaded dynamically (see smoke.c), so nothing links against it at build time. +# +# Usage, from the repository root, after `python python/scripts/build_native.py`: +# cmake -S tests/FixedWidthParser.NativeSmoke -B build/nativesmoke +# cmake --build build/nativesmoke --config Release +# ctest --test-dir build/nativesmoke --output-on-failure +cmake_minimum_required(VERSION 3.16) +project(FixedWidthParserNativeSmoke C) + +set(FIXEDWIDTH_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../..") + +if(WIN32) + set(_lib_name "FixedWidthParser.Native.dll") +elseif(APPLE) + set(_lib_name "FixedWidthParser.Native.dylib") +else() + set(_lib_name "FixedWidthParser.Native.so") +endif() + +# build_native.py publishes the library and copies it to this OS-normalized path for the Python package. +set(FIXEDWIDTH_LIB_PATH "${FIXEDWIDTH_ROOT}/python/src/fixedwidth/_lib/${_lib_name}" + CACHE FILEPATH "Path to the published FixedWidthParser.Native shared library") + +add_executable(fw_smoke smoke.c) +target_include_directories(fw_smoke PRIVATE "${FIXEDWIDTH_ROOT}/src/FixedWidthParser.Native/include") +target_compile_definitions(fw_smoke PRIVATE FIXEDWIDTH_LIB_PATH_DEFAULT="${FIXEDWIDTH_LIB_PATH}") +if(MSVC) + target_compile_options(fw_smoke PRIVATE /W4 /WX) +else() + target_compile_options(fw_smoke PRIVATE -Wall -Wextra -Werror) +endif() +if(NOT WIN32) + target_link_libraries(fw_smoke PRIVATE ${CMAKE_DL_LIBS}) +endif() + +enable_testing() +add_test(NAME fw_smoke COMMAND fw_smoke "${FIXEDWIDTH_LIB_PATH}" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") +set_tests_properties(fw_smoke PROPERTIES PASS_REGULAR_EXPRESSION "smoke ok") diff --git a/tests/FixedWidthParser.NativeSmoke/README.md b/tests/FixedWidthParser.NativeSmoke/README.md new file mode 100644 index 0000000..fd045f8 --- /dev/null +++ b/tests/FixedWidthParser.NativeSmoke/README.md @@ -0,0 +1,15 @@ +# FixedWidthParser.NativeSmoke + +The reference C consumer of `FixedWidthParser.Native`. `smoke.c` loads the published library dynamically +(`LoadLibraryA`/`dlopen`: NativeAOT ships no import library on Windows), pins the offset and size of every +ABI struct with static asserts, and calls every export: layouts, whole and parallel reads, batches, Arrow +arrays and streams, writes, parse errors, and `NULL`/`struct_size`/freed-handle misuse. + + dotnet publish src/FixedWidthParser.Native -c Release -r + cmake -S tests/FixedWidthParser.NativeSmoke -B build/nativesmoke -DFIXEDWIDTH_LIB_PATH= + cmake --build build/nativesmoke --config Release + ctest --test-dir build/nativesmoke --output-on-failure + +Without `-DFIXEDWIDTH_LIB_PATH`, it uses the copy `python/scripts/build_native.py` puts in +`python/src/fixedwidth/_lib/`. A failing static assert means `include/fixedwidth.h` and the managed structs +disagree. diff --git a/tests/FixedWidthParser.NativeSmoke/smoke.c b/tests/FixedWidthParser.NativeSmoke/smoke.c new file mode 100644 index 0000000..94d680b --- /dev/null +++ b/tests/FixedWidthParser.NativeSmoke/smoke.c @@ -0,0 +1,364 @@ +/* Reference C consumer of the FixedWidthParser native library: loads it dynamically (NativeAOT ships + * no import library on Windows), pins every ABI struct's layout, and exercises every export. + * + * fw_smoke [path to FixedWidthParser.Native.{dll,so,dylib}] + * + * Prints "smoke ok" and exits 0 when everything matches. */ +#include "fixedwidth.h" +#include "fixedwidth_arrow.h" + +#include +#include +#include +#include + +#ifdef _WIN32 +#include +typedef HMODULE fw_lib; +#define FW_OPEN(path) LoadLibraryA(path) +#define FW_SYM(lib, name) ((void*)GetProcAddress(lib, name)) +#else +#include +typedef void* fw_lib; +#define FW_OPEN(path) dlopen(path, RTLD_NOW) +#define FW_SYM(lib, name) dlsym(lib, name) +#endif + +#ifndef FIXEDWIDTH_LIB_PATH_DEFAULT +#define FIXEDWIDTH_LIB_PATH_DEFAULT "FixedWidthParser.Native.dll" +#endif + +#define FW_STATIC_ASSERT(cond, name) typedef char fw_static_assert_##name[(cond) ? 1 : -1] + +/* Every published target is 64-bit with natural alignment; the offsets below follow from that. */ +FW_STATIC_ASSERT(sizeof(void*) == 8, pointer_is_64_bit); +FW_STATIC_ASSERT(offsetof(fw_column_spec, name) == 8, spec_name); +FW_STATIC_ASSERT(offsetof(fw_column_spec, start) == 20, spec_start); +FW_STATIC_ASSERT(offsetof(fw_column_spec, scale) == 36, spec_scale); +FW_STATIC_ASSERT(offsetof(fw_column_spec, format) == 40, spec_format); +FW_STATIC_ASSERT(offsetof(fw_column_spec, overflow) == 68, spec_overflow); +FW_STATIC_ASSERT(sizeof(fw_column_spec) == 72, spec_size); +FW_STATIC_ASSERT(sizeof(fw_layout_options) == 12, layout_options_size); +FW_STATIC_ASSERT(offsetof(fw_read_options, header_lines) == 8, read_header_lines); +FW_STATIC_ASSERT(offsetof(fw_read_options, skip_byte_order_mark) == 32, read_bom); +FW_STATIC_ASSERT(sizeof(fw_read_options) == 40, read_options_size); +FW_STATIC_ASSERT(offsetof(fw_column, length) == 8, column_length); +FW_STATIC_ASSERT(offsetof(fw_column, values) == 16, column_values); +FW_STATIC_ASSERT(offsetof(fw_column, data_len) == 40, column_data_len); +FW_STATIC_ASSERT(sizeof(fw_column) == 48, column_size); +FW_STATIC_ASSERT(offsetof(fw_table, row_count) == 8, table_row_count); +FW_STATIC_ASSERT(sizeof(fw_table) == 24, table_size); +FW_STATIC_ASSERT(sizeof(fw_read_result) == 24, read_result_size); +FW_STATIC_ASSERT(sizeof(fw_parse_error) == 24, parse_error_size); +FW_STATIC_ASSERT(offsetof(fw_reader_options, read) == 8, reader_read); +FW_STATIC_ASSERT(offsetof(fw_reader_options, batch_rows) == 48, reader_batch_rows); +FW_STATIC_ASSERT(sizeof(fw_reader_options) == 56, reader_options_size); +FW_STATIC_ASSERT(offsetof(fw_write_options, newline) == 8, write_newline); +FW_STATIC_ASSERT(offsetof(fw_write_options, batch_rows) == 24, write_batch_rows); +FW_STATIC_ASSERT(sizeof(fw_write_options) == 32, write_options_size); +FW_STATIC_ASSERT(sizeof(fw_buffer) == 16, buffer_size); + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "%s:%d: check failed: %s (%s)\n", __FILE__, __LINE__, #cond, \ + last_error()); \ + exit(1); \ + } \ + } while (0) + +static struct { + int32_t (*abi_version)(void); + int32_t (*last_error)(uint8_t*, int32_t, int32_t*); + int32_t (*last_parse_error)(fw_parse_error*); + int32_t (*layout_create)(const fw_column_spec*, int32_t, const fw_layout_options*, fw_layout**); + void (*layout_free)(fw_layout*); + int32_t (*read_file)(const uint8_t*, int32_t, const fw_layout*, const fw_read_options*, fw_table*, fw_read_result*); + int32_t (*read_memory)(const uint8_t*, int64_t, const fw_layout*, const fw_read_options*, fw_table*, fw_read_result*); + void (*free_table)(fw_table*); + int32_t (*reader_open_memory)(const uint8_t*, int64_t, const fw_layout*, const fw_reader_options*, fw_reader**); + int32_t (*reader_next)(fw_reader*, fw_table*); + void (*reader_close)(fw_reader*); + int32_t (*read_arrow_memory)(const uint8_t*, int64_t, const fw_layout*, const fw_read_options*, + struct ArrowArray*, struct ArrowSchema*, fw_read_result*); + int32_t (*read_arrow_stream_memory)(const uint8_t*, int64_t, const fw_layout*, const fw_reader_options*, + struct ArrowArrayStream*); + int32_t (*write_file)(const uint8_t*, int32_t, const fw_table*, const fw_layout*, const fw_write_options*); + int32_t (*write_memory)(const fw_table*, const fw_layout*, const fw_write_options*, fw_buffer*); + void (*free_buffer)(fw_buffer*); +} fw; + +static char error_text[1024]; + +static const char* last_error(void) +{ + int32_t length = 0; + error_text[0] = 0; + if (fw.last_error != NULL && fw.last_error((uint8_t*)error_text, sizeof(error_text) - 1, &length) == FW_OK) { + error_text[length] = 0; + } + return error_text; +} + +static void* symbol(fw_lib lib, const char* name) +{ + void* address = FW_SYM(lib, name); + if (address == NULL) { + fprintf(stderr, "missing export %s\n", name); + exit(1); + } + return address; +} + +static void load(const char* path) +{ + fw_lib lib = FW_OPEN(path); + if (lib == NULL) { + fprintf(stderr, "cannot load %s\n", path); + exit(1); + } + *(void**)&fw.abi_version = symbol(lib, "fw_abi_version"); + *(void**)&fw.last_error = symbol(lib, "fw_last_error"); + *(void**)&fw.last_parse_error = symbol(lib, "fw_last_parse_error"); + *(void**)&fw.layout_create = symbol(lib, "fw_layout_create"); + *(void**)&fw.layout_free = symbol(lib, "fw_layout_free"); + *(void**)&fw.read_file = symbol(lib, "fw_read_file"); + *(void**)&fw.read_memory = symbol(lib, "fw_read_memory"); + *(void**)&fw.free_table = symbol(lib, "fw_free_table"); + *(void**)&fw.reader_open_memory = symbol(lib, "fw_reader_open_memory"); + *(void**)&fw.reader_next = symbol(lib, "fw_reader_next"); + *(void**)&fw.reader_close = symbol(lib, "fw_reader_close"); + *(void**)&fw.read_arrow_memory = symbol(lib, "fw_read_arrow_memory"); + *(void**)&fw.read_arrow_stream_memory = symbol(lib, "fw_read_arrow_stream_memory"); + *(void**)&fw.write_file = symbol(lib, "fw_write_file"); + *(void**)&fw.write_memory = symbol(lib, "fw_write_memory"); + *(void**)&fw.free_buffer = symbol(lib, "fw_free_buffer"); + symbol(lib, "fw_reader_open_file"); + symbol(lib, "fw_read_arrow_file"); + symbol(lib, "fw_read_arrow_stream_file"); +} + +/* A COTAHIST-like file: a header, quotes, an invalid line and a trailer. */ +static const char CONTENT[] = + "00COTAHIST.2023\n" + "01PETR4 0000000123456000000000000001000\n" + "01VALE3 0000000067890 \n" + "01ITUB4 0000000004321000000000000000050\n" + "99TRAILER\n"; + +static fw_layout* cotahist_layout(void) +{ + fw_column_spec columns[4]; + memset(columns, 0, sizeof(columns)); + const char* names[4] = { "tipreg", "codneg", "preabe", "quatot" }; + const int32_t starts[4] = { 0, 2, 14, 27 }; + const int32_t lengths[4] = { 2, 12, 13, 18 }; + const int32_t types[4] = { FW_T_STRING, FW_T_STRING, FW_T_FLOAT64, FW_T_INT64 }; + for (int i = 0; i < 4; i++) { + columns[i].struct_size = sizeof(fw_column_spec); + columns[i].name = (const uint8_t*)names[i]; + columns[i].name_len = (int32_t)strlen(names[i]); + columns[i].start = starts[i]; + columns[i].length = lengths[i]; + columns[i].type = types[i]; + } + columns[2].scale = 2; + columns[2].padding = '0'; + columns[3].nullable = 1; + columns[3].padding = '0'; + fw_layout* layout = NULL; + CHECK(fw.layout_create(columns, 4, NULL, &layout) == FW_OK); + CHECK(layout != NULL); + return layout; +} + +static fw_read_options read_options(int32_t degree) +{ + fw_read_options options; + memset(&options, 0, sizeof(options)); + options.struct_size = sizeof(options); + options.header_lines = 1; + options.skip_invalid = 1; + options.degree_of_parallelism = degree; + return options; +} + +static void check_table(const fw_table* table) +{ + CHECK(table->column_count == 4); + CHECK(table->row_count == 3); + const int32_t* offsets = (const int32_t*)table->columns[1].values; + CHECK(offsets[1] - offsets[0] == 5 && memcmp(table->columns[1].data, "PETR4", 5) == 0); + const double* prices = (const double*)table->columns[2].values; + CHECK(prices[0] == 1234.56 && prices[2] == 43.21); + const int64_t* quantities = (const int64_t*)table->columns[3].values; + CHECK(quantities[0] == 1000 && quantities[2] == 50); + CHECK(table->columns[3].validity != NULL && (table->columns[3].validity[0] & 2) == 0); +} + +static void reads(fw_layout* layout, const char* path) +{ + fw_read_options options = read_options(1); + fw_table table; + fw_read_result result = { sizeof(fw_read_result), 0, 0 }; + CHECK(fw.read_memory((const uint8_t*)CONTENT, sizeof(CONTENT) - 1, layout, &options, &table, &result) == FW_OK); + check_table(&table); + CHECK(result.lines_read == 3 && result.lines_skipped == 2); + fw.free_table(&table); + CHECK(table.columns == NULL); + + options = read_options(4); + CHECK(fw.read_memory((const uint8_t*)CONTENT, sizeof(CONTENT) - 1, layout, &options, &table, NULL) == FW_OK); + check_table(&table); + fw.free_table(&table); + + FILE* file = fopen(path, "wb"); + CHECK(file != NULL); + fwrite(CONTENT, 1, sizeof(CONTENT) - 1, file); + fclose(file); + options = read_options(1); + CHECK(fw.read_file((const uint8_t*)path, (int32_t)strlen(path), layout, &options, &table, NULL) == FW_OK); + check_table(&table); + fw.free_table(&table); +} + +static void parse_error(fw_layout* layout) +{ + fw_read_options options = read_options(1); + options.skip_invalid = 0; + fw_table table; + CHECK(fw.read_memory((const uint8_t*)CONTENT, sizeof(CONTENT) - 1, layout, &options, &table, NULL) == FW_PARSE_ERROR); + fw_parse_error error = { sizeof(fw_parse_error), 0, 0 }; + CHECK(fw.last_parse_error(&error) == FW_OK); + CHECK(error.line_number == 5 && error.column == -1); + CHECK(strstr(last_error(), "Line 5") != NULL); +} + +static void batches(fw_layout* layout) +{ + fw_reader_options options; + memset(&options, 0, sizeof(options)); + options.struct_size = sizeof(options); + options.read = read_options(0); + options.batch_rows = 2; + fw_reader* reader = NULL; + CHECK(fw.reader_open_memory((const uint8_t*)CONTENT, sizeof(CONTENT) - 1, layout, &options, &reader) == FW_OK); + fw_table table; + int64_t rows = 0; + int32_t status; + while ((status = fw.reader_next(reader, &table)) == FW_OK) { + rows += table.row_count; + fw.free_table(&table); + } + CHECK(status == FW_EOF && rows == 3); + fw.reader_close(reader); + CHECK(fw.reader_next(reader, &table) == FW_INVALID_HANDLE); +} + +static void arrow(fw_layout* layout) +{ + fw_read_options options = read_options(1); + struct ArrowArray array; + struct ArrowSchema schema; + CHECK(fw.read_arrow_memory((const uint8_t*)CONTENT, sizeof(CONTENT) - 1, layout, &options, &array, &schema, NULL) == FW_OK); + CHECK(strcmp(schema.format, "+s") == 0 && schema.n_children == 4); + CHECK(strcmp(schema.children[2]->format, "g") == 0 && strcmp(schema.children[2]->name, "preabe") == 0); + CHECK(array.length == 3 && array.n_children == 4 && array.children[3]->null_count == 1); + CHECK(((const double*)array.children[2]->buffers[1])[0] == 1234.56); + array.release(&array); + schema.release(&schema); + CHECK(array.release == NULL && schema.release == NULL); + + fw_reader_options reader = { sizeof(fw_reader_options), read_options(1), 2 }; + struct ArrowArrayStream stream; + CHECK(fw.read_arrow_stream_memory((const uint8_t*)CONTENT, sizeof(CONTENT) - 1, layout, &reader, &stream) == FW_OK); + CHECK(stream.get_schema(&stream, &schema) == 0); + schema.release(&schema); + int64_t rows = 0; + while (stream.get_next(&stream, &array) == 0 && array.release != NULL) { + rows += array.length; + array.release(&array); + } + CHECK(rows == 3); + stream.release(&stream); +} + +static void writes(fw_layout* layout, const char* path) +{ + fw_read_options read = read_options(1); + fw_table table; + CHECK(fw.read_memory((const uint8_t*)CONTENT, sizeof(CONTENT) - 1, layout, &read, &table, NULL) == FW_OK); + fw_write_options options; + memset(&options, 0, sizeof(options)); + options.struct_size = sizeof(options); + options.degree_of_parallelism = 4; + options.batch_rows = 1; + fw_buffer buffer; + CHECK(fw.write_memory(&table, layout, &options, &buffer) == FW_OK); + const char* first = "01PETR4 0000000123456000000000000001000\n"; + CHECK(buffer.length == 3 * 46 && memcmp(buffer.data, first, 46) == 0); + CHECK(fw.write_file((const uint8_t*)path, (int32_t)strlen(path), &table, layout, &options) == FW_OK); + FILE* file = fopen(path, "rb"); + CHECK(file != NULL); + char written[3 * 46]; + CHECK(fread(written, 1, sizeof(written), file) == sizeof(written)); + fclose(file); + CHECK(memcmp(written, buffer.data, sizeof(written)) == 0); + fw.free_buffer(&buffer); + CHECK(buffer.data == NULL); + fw.free_table(&table); +} + +static void null_arguments(fw_layout* layout) +{ + fw_table table; + fw_layout* other = NULL; + CHECK(fw.layout_create(NULL, 1, NULL, &other) == FW_INVALID_ARGUMENT); + CHECK(fw.read_memory((const uint8_t*)CONTENT, 4, layout, NULL, NULL, NULL) == FW_INVALID_ARGUMENT); + CHECK(fw.read_memory(NULL, 4, layout, NULL, &table, NULL) == FW_INVALID_ARGUMENT); + CHECK(fw.last_error(NULL, 0, NULL) == FW_INVALID_ARGUMENT); + CHECK(fw.last_parse_error(NULL) == FW_INVALID_ARGUMENT); + CHECK(fw.write_memory(NULL, layout, NULL, NULL) == FW_INVALID_ARGUMENT); + + fw_read_options bad = read_options(1); + bad.struct_size = 12; + CHECK(fw.read_memory((const uint8_t*)CONTENT, 4, layout, &bad, &table, NULL) == FW_INVALID_ARGUMENT); + CHECK(strstr(last_error(), "struct_size") != NULL); + + fw_column_spec overlap[2]; + memset(overlap, 0, sizeof(overlap)); + for (int i = 0; i < 2; i++) { + overlap[i].struct_size = sizeof(fw_column_spec); + overlap[i].name = (const uint8_t*)(i == 0 ? "a" : "b"); + overlap[i].name_len = 1; + overlap[i].start = i * 2; + overlap[i].length = 4; + } + CHECK(fw.layout_create(overlap, 2, NULL, &other) == FW_INVALID_ARGUMENT && other == NULL); + CHECK(strstr(last_error(), "Overlapping") != NULL); + + fw.layout_free(layout); + CHECK(fw.read_memory((const uint8_t*)CONTENT, 4, layout, NULL, &table, NULL) == FW_INVALID_HANDLE); + fw.layout_free(layout); + fw.free_table(NULL); + fw.free_buffer(NULL); +} + +int main(int argc, char** argv) +{ + load(argc > 1 ? argv[1] : FIXEDWIDTH_LIB_PATH_DEFAULT); + CHECK(fw.abi_version() == FW_ABI_VERSION); + const char* path = "fw_smoke.txt"; + + fw_layout* layout = cotahist_layout(); + reads(layout, path); + parse_error(layout); + batches(layout); + arrow(layout); + writes(layout, path); + null_arguments(layout); + remove(path); + + printf("smoke ok\n"); + return 0; +} From 5e3ac885fff1004a44c843974a917370605b6956 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 01:40:00 -0300 Subject: [PATCH 42/85] feat(python): fixedwidth package over the native library A ctypes binding (fixedwidthparser-native, import fixedwidth) with Column/Layout and options dataclasses mirroring the C structs. read() returns a Table of NumPy views that keep the native memory alive; read_arrow/read_pandas/read_polars go through the Arrow C Data Interface; read_batches and read_arrow_stream (a PyCapsule stream) read sequentially; write/write_bytes take pandas, polars or pyarrow. build_native.py publishes the library into the package, and the hatch hook tags the wheel per platform, as in ExcelReader. --- .gitignore | 7 + python/README.md | 62 +++++++ python/hatch_build.py | 60 +++++++ python/pyproject.toml | 41 +++++ python/scripts/build_native.py | 75 ++++++++ python/src/fixedwidth/__init__.py | 69 +++++++ python/src/fixedwidth/_lib/.gitkeep | 0 python/src/fixedwidth/_native.py | 248 ++++++++++++++++++++++++++ python/src/fixedwidth/arrow_stream.py | 127 +++++++++++++ python/src/fixedwidth/layout.py | 94 ++++++++++ python/src/fixedwidth/py.typed | 0 python/src/fixedwidth/reader.py | 157 ++++++++++++++++ python/src/fixedwidth/table.py | 135 ++++++++++++++ python/src/fixedwidth/types.py | 136 ++++++++++++++ python/src/fixedwidth/writer.py | 137 ++++++++++++++ python/tests/conftest.py | 40 +++++ python/tests/test_arrow.py | 57 ++++++ python/tests/test_native.py | 33 ++++ python/tests/test_read.py | 127 +++++++++++++ python/tests/test_write.py | 62 +++++++ 20 files changed, 1667 insertions(+) create mode 100644 python/README.md create mode 100644 python/hatch_build.py create mode 100644 python/pyproject.toml create mode 100644 python/scripts/build_native.py create mode 100644 python/src/fixedwidth/__init__.py create mode 100644 python/src/fixedwidth/_lib/.gitkeep create mode 100644 python/src/fixedwidth/_native.py create mode 100644 python/src/fixedwidth/arrow_stream.py create mode 100644 python/src/fixedwidth/layout.py create mode 100644 python/src/fixedwidth/py.typed create mode 100644 python/src/fixedwidth/reader.py create mode 100644 python/src/fixedwidth/table.py create mode 100644 python/src/fixedwidth/types.py create mode 100644 python/src/fixedwidth/writer.py create mode 100644 python/tests/conftest.py create mode 100644 python/tests/test_arrow.py create mode 100644 python/tests/test_native.py create mode 100644 python/tests/test_read.py create mode 100644 python/tests/test_write.py diff --git a/.gitignore b/.gitignore index a98beaf..8409f8c 100644 --- a/.gitignore +++ b/.gitignore @@ -483,3 +483,10 @@ $RECYCLE.BIN/ # CMake build trees (native smoke and C++ tests) build/ + +# Python package +python/.venv/ +python/src/fixedwidth/_lib/* +!python/src/fixedwidth/_lib/.gitkeep +__pycache__/ +.pytest_cache/ diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..3fd5fa9 --- /dev/null +++ b/python/README.md @@ -0,0 +1,62 @@ +# fixedwidthparser-native + +Read and write fixed-width files from Python through FixedWidthParser's NativeAOT library: no .NET runtime, +parallel reads, results as NumPy views, Arrow, pandas or polars. + +```python +import fixedwidth as fw + +# B3's COTAHIST quote history: 245-byte lines, a header (00) and a trailer (99) around the quotes (01). +layout = fw.Layout([ + fw.Column("tipreg", 0, 2, fw.String), + fw.Column("data_pregao", 2, 8, fw.Date, format="yyyyMMdd"), + fw.Column("codneg", 12, 12, fw.String), + fw.Column("preabe", 56, 13, fw.Float64, scale=2), # N(11)V99: two implied decimals + fw.Column("quatot", 152, 18, fw.Int64), +]) +options = fw.ReadOptions(header_lines=1, skip_invalid=True) # the trailer fails to parse and is skipped + +df = fw.read_pandas("COTAHIST_A2023.TXT", layout, options) +``` + +## Reading + +| Function | Returns | +|---|---| +| `fw.read(source, layout, options)` | `fw.Table`: NumPy views over the native columns, no copy | +| `fw.read_arrow(source, layout, options)` | `pyarrow.Table` (needs pyarrow) | +| `fw.read_pandas(source, layout, options)` | `pandas.DataFrame` (needs pyarrow and pandas) | +| `fw.read_polars(source, layout, options)` | `polars.DataFrame` | +| `fw.read_batches(source, layout, fw.ReaderOptions(batch_rows=...))` | an iterator of `fw.Table` | +| `fw.read_arrow_stream(source, layout, fw.ReaderOptions(...))` | an Arrow PyCapsule stream for `polars.DataFrame(...)` or `pyarrow.RecordBatchReader.from_stream(...)` | + +`source` is a path, or `bytes`/`bytearray`/`memoryview` to read from memory. Every optional setting lives in +`fw.ReadOptions`: `header_lines`, `skip_invalid`, `degree_of_parallelism` (0 = one thread per processor, +1 = sequential), `encoding` (a code page or `"latin-1"`, `"cp1252"`, ...), `max_line_length` and +`skip_byte_order_mark`. A parallel read gives the same rows, in the same order, as a sequential one. + +A line that fails to parse raises `fw.ParseError` with `line_number` and `column`, unless `skip_invalid` is set. + +## Columns + +`fw.Column(name, start, length, type, *, nullable, scale, format, trim_char, trim_mode, alignment, padding, overflow)`. +Positions are bytes. Types: `fw.String`, `fw.Int64`, `fw.Float64`, `fw.Decimal` (int64 unscaled, Arrow +`decimal128(18, scale)`), `fw.Date`, `fw.Time`, `fw.Timestamp`. `scale` is the number of implied decimals: +`0000000123456` with `scale=2` reads as `1234.56`. `format` uses .NET date syntax (`yyyyMMdd`, `HHmmss`). + +## Writing + +```python +fw.write(df, "out.txt", layout, fw.WriteOptions(degree_of_parallelism=0)) +data = fw.write_bytes(df, layout) +``` + +Takes pandas, polars, pyarrow or anything with `__arrow_c_stream__` (needs pyarrow), matching columns by name. +`padding="0"` writes zero-padded numbers with the sign leftmost; a null writes as its column filled with +`trim_char`. Parallel writes are byte-identical to serial ones. + +## Building from source + + python python/scripts/build_native.py # publishes the library into src/fixedwidth/_lib + pip install -e "python[dev,pandas,polars]" + pytest python/tests diff --git a/python/hatch_build.py b/python/hatch_build.py new file mode 100644 index 0000000..1a76fb3 --- /dev/null +++ b/python/hatch_build.py @@ -0,0 +1,60 @@ +"""Forces a platform-specific wheel tag: the wheel bundles a native library per OS +(see _lib/), so it isn't pure Python even though every .py file in it is. The package +has no CPython C-extension (it's ctypes calling a NativeAOT shared library), so it's +ABI/interpreter-independent - only the platform component of the tag should be specific. + +The platform component comes from the bundled library, not from the build machine: a +runner's own tag (macosx_26_0, manylinux_2_39) shuts out every older system the library +actually loads on.""" + +import platform +import re +import struct +from pathlib import Path + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface +from packaging.tags import platform_tags + +LIB_DIR = Path(__file__).parent / "src" / "fixedwidth" / "_lib" +LC_BUILD_VERSION = 0x32 + + +def glibc_floor(library: bytes) -> str: + """The newest GLIBC_2.N symbol version the library needs, as manylinux's `2_N`.""" + minors = [int(minor) for minor in re.findall(rb"GLIBC_2\.(\d+)", library)] + return f"2_{max(minors, default=17)}" + + +def macos_floor(library: bytes) -> str: + """The minimum macOS a thin 64-bit Mach-O declares in LC_BUILD_VERSION, as `MAJOR_0`.""" + commands = struct.unpack_from("> 16}_0" + offset += size + raise ValueError("the native library has no LC_BUILD_VERSION load command") + + +def platform_tag() -> str: + machine = platform.machine().lower() + system = platform.system() + if system == "Linux": + library = (LIB_DIR / "FixedWidthParser.Native.so").read_bytes() + if b"libc.musl-" in library: + return next(tag for tag in platform_tags() if tag.startswith("musllinux_")) + return f"manylinux_{glibc_floor(library)}_{machine}" + if system == "Darwin": + library = (LIB_DIR / "FixedWidthParser.Native.dylib").read_bytes() + return f"macosx_{macos_floor(library)}_{machine}" + return next(iter(platform_tags())) + + +class NativeLibraryBuildHook(BuildHookInterface): + def initialize(self, version, build_data): + build_data["pure_python"] = False + build_data["infer_tag"] = True + tag = next(iter(platform_tags())) if version == "editable" else platform_tag() + build_data["tag"] = f"py3-none-{tag}" diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000..3f446b4 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,41 @@ +[build-system] +requires = ["hatchling", "packaging"] +build-backend = "hatchling.build" + +[project] +name = "fixedwidthparser-native" +version = "0.1.0" +description = "Read and write fixed-width files through the FixedWidthParser NativeAOT library" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +authors = [{ name = "Gabriel Matte" }] +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", +] +dependencies = [] + +[project.optional-dependencies] +dev = ["pytest>=7.0"] +numpy = ["numpy>=1.21"] +pyarrow = ["pyarrow>=14.0"] +pandas = ["pandas>=2.0", "pyarrow>=14.0"] +polars = ["polars>=1.0"] + +[project.urls] +Homepage = "https://github.com/GabrielMarquezMatte/FixedWidthParser" + +[tool.hatch.build.targets.wheel] +packages = ["src/fixedwidth"] +exclude = ["src/fixedwidth/_lib"] + +[tool.hatch.build.targets.wheel.force-include] +"src/fixedwidth/_lib" = "fixedwidth/_lib" + +[tool.hatch.build.targets.wheel.hooks.custom] +path = "hatch_build.py" + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["tests"] diff --git a/python/scripts/build_native.py b/python/scripts/build_native.py new file mode 100644 index 0000000..c50500f --- /dev/null +++ b/python/scripts/build_native.py @@ -0,0 +1,75 @@ +"""Publish FixedWidthParser.Native for this machine and copy the binary into the Python package. + +Run from anywhere: python python/scripts/build_native.py +""" + +import argparse +import os +import platform +import shutil +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +CSPROJ = REPO_ROOT / "src" / "FixedWidthParser.Native" / "FixedWidthParser.Native.csproj" +PACKAGE_LIB_DIR = REPO_ROOT / "python" / "src" / "fixedwidth" / "_lib" + +LIB_NAMES = { + "Windows": "FixedWidthParser.Native.dll", + "Linux": "FixedWidthParser.Native.so", + "Darwin": "FixedWidthParser.Native.dylib", +} + + +def default_rid() -> str: + system = platform.system() + machine = platform.machine().lower() + arch = "arm64" if machine in {"arm64", "aarch64"} else "x64" + if system == "Windows": + return f"win-{arch}" + if system == "Darwin": + return f"osx-{arch}" + if platform.libc_ver()[0] != "glibc": + return f"linux-musl-{arch}" + return f"linux-{arch}" + + +def publish_environment() -> dict[str, str]: + """The NativeAOT linker step finds MSVC through vswhere, which the installer keeps off PATH.""" + environment = dict(os.environ) + if platform.system() == "Windows" and shutil.which("vswhere") is None: + program_files = environment.get("ProgramFiles(x86)", r"C:\Program Files (x86)") + installer = Path(program_files) / "Microsoft Visual Studio" / "Installer" + if (installer / "vswhere.exe").exists(): + environment["PATH"] = environment.get("PATH", "") + os.pathsep + str(installer) + return environment + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rid", default=default_rid(), help="dotnet runtime identifier") + parser.add_argument("--framework", default="net10.0") + args = parser.parse_args() + + subprocess.run( + ["dotnet", "publish", str(CSPROJ), "-c", "Release", "-f", args.framework, "-r", args.rid], + check=True, + env=publish_environment(), + ) + + publish_dir = CSPROJ.parent / "bin" / "Release" / args.framework / args.rid / "publish" + name = LIB_NAMES[platform.system()] + source = publish_dir / name + if not source.exists(): + print(f"error: expected native library at {source}", file=sys.stderr) + return 1 + + PACKAGE_LIB_DIR.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, PACKAGE_LIB_DIR / name) + print(f"copied {source} -> {PACKAGE_LIB_DIR / name}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/src/fixedwidth/__init__.py b/python/src/fixedwidth/__init__.py new file mode 100644 index 0000000..edbed2c --- /dev/null +++ b/python/src/fixedwidth/__init__.py @@ -0,0 +1,69 @@ +"""Read and write fixed-width files through the FixedWidthParser native library. + + import fixedwidth as fw + + layout = fw.Layout([ + fw.Column("codneg", 12, 12, fw.String), + fw.Column("preabe", 56, 13, fw.Float64, scale=2), + ]) + df = fw.read_pandas("COTAHIST_A2023.TXT", layout, fw.ReadOptions(header_lines=1, skip_invalid=True)) +""" + +from fixedwidth.arrow_stream import ArrowStream +from fixedwidth.layout import Layout +from fixedwidth.reader import read, read_arrow, read_arrow_stream, read_batches, read_pandas, read_polars +from fixedwidth.table import Table +from fixedwidth.types import ( + Alignment, + Column, + ColumnType, + FixedWidthError, + LayoutOptions, + Overflow, + ParseError, + ReaderOptions, + ReadOptions, + TrimMode, + WriteOptions, +) +from fixedwidth.writer import write, write_bytes + +String = ColumnType.STRING +Int64 = ColumnType.INT64 +Float64 = ColumnType.FLOAT64 +Decimal = ColumnType.DECIMAL +Date = ColumnType.DATE +Time = ColumnType.TIME +Timestamp = ColumnType.TIMESTAMP + +__all__ = [ + "Alignment", + "ArrowStream", + "Column", + "ColumnType", + "Date", + "Decimal", + "FixedWidthError", + "Float64", + "Int64", + "Layout", + "LayoutOptions", + "Overflow", + "ParseError", + "ReadOptions", + "ReaderOptions", + "String", + "Table", + "Time", + "Timestamp", + "TrimMode", + "WriteOptions", + "read", + "read_arrow", + "read_arrow_stream", + "read_batches", + "read_pandas", + "read_polars", + "write", + "write_bytes", +] diff --git a/python/src/fixedwidth/_lib/.gitkeep b/python/src/fixedwidth/_lib/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/python/src/fixedwidth/_native.py b/python/src/fixedwidth/_native.py new file mode 100644 index 0000000..b3369f1 --- /dev/null +++ b/python/src/fixedwidth/_native.py @@ -0,0 +1,248 @@ +"""ctypes binding to the FixedWidthParser NativeAOT shared library. + +Everything here mirrors src/FixedWidthParser.Native/include/fixedwidth.h. If you change one, change both. +""" + +from __future__ import annotations + +import ctypes +import os +import platform +from functools import lru_cache +from pathlib import Path + +FW_OK = 0 +FW_EOF = -1 +FW_BUFFER_TOO_SMALL = -2 +FW_INVALID_HANDLE = -3 +FW_INVALID_ARGUMENT = -4 +FW_ERROR = -5 +FW_PARSE_ERROR = -6 + +FW_ABI_VERSION = 1 + +FW_OPT_DEFAULT = 0 +FW_OPT_FALSE = 1 +FW_OPT_TRUE = 2 + +_u8p = ctypes.POINTER(ctypes.c_uint8) + + +class ColumnSpec(ctypes.Structure): + _fields_ = [ + ("struct_size", ctypes.c_int32), + ("name", _u8p), + ("name_len", ctypes.c_int32), + ("start", ctypes.c_int32), + ("length", ctypes.c_int32), + ("type", ctypes.c_int32), + ("nullable", ctypes.c_int32), + ("scale", ctypes.c_int32), + ("format", _u8p), + ("format_len", ctypes.c_int32), + ("trim_char", ctypes.c_int32), + ("trim_mode", ctypes.c_int32), + ("alignment", ctypes.c_int32), + ("padding", ctypes.c_int32), + ("overflow", ctypes.c_int32), + ] + + +class LayoutOptions(ctypes.Structure): + _fields_ = [("struct_size", ctypes.c_int32), ("line_length", ctypes.c_int32), ("filler", ctypes.c_int32)] + + +class ReadOptions(ctypes.Structure): + _fields_ = [ + ("struct_size", ctypes.c_int32), + ("header_lines", ctypes.c_int64), + ("skip_invalid", ctypes.c_int32), + ("degree_of_parallelism", ctypes.c_int32), + ("encoding", ctypes.c_int32), + ("max_line_length", ctypes.c_int32), + ("skip_byte_order_mark", ctypes.c_int32), + ] + + +class ReaderOptions(ctypes.Structure): + _fields_ = [("struct_size", ctypes.c_int32), ("read", ReadOptions), ("batch_rows", ctypes.c_int64)] + + +class Column(ctypes.Structure): + _fields_ = [ + ("type", ctypes.c_int32), + ("scale", ctypes.c_int32), + ("length", ctypes.c_int64), + ("values", ctypes.c_void_p), + ("validity", ctypes.c_void_p), + ("data", ctypes.c_void_p), + ("data_len", ctypes.c_int64), + ] + + +class Table(ctypes.Structure): + _fields_ = [("column_count", ctypes.c_int32), ("row_count", ctypes.c_int64), ("columns", ctypes.POINTER(Column))] + + +class ReadResult(ctypes.Structure): + _fields_ = [("struct_size", ctypes.c_int32), ("lines_read", ctypes.c_int64), ("lines_skipped", ctypes.c_int64)] + + +class ParseErrorInfo(ctypes.Structure): + _fields_ = [("struct_size", ctypes.c_int32), ("line_number", ctypes.c_int64), ("column", ctypes.c_int32)] + + +class WriteOptions(ctypes.Structure): + _fields_ = [ + ("struct_size", ctypes.c_int32), + ("encoding", ctypes.c_int32), + ("newline", _u8p), + ("newline_len", ctypes.c_int32), + ("degree_of_parallelism", ctypes.c_int32), + ("batch_rows", ctypes.c_int64), + ] + + +class Buffer(ctypes.Structure): + _fields_ = [("data", ctypes.c_void_p), ("length", ctypes.c_int64)] + + +class ArrowSchema(ctypes.Structure): + pass + + +class ArrowArray(ctypes.Structure): + pass + + +_ArrowSchemaRelease = ctypes.CFUNCTYPE(None, ctypes.POINTER(ArrowSchema)) +_ArrowArrayRelease = ctypes.CFUNCTYPE(None, ctypes.POINTER(ArrowArray)) + +ArrowSchema._fields_ = [ + ("format", ctypes.c_char_p), + ("name", ctypes.c_char_p), + ("metadata", ctypes.c_char_p), + ("flags", ctypes.c_int64), + ("n_children", ctypes.c_int64), + ("children", ctypes.POINTER(ctypes.POINTER(ArrowSchema))), + ("dictionary", ctypes.POINTER(ArrowSchema)), + ("release", _ArrowSchemaRelease), + ("private_data", ctypes.c_void_p), +] +ArrowArray._fields_ = [ + ("length", ctypes.c_int64), + ("null_count", ctypes.c_int64), + ("offset", ctypes.c_int64), + ("n_buffers", ctypes.c_int64), + ("n_children", ctypes.c_int64), + ("buffers", ctypes.POINTER(ctypes.c_void_p)), + ("children", ctypes.POINTER(ctypes.POINTER(ArrowArray))), + ("dictionary", ctypes.POINTER(ArrowArray)), + ("release", _ArrowArrayRelease), + ("private_data", ctypes.c_void_p), +] + + +class ArrowArrayStream(ctypes.Structure): + pass + + +_ArrowStreamGetSchema = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.POINTER(ArrowArrayStream), ctypes.POINTER(ArrowSchema)) +_ArrowStreamGetNext = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.POINTER(ArrowArrayStream), ctypes.POINTER(ArrowArray)) +_ArrowStreamGetLastError = ctypes.CFUNCTYPE(ctypes.c_char_p, ctypes.POINTER(ArrowArrayStream)) +_ArrowStreamRelease = ctypes.CFUNCTYPE(None, ctypes.POINTER(ArrowArrayStream)) + +ArrowArrayStream._fields_ = [ + ("get_schema", _ArrowStreamGetSchema), + ("get_next", _ArrowStreamGetNext), + ("get_last_error", _ArrowStreamGetLastError), + ("release", _ArrowStreamRelease), + ("private_data", ctypes.c_void_p), +] + +_P = ctypes.POINTER +_SIGNATURES: dict[str, tuple[object, list[object]]] = { + "fw_abi_version": (ctypes.c_int32, []), + "fw_last_error": (ctypes.c_int32, [_u8p, ctypes.c_int32, _P(ctypes.c_int32)]), + "fw_last_parse_error": (ctypes.c_int32, [_P(ParseErrorInfo)]), + "fw_layout_create": (ctypes.c_int32, [_P(ColumnSpec), ctypes.c_int32, _P(LayoutOptions), _P(ctypes.c_void_p)]), + "fw_layout_free": (None, [ctypes.c_void_p]), + "fw_read_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, ctypes.c_void_p, _P(ReadOptions), _P(Table), _P(ReadResult)]), + "fw_read_memory": (ctypes.c_int32, [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p, _P(ReadOptions), _P(Table), _P(ReadResult)]), + "fw_free_table": (None, [_P(Table)]), + "fw_reader_open_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, ctypes.c_void_p, _P(ReaderOptions), _P(ctypes.c_void_p)]), + "fw_reader_open_memory": (ctypes.c_int32, [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p, _P(ReaderOptions), _P(ctypes.c_void_p)]), + "fw_reader_next": (ctypes.c_int32, [ctypes.c_void_p, _P(Table)]), + "fw_reader_close": (None, [ctypes.c_void_p]), + "fw_read_arrow_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, ctypes.c_void_p, _P(ReadOptions), _P(ArrowArray), _P(ArrowSchema), _P(ReadResult)]), + "fw_read_arrow_memory": (ctypes.c_int32, [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p, _P(ReadOptions), _P(ArrowArray), _P(ArrowSchema), _P(ReadResult)]), + "fw_read_arrow_stream_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, ctypes.c_void_p, _P(ReaderOptions), _P(ArrowArrayStream)]), + "fw_read_arrow_stream_memory": (ctypes.c_int32, [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p, _P(ReaderOptions), _P(ArrowArrayStream)]), + "fw_write_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, _P(Table), ctypes.c_void_p, _P(WriteOptions)]), + "fw_write_memory": (ctypes.c_int32, [_P(Table), ctypes.c_void_p, _P(WriteOptions), _P(Buffer)]), + "fw_free_buffer": (None, [_P(Buffer)]), +} + + +def library_filename() -> str: + system = platform.system() + if system == "Windows": + return "FixedWidthParser.Native.dll" + if system == "Darwin": + return "FixedWidthParser.Native.dylib" + return "FixedWidthParser.Native.so" + + +def _candidate_paths() -> list[Path]: + paths = [] + override = os.environ.get("FIXEDWIDTH_NATIVE_LIB") + if override: + paths.append(Path(override)) + paths.append(Path(__file__).parent / "_lib" / library_filename()) + return paths + + +@lru_cache(maxsize=1) +def load_library() -> ctypes.CDLL: + """Loads the bundled library (or FIXEDWIDTH_NATIVE_LIB) once and checks its ABI version.""" + tried = [] + for path in _candidate_paths(): + if not path.exists(): + tried.append(str(path)) + continue + lib = ctypes.CDLL(str(path)) + for name, (restype, argtypes) in _SIGNATURES.items(): + function = getattr(lib, name) + function.restype = restype + function.argtypes = argtypes + version = lib.fw_abi_version() + if version != FW_ABI_VERSION: + raise OSError(f"{path} has ABI version {version}; this package needs {FW_ABI_VERSION}") + return lib + raise OSError( + "FixedWidthParser.Native was not found (looked in " + ", ".join(tried) + "). " + "Build it with `python python/scripts/build_native.py` or set FIXEDWIDTH_NATIVE_LIB." + ) + + +def last_error() -> str: + lib = load_library() + buffer = (ctypes.c_uint8 * 4096)() + length = ctypes.c_int32() + status = lib.fw_last_error(buffer, len(buffer), ctypes.byref(length)) + if status == FW_BUFFER_TOO_SMALL: + buffer = (ctypes.c_uint8 * length.value)() + lib.fw_last_error(buffer, len(buffer), ctypes.byref(length)) + return bytes(buffer[: length.value]).decode("utf-8", "replace") + + +def last_parse_error() -> tuple[int, int]: + info = ParseErrorInfo(struct_size=ctypes.sizeof(ParseErrorInfo)) + load_library().fw_last_parse_error(ctypes.byref(info)) + return int(info.line_number), int(info.column) + + +def utf8(text: str) -> tuple[ctypes.Array, int]: + """A UTF-8 buffer and its length, for a pointer + length argument.""" + encoded = text.encode("utf-8") + return (ctypes.c_uint8 * max(1, len(encoded))).from_buffer_copy(encoded.ljust(1, b"\0")), len(encoded) diff --git a/python/src/fixedwidth/arrow_stream.py b/python/src/fixedwidth/arrow_stream.py new file mode 100644 index 0000000..61eb440 --- /dev/null +++ b/python/src/fixedwidth/arrow_stream.py @@ -0,0 +1,127 @@ +"""An Arrow C stream exposed through the Arrow PyCapsule interface, with no pyarrow dependency.""" + +from __future__ import annotations + +import ctypes + +from fixedwidth import _native +from fixedwidth.types import FixedWidthError + +_CAPSULE_NAME = b"arrow_array_stream" + +_capsule_new = ctypes.pythonapi.PyCapsule_New +_capsule_new.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_void_p] +_capsule_new.restype = ctypes.py_object + +_capsule_pointer = ctypes.pythonapi.PyCapsule_GetPointer +_capsule_pointer.argtypes = [ctypes.c_void_p, ctypes.c_char_p] +_capsule_pointer.restype = ctypes.c_void_p + +_ReadFn = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p) +_ErrorFn = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p) +_ReleaseFn = ctypes.CFUNCTYPE(None, ctypes.c_void_p) + + +class _Target: + """The native stream behind a proxy, re-typed so ctypes releases the GIL in its callbacks, and the + caller's bytes a memory read reads lazily.""" + + def __init__(self, stream: _native.ArrowArrayStream, keep: object) -> None: + self.stream = stream + self.keep = keep + self.address = ctypes.addressof(stream) + self.get_schema = _ReadFn(_address_of(stream.get_schema)) + self.get_next = _ReadFn(_address_of(stream.get_next)) + self.get_last_error = _ErrorFn(_address_of(stream.get_last_error)) + + +def _address_of(function: object) -> int: + return ctypes.cast(function, ctypes.c_void_p).value + + +_streams: dict[int, _Target] = {} +_proxies: dict[int, _native.ArrowArrayStream] = {} + + +def _target(stream: int) -> _Target: + return _streams[_native.ArrowArrayStream.from_address(stream).private_data] + + +@_ReadFn +def _get_schema(stream: int, out: int) -> int: + target = _target(stream) + return target.get_schema(target.address, out) + + +@_ReadFn +def _get_next(stream: int, out: int) -> int: + target = _target(stream) + return target.get_next(target.address, out) + + +@_ErrorFn +def _get_last_error(stream: int) -> int | None: + target = _target(stream) + return target.get_last_error(target.address) + + +def _close(proxy: _native.ArrowArrayStream) -> None: + _release(_streams.pop(proxy.private_data).stream) + proxy.release = _native._ArrowStreamRelease() + + +@_ReleaseFn +def _release_proxy(stream: int) -> None: + _close(_native.ArrowArrayStream.from_address(stream)) + + +def _release(stream: _native.ArrowArrayStream) -> None: + if stream.release: + stream.release(ctypes.byref(stream)) + + +def _proxy(stream: _native.ArrowArrayStream, keep: object) -> _native.ArrowArrayStream: + target = _Target(stream, keep) + _streams[id(target)] = target + proxy = _native.ArrowArrayStream() + proxy.get_schema = ctypes.cast(_get_schema, _native._ArrowStreamGetSchema) + proxy.get_next = ctypes.cast(_get_next, _native._ArrowStreamGetNext) + proxy.get_last_error = ctypes.cast(_get_last_error, _native._ArrowStreamGetLastError) + proxy.release = ctypes.cast(_release_proxy, _native._ArrowStreamRelease) + proxy.private_data = id(target) + return proxy + + +def _destroy_capsule(capsule: int) -> None: + proxy = _proxies.pop(_capsule_pointer(capsule, _CAPSULE_NAME)) + if proxy.release: + _close(proxy) + + +_destroy_capsule_callback = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(_destroy_capsule) + + +class ArrowStream: + """A single-use stream of Arrow record batches implementing `__arrow_c_stream__`. + + Hand it to any consumer of the Arrow PyCapsule interface — `polars.DataFrame(stream)`, + `pyarrow.RecordBatchReader.from_stream(stream)`. Once a consumer takes it, the stream is spent; + dropping it unconsumed releases the native read. + """ + + def __init__(self, stream: _native.ArrowArrayStream, keep: object = None) -> None: + self._stream: _native.ArrowArrayStream | None = stream + self._keep = keep + + def __arrow_c_stream__(self, requested_schema: object = None) -> object: + if self._stream is None: + raise FixedWidthError("this Arrow stream was already consumed") + stream, self._stream = self._stream, None + proxy = _proxy(stream, self._keep) + address = ctypes.addressof(proxy) + _proxies[address] = proxy + return _capsule_new(address, _CAPSULE_NAME, _destroy_capsule_callback) + + def __del__(self) -> None: + if getattr(self, "_stream", None) is not None: + _release(self._stream) diff --git a/python/src/fixedwidth/layout.py b/python/src/fixedwidth/layout.py new file mode 100644 index 0000000..024ad72 --- /dev/null +++ b/python/src/fixedwidth/layout.py @@ -0,0 +1,94 @@ +"""Layout: the columns of a line, validated once by the native library and shared by every call.""" + +from __future__ import annotations + +import ctypes +from collections.abc import Sequence + +from fixedwidth import _native +from fixedwidth.types import Column, ColumnType, FixedWidthError, LayoutOptions, ParseError + + +def check(status: int) -> None: + """Raises the error a native status reports.""" + if status == _native.FW_OK: + return + message = _native.last_error() or f"native status {status}" + if status == _native.FW_PARSE_ERROR: + line_number, column = _native.last_parse_error() + raise ParseError(message, line_number, column) + raise FixedWidthError(message) + + +def ascii_option(value: str | None, what: str) -> int: + if value is None: + return 0 + if len(value) != 1 or ord(value) > 127: + raise ValueError(f"{what} must be one ASCII character, got {value!r}") + return ord(value) + + +class Layout: + """How the columns of a line map to names and types; pass it to every read and write. + + The native layout is immutable and may be shared across threads. + """ + + def __init__(self, columns: Sequence[Column], options: LayoutOptions | None = None) -> None: + self._lib = _native.load_library() + self._handle: int | None = None + self._columns = list(columns) + specs = (_native.ColumnSpec * max(1, len(self._columns)))() + keep = [self._fill(specs[i], column) for i, column in enumerate(self._columns)] + raw = None + if options is not None: + raw = _native.LayoutOptions( + struct_size=ctypes.sizeof(_native.LayoutOptions), + line_length=options.line_length, + filler=ascii_option(options.filler, "filler"), + ) + handle = ctypes.c_void_p() + check(self._lib.fw_layout_create(specs, len(self._columns), ctypes.byref(raw) if raw else None, ctypes.byref(handle))) + self._handle = handle.value + del keep + + @staticmethod + def _fill(spec: _native.ColumnSpec, column: Column) -> tuple[object, ...]: + name, name_len = _native.utf8(column.name) + fmt, fmt_len = _native.utf8(column.format) if column.format is not None else (None, 0) + spec.struct_size = ctypes.sizeof(_native.ColumnSpec) + spec.name = ctypes.cast(name, ctypes.POINTER(ctypes.c_uint8)) + spec.name_len = name_len + spec.start = column.start + spec.length = column.length + spec.type = int(ColumnType(column.type)) + spec.nullable = int(column.nullable) + spec.scale = column.scale + spec.format = ctypes.cast(fmt, ctypes.POINTER(ctypes.c_uint8)) if fmt is not None else None + spec.format_len = fmt_len + spec.trim_char = ascii_option(column.trim_char, "trim_char") + spec.trim_mode = int(column.trim_mode or 0) + spec.alignment = int(column.alignment or 0) + spec.padding = ascii_option(column.padding, "padding") + spec.overflow = int(column.overflow or 0) + return name, fmt + + @property + def columns(self) -> list[Column]: + return list(self._columns) + + @property + def names(self) -> list[str]: + return [column.name for column in self._columns] + + @property + def handle(self) -> int: + return self._handle or 0 + + def __del__(self) -> None: + if getattr(self, "_handle", None): + self._lib.fw_layout_free(self._handle) + self._handle = None + + def __repr__(self) -> str: + return f"Layout({len(self._columns)} columns: {', '.join(self.names)})" diff --git a/python/src/fixedwidth/py.typed b/python/src/fixedwidth/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/python/src/fixedwidth/reader.py b/python/src/fixedwidth/reader.py new file mode 100644 index 0000000..9c7721d --- /dev/null +++ b/python/src/fixedwidth/reader.py @@ -0,0 +1,157 @@ +"""Reading: whole sources into a Table, Arrow, pandas or polars, and batched reads.""" + +from __future__ import annotations + +import ctypes +import os +from collections.abc import Iterator +from typing import Any, Union + +from fixedwidth import _native +from fixedwidth.arrow_stream import ArrowStream +from fixedwidth.layout import Layout, check +from fixedwidth.table import Table, _Owner +from fixedwidth.types import ReaderOptions, ReadOptions, code_page + +Source = Union[str, "os.PathLike[str]", bytes, bytearray, memoryview] + + +class _Input: + """A file path or a buffer, as the pointer + length arguments the native calls take.""" + + def __init__(self, source: Source) -> None: + self.keep: object = None + if isinstance(source, (str, os.PathLike)): + self.path, self.path_len = _native.utf8(os.fspath(source)) + return + self.path = None + view = memoryview(source) + if view.readonly or not view.contiguous: + data = bytes(view) + self.keep = data + self.address = ctypes.cast(ctypes.c_char_p(data), ctypes.c_void_p).value + else: + buffer = (ctypes.c_char * view.nbytes).from_buffer(view) + self.keep = (view, buffer) + self.address = ctypes.addressof(buffer) + self.length = view.nbytes + + @property + def is_file(self) -> bool: + return self.path is not None + + +def _read_options(options: ReadOptions | None) -> _native.ReadOptions | None: + if options is None: + return None + bom = options.skip_byte_order_mark + return _native.ReadOptions( + struct_size=ctypes.sizeof(_native.ReadOptions), + header_lines=options.header_lines, + skip_invalid=int(options.skip_invalid), + degree_of_parallelism=options.degree_of_parallelism, + encoding=code_page(options.encoding), + max_line_length=options.max_line_length, + skip_byte_order_mark=_native.FW_OPT_DEFAULT if bom is None else (_native.FW_OPT_TRUE if bom else _native.FW_OPT_FALSE), + ) + + +def _reader_options(options: ReaderOptions | None) -> _native.ReaderOptions | None: + if options is None: + return None + raw = _native.ReaderOptions(struct_size=ctypes.sizeof(_native.ReaderOptions), batch_rows=options.batch_rows) + read = _read_options(options.read) + if read is not None: + raw.read = read + return raw + + +def _ref(value: object) -> object: + return ctypes.byref(value) if value is not None else None + + +def read(source: Source, layout: Layout, options: ReadOptions | None = None) -> Table: + """Reads a whole file or buffer into a Table (see `ReadOptions` for header lines, skipping and threads).""" + lib = _native.load_library() + data = _Input(source) + raw = _read_options(options) + table = _native.Table() + result = _native.ReadResult(struct_size=ctypes.sizeof(_native.ReadResult)) + if data.is_file: + status = lib.fw_read_file(data.path, data.path_len, layout.handle, _ref(raw), ctypes.byref(table), ctypes.byref(result)) + else: + status = lib.fw_read_memory(data.address, data.length, layout.handle, _ref(raw), ctypes.byref(table), ctypes.byref(result)) + check(status) + return Table(_Owner(table), layout.names, int(result.lines_read), int(result.lines_skipped)) + + +def read_arrow(source: Source, layout: Layout, options: ReadOptions | None = None) -> Any: + """Reads a whole file or buffer into a `pyarrow.Table`, handed over without a copy. Requires pyarrow.""" + import pyarrow + + lib = _native.load_library() + data = _Input(source) + raw = _read_options(options) + array = _native.ArrowArray() + schema = _native.ArrowSchema() + if data.is_file: + status = lib.fw_read_arrow_file(data.path, data.path_len, layout.handle, _ref(raw), ctypes.byref(array), ctypes.byref(schema), None) + else: + status = lib.fw_read_arrow_memory(data.address, data.length, layout.handle, _ref(raw), ctypes.byref(array), ctypes.byref(schema), None) + check(status) + batch = pyarrow.RecordBatch._import_from_c(ctypes.addressof(array), ctypes.addressof(schema)) + return pyarrow.Table.from_batches([batch]) + + +def read_pandas(source: Source, layout: Layout, options: ReadOptions | None = None) -> Any: + """`read_arrow()` as a `pandas.DataFrame`; each Arrow buffer is freed as pandas takes it. Requires pyarrow and pandas.""" + return read_arrow(source, layout, options).to_pandas(self_destruct=True, split_blocks=True) + + +def read_polars(source: Source, layout: Layout, options: ReadOptions | None = None) -> Any: + """`read_arrow()` as a `polars.DataFrame`, zero-copy. Without pyarrow, reads sequentially through an Arrow stream.""" + import polars + + try: + return polars.from_arrow(read_arrow(source, layout, options)) + except ImportError: + return polars.DataFrame(read_arrow_stream(source, layout, ReaderOptions(read=options or ReadOptions()))) + + +def read_batches(source: Source, layout: Layout, options: ReaderOptions | None = None) -> Iterator[Table]: + """Reads sequentially, yielding a Table of at most `options.batch_rows` rows at a time.""" + lib = _native.load_library() + data = _Input(source) + raw = _reader_options(options) + reader = ctypes.c_void_p() + if data.is_file: + status = lib.fw_reader_open_file(data.path, data.path_len, layout.handle, _ref(raw), ctypes.byref(reader)) + else: + status = lib.fw_reader_open_memory(data.address, data.length, layout.handle, _ref(raw), ctypes.byref(reader)) + check(status) + try: + while True: + table = _native.Table() + status = lib.fw_reader_next(reader, ctypes.byref(table)) + if status == _native.FW_EOF: + return + check(status) + yield Table(_Owner(table), layout.names) + finally: + lib.fw_reader_close(reader) + del data + + +def read_arrow_stream(source: Source, layout: Layout, options: ReaderOptions | None = None) -> ArrowStream: + """The batches of `read_batches()` as a single-use Arrow PyCapsule stream, for `polars.DataFrame(stream)` + or `pyarrow.RecordBatchReader.from_stream(stream)`. Needs no third-party package to create.""" + lib = _native.load_library() + data = _Input(source) + raw = _reader_options(options) + stream = _native.ArrowArrayStream() + if data.is_file: + status = lib.fw_read_arrow_stream_file(data.path, data.path_len, layout.handle, _ref(raw), ctypes.byref(stream)) + else: + status = lib.fw_read_arrow_stream_memory(data.address, data.length, layout.handle, _ref(raw), ctypes.byref(stream)) + check(status) + return ArrowStream(stream, keep=data) diff --git a/python/src/fixedwidth/table.py b/python/src/fixedwidth/table.py new file mode 100644 index 0000000..899a2b3 --- /dev/null +++ b/python/src/fixedwidth/table.py @@ -0,0 +1,135 @@ +"""Table: a columnar read whose numeric columns are NumPy views over the native buffers.""" + +from __future__ import annotations + +import ctypes +from collections.abc import Callable +from decimal import Decimal +from typing import Any + +from fixedwidth import _native +from fixedwidth.types import ColumnType + +_ELEMENTS = { + ColumnType.INT64: (ctypes.c_int64, "int64"), + ColumnType.FLOAT64: (ctypes.c_double, "float64"), + ColumnType.DECIMAL: (ctypes.c_int64, "int64"), + ColumnType.DATE: (ctypes.c_int32, "int32"), + ColumnType.TIME: (ctypes.c_int64, "int64"), + ColumnType.TIMESTAMP: (ctypes.c_int64, "int64"), +} + + +class _Owner: + """Owns one native fw_table; frees it once nothing (the Table or any view) references it.""" + + def __init__(self, table: _native.Table) -> None: + self.table = table + self._on_free: Callable[[bool], object] | None = None + + def view(self, address: int, element: type, count: int) -> ctypes.Array: + buffer = (element * count).from_address(address) + buffer._owner = self # type: ignore[attr-defined] # a view keeps the native memory alive + return buffer + + def __del__(self) -> None: + _native.load_library().fw_free_table(ctypes.byref(self.table)) + if self._on_free is not None: + self._on_free(True) + + +class Table: + """One read: `table[name]` is a NumPy view for numeric and date/time columns (as stored: int64, + float64, DECIMAL's unscaled int64, DATE's int32 days, TIME/TIMESTAMP's int64 microseconds) and a + list of `str` for strings. Null rows are marked by `validity(name)`. Views stay valid after the + Table is gone; the native memory is freed when the last one is.""" + + def __init__(self, owner: _Owner, names: list[str], lines_read: int = 0, lines_skipped: int = 0) -> None: + self._owner = owner + self._names = names + self.lines_read = lines_read + self.lines_skipped = lines_skipped + + @property + def num_rows(self) -> int: + return int(self._owner.table.row_count) + + @property + def column_names(self) -> list[str]: + return list(self._names) + + def __len__(self) -> int: + return self.num_rows + + def _column(self, name: str) -> _native.Column: + try: + return self._owner.table.columns[self._names.index(name)] + except ValueError: + raise KeyError(name) from None + + def __getitem__(self, name: str) -> Any: + column = self._column(name) + kind = ColumnType(column.type) + if kind is ColumnType.STRING: + return _strings(column) + import numpy + + element, dtype = _ELEMENTS[kind] + if column.length == 0: + return numpy.empty(0, dtype=dtype) + return numpy.frombuffer(self._owner.view(column.values, element, column.length), dtype=dtype) + + def validity(self, name: str) -> Any: + """A NumPy bool array, True where the row holds a value, or None when the column has no nulls.""" + column = self._column(name) + if not column.validity: + return None + import numpy + + bits = numpy.frombuffer(ctypes.string_at(column.validity, (column.length + 7) // 8), dtype=numpy.uint8) + return numpy.unpackbits(bits, bitorder="little")[: column.length].astype(bool) + + def to_arrow(self) -> Any: + """The same columns as a `pyarrow.Table`, sharing the native buffers. Requires pyarrow.""" + import pyarrow + + arrays = [self._arrow_column(name, pyarrow) for name in self._names] + return pyarrow.Table.from_arrays(arrays, names=self._names) + + def _arrow_column(self, name: str, pa: Any) -> Any: + column = self._column(name) + kind = ColumnType(column.type) + n = int(column.length) + validity = pa.foreign_buffer(column.validity, (n + 7) // 8, base=self._owner) if column.validity else None + if kind is ColumnType.DECIMAL: + # ponytail: decimals go through Python objects; fw.read_arrow() hands them over natively. + mask = self.validity(name) + values = self[name] + decimals = [None if mask is not None and not mask[i] else Decimal(int(v)).scaleb(-column.scale) for i, v in enumerate(values)] + return pa.array(decimals, type=pa.decimal128(18, column.scale)) + if kind is ColumnType.STRING: + offsets = pa.foreign_buffer(column.values, (n + 1) * 4, base=self._owner) + data = pa.foreign_buffer(column.data, column.data_len, base=self._owner) if column.data else pa.py_buffer(b"") + return pa.Array.from_buffers(pa.string(), n, [validity, offsets, data]) + arrow_type = { + ColumnType.INT64: pa.int64(), + ColumnType.FLOAT64: pa.float64(), + ColumnType.DATE: pa.date32(), + ColumnType.TIME: pa.time64("us"), + ColumnType.TIMESTAMP: pa.timestamp("us"), + }[kind] + width = 4 if kind is ColumnType.DATE else 8 + values = pa.foreign_buffer(column.values, n * width, base=self._owner) if n else pa.py_buffer(b"") + return pa.Array.from_buffers(arrow_type, n, [validity, values]) + + def __repr__(self) -> str: + return f"Table({self.num_rows} rows: {', '.join(self._names)})" + + +def _strings(column: _native.Column) -> list[str]: + n = int(column.length) + if n == 0: + return [] + offsets = (ctypes.c_int32 * (n + 1)).from_address(column.values) + data = ctypes.string_at(column.data, column.data_len) if column.data else b"" + return [data[offsets[i] : offsets[i + 1]].decode("utf-8") for i in range(n)] diff --git a/python/src/fixedwidth/types.py b/python/src/fixedwidth/types.py new file mode 100644 index 0000000..98fa0ba --- /dev/null +++ b/python/src/fixedwidth/types.py @@ -0,0 +1,136 @@ +"""The public value types: column types, column descriptions, options and errors.""" + +from __future__ import annotations + +import re +from dataclasses import KW_ONLY, dataclass, field +from enum import IntEnum + + +class ColumnType(IntEnum): + """FW_T_*: how a column is parsed and what its values are.""" + + STRING = 0 + INT64 = 1 + FLOAT64 = 2 + DECIMAL = 3 + DATE = 4 + TIME = 5 + TIMESTAMP = 6 + + +class TrimMode(IntEnum): + BOTH = 0 + TRAILING = 1 + LEADING = 2 + + +class Alignment(IntEnum): + DEFAULT = 0 + LEFT = 1 + RIGHT = 2 + + +class Overflow(IntEnum): + THROW = 0 + TRUNCATE = 1 + + +@dataclass(frozen=True) +class Column: + """One column of a layout: its name, zero-based byte `start`, byte `length` and `type`. + + - `nullable`: a blank column (only `trim_char` and spaces) reads as null instead of failing. + - `scale`: implied decimals. FLOAT64 divides digits-only text by 10**scale (B3's `N(11)V99`); + DECIMAL keeps the unscaled integer. + - `format`: the exact date/time format (.NET syntax, e.g. `yyyyMMdd`), or a number format on write. + - `trim_char`, `trim_mode`: what is stripped before parsing; a null is written as `trim_char`. + - `alignment`, `padding`, `overflow`: how a value is written. `padding='0'` keeps a sign leftmost. + """ + + name: str + start: int + length: int + type: ColumnType + _: KW_ONLY + nullable: bool = False + scale: int = 0 + format: str | None = None + trim_char: str | None = None + trim_mode: TrimMode | None = None + alignment: Alignment | None = None + padding: str | None = None + overflow: Overflow | None = None + + +@dataclass(frozen=True) +class LayoutOptions: + """`line_length`: 0 ends lines at the last column; longer pads them with `filler` on write.""" + + line_length: int = 0 + filler: str | None = None + + +@dataclass(frozen=True) +class ReadOptions: + """Mirrors fw_read_options. `encoding` is a code page or a codec name (`"latin-1"`, `"cp1252"`).""" + + header_lines: int = 0 + skip_invalid: bool = False + degree_of_parallelism: int = 0 + encoding: int | str | None = None + max_line_length: int = 0 + skip_byte_order_mark: bool | None = None + + +@dataclass(frozen=True) +class ReaderOptions: + """Mirrors fw_reader_options: a sequential read in tables of `batch_rows` rows (0 = 65,536).""" + + read: ReadOptions = field(default_factory=ReadOptions) + batch_rows: int = 0 + + +@dataclass(frozen=True) +class WriteOptions: + """Mirrors fw_write_options.""" + + encoding: int | str | None = None + newline: str = "\n" + degree_of_parallelism: int = 0 + batch_rows: int = 0 + + +class FixedWidthError(Exception): + """Raised when the native library reports a failure.""" + + +class ParseError(FixedWidthError): + """A line failed to parse. `line_number` is 1-based; `column` is the layout index, -1 for a short line.""" + + def __init__(self, message: str, line_number: int, column: int) -> None: + super().__init__(message) + self.line_number = line_number + self.column = column + + +_CODE_PAGES = { + "utf-8": 0, "utf8": 0, "ascii": 20127, "us-ascii": 20127, + "latin-1": 28591, "latin1": 28591, "iso-8859-1": 28591, "iso8859-1": 28591, + "windows-1252": 1252, "cp1252": 1252, "iso-8859-15": 28605, "latin-9": 28605, +} + + +def code_page(encoding: int | str | None) -> int: + """The code page the native options take for `encoding`.""" + if encoding is None: + return 0 + if isinstance(encoding, int): + return encoding + name = encoding.strip().lower().replace("_", "-") + if name in _CODE_PAGES: + return _CODE_PAGES[name] + match = re.fullmatch(r"(?:cp|windows-)(\d+)", name) + if match: + return int(match.group(1)) + raise ValueError(f"unknown encoding {encoding!r}; pass a code page number") diff --git a/python/src/fixedwidth/writer.py b/python/src/fixedwidth/writer.py new file mode 100644 index 0000000..35b4534 --- /dev/null +++ b/python/src/fixedwidth/writer.py @@ -0,0 +1,137 @@ +"""Writing: a pandas, polars or pyarrow table through a layout, to a file or to bytes. Requires pyarrow.""" + +from __future__ import annotations + +import ctypes +import os +from typing import Any + +from fixedwidth import _native +from fixedwidth.layout import Layout, check +from fixedwidth.types import Column, ColumnType, FixedWidthError, WriteOptions, code_page + + +def write(data: Any, path: str | os.PathLike[str], layout: Layout, options: WriteOptions | None = None) -> None: + """Writes every row of `data` (pandas, polars, pyarrow, or anything with `__arrow_c_stream__`) to `path`. + + Columns are matched to the layout by name. A null writes as its column filled with `trim_char`.""" + lib = _native.load_library() + table, keep = _native_table(data, layout) + raw, newline = _write_options(options) + name, name_len = _native.utf8(os.fspath(path)) + check(lib.fw_write_file(name, name_len, ctypes.byref(table), layout.handle, ctypes.byref(raw) if raw else None)) + del keep, newline + + +def write_bytes(data: Any, layout: Layout, options: WriteOptions | None = None) -> bytes: + """Same as `write()`, returning the bytes.""" + lib = _native.load_library() + table, keep = _native_table(data, layout) + raw, newline = _write_options(options) + buffer = _native.Buffer() + check(lib.fw_write_memory(ctypes.byref(table), layout.handle, ctypes.byref(raw) if raw else None, ctypes.byref(buffer))) + try: + return ctypes.string_at(buffer.data, buffer.length) if buffer.length else b"" + finally: + lib.fw_free_buffer(ctypes.byref(buffer)) + del keep, newline + + +def _write_options(options: WriteOptions | None) -> tuple[_native.WriteOptions | None, object]: + if options is None: + return None, None + newline, newline_len = _native.utf8(options.newline) + raw = _native.WriteOptions( + struct_size=ctypes.sizeof(_native.WriteOptions), + encoding=code_page(options.encoding), + newline=ctypes.cast(newline, ctypes.POINTER(ctypes.c_uint8)), + newline_len=newline_len, + degree_of_parallelism=options.degree_of_parallelism, + batch_rows=options.batch_rows, + ) + return raw, newline + + +def _arrow_table(data: Any) -> Any: + import pyarrow + + if isinstance(data, pyarrow.RecordBatch): + return pyarrow.Table.from_batches([data]) + if isinstance(data, pyarrow.Table): + return data + if type(data).__module__.startswith("pandas"): + return pyarrow.Table.from_pandas(data, preserve_index=False) + if hasattr(data, "__arrow_c_stream__"): + return pyarrow.table(data) + raise TypeError(f"cannot write {type(data).__name__}: pass a pandas, polars or pyarrow table") + + +def _native_table(data: Any, layout: Layout) -> tuple[_native.Table, list[object]]: + import pyarrow + + arrow = _arrow_table(data).combine_chunks() + columns = (_native.Column * max(1, len(layout.columns)))() + keep: list[object] = [columns] + for index, spec in enumerate(layout.columns): + if spec.name not in arrow.column_names: + raise FixedWidthError(f"column '{spec.name}' of the layout is not in the data") + chunked = arrow.column(spec.name) + array = chunked.chunk(0) if chunked.num_chunks else pyarrow.array([], type=chunked.type) + _fill(columns[index], spec, array, keep) + table = _native.Table(column_count=len(layout.columns), row_count=arrow.num_rows, columns=columns) + return table, keep + + +def _fill(target: _native.Column, spec: Column, array: Any, keep: list[object]) -> None: + import numpy + import pyarrow + + kind = ColumnType(spec.type) + target.type = int(kind) + target.scale = spec.scale if kind is ColumnType.DECIMAL else 0 + target.length = len(array) + if array.null_count: + valid = numpy.packbits(~array.is_null().to_numpy(zero_copy_only=False), bitorder="little") + keep.append(valid) + target.validity = valid.ctypes.data + if kind is ColumnType.STRING: + _fill_strings(target, array.cast(pyarrow.string()).fill_null(""), keep) + return + values = numpy.ascontiguousarray(_values(kind, spec, array, pyarrow, numpy)) + keep.append(values) + target.values = values.ctypes.data if len(values) else None + + +def _values(kind: ColumnType, spec: Column, array: Any, pa: Any, np: Any) -> Any: + if kind is ColumnType.INT64: + return array.cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) + if kind is ColumnType.FLOAT64: + return array.cast(pa.float64()).fill_null(0).to_numpy(zero_copy_only=False) + if kind is ColumnType.DECIMAL: + if pa.types.is_decimal(array.type): + return np.array([0 if v is None else int(v.scaleb(spec.scale)) for v in array.to_pylist()], dtype=np.int64) + floats = array.cast(pa.float64()).fill_null(0).to_numpy(zero_copy_only=False) + return np.round(floats * 10**spec.scale).astype(np.int64) + if kind is ColumnType.DATE: + return array.cast(pa.date32()).cast(pa.int32()).fill_null(0).to_numpy(zero_copy_only=False) + if kind is ColumnType.TIME: + return array.cast(pa.time64("us")).cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) + if pa.types.is_timestamp(array.type) and array.type.tz is not None: + array = array.cast(pa.timestamp(array.type.unit)) + return array.cast(pa.timestamp("us")).cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) + + +def _fill_strings(target: _native.Column, array: Any, keep: list[object]) -> None: + import numpy + + buffers = array.buffers() + n = len(array) + offsets = numpy.frombuffer(buffers[1], dtype=numpy.int32)[array.offset : array.offset + n + 1] if n else numpy.zeros(1, numpy.int32) + base = int(offsets[0]) + rebased = numpy.ascontiguousarray(offsets - base, dtype=numpy.int32) + keep.append(rebased) + keep.append(array) + target.values = rebased.ctypes.data + data_len = int(rebased[-1]) + target.data_len = data_len + target.data = buffers[2].address + base if data_len and buffers[2] is not None else None diff --git a/python/tests/conftest.py b/python/tests/conftest.py new file mode 100644 index 0000000..94cb3ec --- /dev/null +++ b/python/tests/conftest.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import pytest + +import fixedwidth as fw + +# A COTAHIST-like file: a header, three quotes (one with a blank quantity), an invalid line, a trailer. +CONTENT = ( + "00COTAHIST.2023\n" + "0120231024PETR4 PETROBRAS PN0000000123456000000000000001000\n" + "0120231024VALE3 VALE ON 0000000067890 \n" + "0120231025ITUB4 ITAU UNI PN 0000000004321000000000000000050\n" + "0120231025XXXXX BROKEN 00000000ABCDE000000000000000001\n" + "99TRAILER\n" +) + + +def layout() -> fw.Layout: + return fw.Layout( + [ + fw.Column("tipreg", 0, 2, fw.String), + fw.Column("data", 2, 8, fw.Date, format="yyyyMMdd"), + fw.Column("codneg", 10, 12, fw.String), + fw.Column("nomres", 22, 12, fw.String), + fw.Column("preabe", 34, 13, fw.Float64, scale=2, padding="0"), + fw.Column("quatot", 47, 18, fw.Int64, nullable=True, padding="0"), + ] + ) + + +@pytest.fixture +def cotahist(tmp_path): + path = tmp_path / "cotahist.txt" + path.write_bytes(CONTENT.encode("utf-8")) + return path + + +@pytest.fixture +def options() -> fw.ReadOptions: + return fw.ReadOptions(header_lines=1, skip_invalid=True, degree_of_parallelism=1) diff --git a/python/tests/test_arrow.py b/python/tests/test_arrow.py new file mode 100644 index 0000000..7ef4713 --- /dev/null +++ b/python/tests/test_arrow.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import datetime as dt + +import polars as pl +import pyarrow as pa + +import fixedwidth as fw +from conftest import CONTENT, layout + + +def test_read_arrow(cotahist, options): + table = fw.read_arrow(cotahist, layout(), options) + + assert table.schema.field("preabe").type == pa.float64() + assert table.schema.field("data").type == pa.date32() + assert table.schema.field("quatot").nullable + assert table.column("codneg").to_pylist() == ["PETR4", "VALE3", "ITUB4"] + assert table.column("data").to_pylist()[2] == dt.date(2023, 10, 25) + + +def test_read_arrow_decimal(): + columns = fw.Layout([fw.Column("v", 0, 8, fw.Decimal, scale=2)]) + + table = fw.read_arrow(b"00012345\n", columns) + + assert table.schema.field("v").type == pa.decimal128(18, 2) + assert str(table.column("v")[0].as_py()) == "123.45" + + +def test_arrow_stream_to_pyarrow_and_polars(cotahist, options): + reader_options = fw.ReaderOptions(read=options, batch_rows=2) + + batches = list(pa.RecordBatchReader.from_stream(fw.read_arrow_stream(cotahist, layout(), reader_options))) + frame = pl.DataFrame(fw.read_arrow_stream(CONTENT.encode("utf-8"), layout(), reader_options)) + + assert [b.num_rows for b in batches] == [2, 1] + assert frame["codneg"].to_list() == ["PETR4", "VALE3", "ITUB4"] + + +def test_arrow_stream_error_surfaces(options): + columns = fw.Layout([fw.Column("n", 0, 2, fw.Int64)]) + stream = fw.read_arrow_stream(b"01\nXX\n", columns, fw.ReaderOptions(batch_rows=1)) + + reader = pa.RecordBatchReader.from_stream(stream) + reader.read_next_batch() + try: + reader.read_next_batch() + raise AssertionError("expected an error") + except OSError as error: + assert "Line 2" in str(error) + + +def test_table_to_arrow(cotahist, options): + table = fw.read(cotahist, layout(), options) + + assert table.to_arrow().column("preabe").to_pylist() == [1234.56, 678.9, 43.21] diff --git a/python/tests/test_native.py b/python/tests/test_native.py new file mode 100644 index 0000000..e2be040 --- /dev/null +++ b/python/tests/test_native.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import pytest + +import fixedwidth as fw +from fixedwidth import _native + + +def test_abi_version_matches(): + assert _native.load_library().fw_abi_version() == _native.FW_ABI_VERSION + + +def test_overlapping_columns_raise(): + with pytest.raises(fw.FixedWidthError, match="Overlapping"): + fw.Layout([fw.Column("a", 0, 5, fw.String), fw.Column("b", 3, 5, fw.String)]) + + +def test_layout_validates_options(): + with pytest.raises(fw.FixedWidthError, match="scale"): + fw.Layout([fw.Column("a", 0, 5, fw.Int64, scale=2)]) + with pytest.raises(ValueError, match="one ASCII character"): + fw.Layout([fw.Column("a", 0, 5, fw.String, padding="ab")]) + + +def test_layout_lists_its_columns(): + columns = [fw.Column("a", 0, 2, fw.String), fw.Column("b", 2, 3, fw.Int64)] + + assert fw.Layout(columns).columns == columns + + +def test_unknown_encoding_raises(): + with pytest.raises(ValueError, match="unknown encoding"): + fw.read(b"ab\n", fw.Layout([fw.Column("a", 0, 2, fw.String)]), fw.ReadOptions(encoding="klingon")) diff --git a/python/tests/test_read.py b/python/tests/test_read.py new file mode 100644 index 0000000..d1448d0 --- /dev/null +++ b/python/tests/test_read.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import datetime as dt +import gc +import io + +import numpy as np +import pandas as pd +import pytest + +import fixedwidth as fw +from conftest import CONTENT, layout + + +def test_read_every_type(cotahist, options): + table = fw.read(cotahist, layout(), options) + + assert table.num_rows == 3 + assert table.column_names == ["tipreg", "data", "codneg", "nomres", "preabe", "quatot"] + assert table["codneg"] == ["PETR4", "VALE3", "ITUB4"] + assert table["nomres"][2] == "ITAU UNI PN" + assert table["preabe"].tolist() == [1234.56, 678.9, 43.21] + assert table["quatot"].dtype == np.int64 + assert table.validity("quatot").tolist() == [True, False, True] + assert table.validity("preabe") is None + days = (dt.date(2023, 10, 24) - dt.date(1970, 1, 1)).days + assert table["data"].tolist() == [days, days, days + 1] + + +def test_read_options_header_and_skip(cotahist): + table = fw.read(cotahist, layout(), fw.ReadOptions(header_lines=1, skip_invalid=True)) + + assert table.num_rows == 3 + assert table.lines_skipped == 3 + + +def test_parse_error_has_line_and_column(cotahist): + with pytest.raises(fw.ParseError) as error: + fw.read(cotahist, layout(), fw.ReadOptions(header_lines=1)) + + assert error.value.line_number == 5 + assert error.value.column == 4 + assert "ABCDE" in str(error.value) + + +def test_read_memory_equals_file(cotahist, options): + from_file = fw.read(cotahist, layout(), options) + from_bytes = fw.read(CONTENT.encode("utf-8"), layout(), options) + from_view = fw.read(memoryview(bytearray(CONTENT.encode("utf-8"))), layout(), options) + + for name in from_file.column_names: + assert list(from_bytes[name]) == list(from_file[name]) + assert list(from_view[name]) == list(from_file[name]) + + +def test_parallel_equals_sequential(tmp_path): + body = "".join(f"{i:06d}{i * 7 % 1000:04d}\n" for i in range(20000)) + path = tmp_path / "big.txt" + path.write_text(body) + columns = fw.Layout([fw.Column("id", 0, 6, fw.Int64), fw.Column("n", 6, 4, fw.Float64, scale=1)]) + + sequential = fw.read(path, columns, fw.ReadOptions(degree_of_parallelism=1)) + parallel = fw.read(path, columns, fw.ReadOptions(degree_of_parallelism=4)) + + assert np.array_equal(sequential["id"], parallel["id"]) + assert np.array_equal(sequential["n"], parallel["n"]) + assert parallel.num_rows == 20000 + + +def test_views_keep_table_alive(cotahist, options): + table = fw.read(cotahist, layout(), options) + prices = table["preabe"] + collected = [] + table._owner._on_free = collected.append + + del table + gc.collect() + assert prices.tolist() == [1234.56, 678.9, 43.21] + assert collected == [] + + del prices + gc.collect() + assert collected == [True] + + +def test_read_pandas_matches_read_fwf(cotahist, options): + frame = fw.read_pandas(cotahist, layout(), options) + + expected = pd.read_fwf( + io.StringIO(CONTENT), + colspecs=[(0, 2), (2, 10), (10, 22), (22, 34), (34, 47), (47, 65)], + names=["tipreg", "data", "codneg", "nomres", "preabe", "quatot"], + dtype={"tipreg": str}, + skiprows=1, + skipfooter=2, + engine="python", + ) + expected = expected[expected["codneg"] != "XXXXX"].reset_index(drop=True) + assert frame["codneg"].tolist() == expected["codneg"].tolist() + assert frame["nomres"].tolist() == expected["nomres"].tolist() + assert frame["preabe"].tolist() == (expected["preabe"] / 100).tolist() + assert frame["quatot"].tolist()[0::2] == expected["quatot"].tolist()[0::2] + assert pd.isna(frame["quatot"][1]) + assert frame["data"].tolist()[0] == dt.date(2023, 10, 24) + + +def test_read_polars(cotahist, options): + frame = fw.read_polars(cotahist, layout(), options) + + assert frame.shape == (3, 6) + assert frame["codneg"].to_list() == ["PETR4", "VALE3", "ITUB4"] + assert frame["quatot"].null_count() == 1 + + +def test_read_batches(cotahist, options): + batches = list(fw.read_batches(cotahist, layout(), fw.ReaderOptions(read=options, batch_rows=2))) + + assert [b.num_rows for b in batches] == [2, 1] + assert batches[0]["codneg"] + batches[1]["codneg"] == ["PETR4", "VALE3", "ITUB4"] + + +def test_read_latin1(tmp_path): + path = tmp_path / "latin1.txt" + path.write_bytes("AÇÃO 01\n".encode("latin-1")) + columns = fw.Layout([fw.Column("name", 0, 6, fw.String), fw.Column("n", 6, 2, fw.Int64)]) + + assert fw.read(path, columns, fw.ReadOptions(encoding="latin-1"))["name"] == ["AÇÃO"] diff --git a/python/tests/test_write.py b/python/tests/test_write.py new file mode 100644 index 0000000..be260cb --- /dev/null +++ b/python/tests/test_write.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import datetime as dt + +import pandas as pd +import polars as pl +import pyarrow as pa +import pytest + +import fixedwidth as fw +from conftest import CONTENT, layout + +QUOTES = "".join(line + "\n" for line in CONTENT.splitlines()[1:4]) + + +def test_write_roundtrip_pandas_polars_pyarrow(cotahist, options): + frame = fw.read_pandas(cotahist, layout(), options) + + from_pandas = fw.write_bytes(frame, layout()) + from_polars = fw.write_bytes(pl.from_pandas(frame), layout()) + from_arrow = fw.write_bytes(pa.Table.from_pandas(frame, preserve_index=False), layout()) + + assert from_pandas == from_polars == from_arrow + assert from_pandas.decode("utf-8") == QUOTES + + +def test_write_bytes_equals_file(tmp_path, cotahist, options): + frame = fw.read_arrow(cotahist, layout(), options) + path = tmp_path / "out.txt" + + fw.write(frame, path, layout(), fw.WriteOptions(degree_of_parallelism=4, batch_rows=1)) + + assert path.read_bytes() == fw.write_bytes(frame, layout()) + + +def test_write_options(): + columns = fw.Layout([fw.Column("s", 0, 4, fw.String)], fw.LayoutOptions(line_length=6, filler=".")) + data = pa.table({"s": ["AÇÃO", None]}) + + written = fw.write_bytes(data, columns, fw.WriteOptions(encoding="latin-1", newline="\r\n")) + + assert written == "AÇÃO..\r\n".encode("latin-1") + b" ..\r\n" + + +def test_write_values_from_python_types(): + columns = fw.Layout( + [ + fw.Column("d", 0, 8, fw.Date, format="yyyyMMdd"), + fw.Column("p", 8, 8, fw.Float64, scale=2, padding="0"), + fw.Column("q", 16, 4, fw.Int64), + ] + ) + frame = pd.DataFrame({"d": [dt.date(2023, 10, 24)], "p": [-12.5], "q": [7]}) + + assert fw.write_bytes(frame, columns) == b"20231024-0001250 7\n" + + +def test_write_mismatched_columns_raise(): + columns = fw.Layout([fw.Column("a", 0, 2, fw.String), fw.Column("b", 2, 2, fw.Int64)]) + + with pytest.raises(fw.FixedWidthError, match="b"): + fw.write_bytes(pa.table({"a": ["x"]}), columns) From 60c45c0e8f97d1d9c70de283ee89153884d8fb20 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 01:44:59 -0300 Subject: [PATCH 43/85] docs: benchmark the Python reader against pandas.read_fwf python/benchmarks/bench_read.py generates a COTAHIST-shaped file (all 26 fields of B3's record type 01, with header and trailer), checks every reader against pandas.read_fwf, and times them. On 1,000,000 lines fw.read_pandas is 52x faster than read_fwf with identical values, 16x on a single thread. --- docs/README.md | 2 + docs/python-benchmarks.md | 29 ++++++++ python/benchmarks/bench_read.py | 124 ++++++++++++++++++++++++++++++++ python/benchmarks/cotahist.py | 118 ++++++++++++++++++++++++++++++ 4 files changed, 273 insertions(+) create mode 100644 docs/python-benchmarks.md create mode 100644 python/benchmarks/bench_read.py create mode 100644 python/benchmarks/cotahist.py diff --git a/docs/README.md b/docs/README.md index f8b95f4..1e074e5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -12,6 +12,8 @@ a constraint; the numbers that justified it live here, so they can be re-run and its own enumerator. - [fluent-layout.md](fluent-layout.md) — reading and writing through a `FixedWidthLayout` against the classes it replaced. +- [python-benchmarks.md](python-benchmarks.md) — the Python package reading a COTAHIST-shaped file + against `pandas.read_fwf`. Usage documentation is the [README](../README.md); the shape of the code is [ARCHITECTURE.md](../ARCHITECTURE.md). diff --git a/docs/python-benchmarks.md b/docs/python-benchmarks.md new file mode 100644 index 0000000..43efc7d --- /dev/null +++ b/docs/python-benchmarks.md @@ -0,0 +1,29 @@ +# Python: fixedwidth against pandas.read_fwf + +`python/benchmarks/bench_read.py` generates a synthetic file shaped like B3's COTAHIST quote history (245-byte +lines, all 26 fields of record type 01, a 00 header and a 99 trailer; layout in `python/benchmarks/cotahist.py`), +checks that every reader returns the same values as `pandas.read_fwf` given the same column specs (fixedwidth +exactly, the polars reference within floating-point rounding), and times each one, best of several runs. + + python python/benchmarks/bench_read.py --lines 1000000 --runs 5 + +`read_fwf` uses its Python engine, which `skipfooter` needs to drop the trailer. + +## Results + +1,000,000 lines (235 MiB). AMD Ryzen 7 5700X (8 cores, 16 threads), Windows 10, Python 3.14.7, pandas 3.1.0, +pyarrow 26.0.0, polars 2.0.0. `read_fwf` best of 1 run, the others best of 5. + +| Reader | Seconds | vs read_fwf | +|---|---:|---:| +| pandas.read_fwf | 14.955 | 1.0x | +| polars str.slice (reference) | 0.306 | 49x | +| fw.read_pandas | 0.290 | 52x | +| fw.read_pandas, 1 thread | 0.940 | 16x | +| fw.read_polars | 0.273 | 55x | +| fw.read_arrow | 0.253 | 59x | +| fw.read (NumPy views) | 0.263 | 57x | +| fw.read, 1 thread | 0.947 | 16x | + +A single thread is already 16x faster than `read_fwf`; the default, one thread per processor, adds another 3.6x. +`fw.read_pandas` costs about 40 ms over `fw.read_arrow`, pyarrow's conversion to pandas. diff --git a/python/benchmarks/bench_read.py b/python/benchmarks/bench_read.py new file mode 100644 index 0000000..5055c96 --- /dev/null +++ b/python/benchmarks/bench_read.py @@ -0,0 +1,124 @@ +"""Reads a synthetic COTAHIST file with fixedwidth and with pandas.read_fwf, checks they agree, and times them. + + python python/benchmarks/bench_read.py [--lines 1000000] [--runs 3] [--fwf-runs 1] +""" + +from __future__ import annotations + +import argparse +import platform +import sys +import tempfile +import time +from pathlib import Path + +import numpy as np +import pandas as pd +import polars as pl + +import fixedwidth as fw + +sys.path.insert(0, str(Path(__file__).parent)) +import cotahist # noqa: E402 + +STRING_FIELDS = [name for name, _, _, kind, _ in cotahist.FIELDS if kind is fw.String] + + +def read_fwf(path: Path) -> pd.DataFrame: + frame = pd.read_fwf( + path, + colspecs=cotahist.colspecs(), + names=[name for name, *_ in cotahist.FIELDS], + dtype={name: str for name in STRING_FIELDS}, + skiprows=1, + skipfooter=1, + engine="python", + ) + for name, _, _, kind, scale in cotahist.FIELDS: + if kind is fw.Float64: + frame[name] = frame[name] / 10**scale + if kind is fw.Date: + frame[name] = pd.to_datetime(frame[name].astype(str), format="%Y%m%d").dt.date + return frame + + +def polars_slices(path: Path) -> pl.DataFrame: + lines = pl.read_csv(path, has_header=False, separator="\x01", new_columns=["line"], quote_char=None, skip_rows=1) + lines = lines.filter(pl.col("line").str.starts_with("01")) + expressions = [] + for name, first, length, kind, scale in cotahist.FIELDS: + text = pl.col("line").str.slice(first - 1, length).str.strip_chars() + if kind is fw.String: + expressions.append(text.alias(name)) + elif kind is fw.Date: + expressions.append(text.str.to_date("%Y%m%d").alias(name)) + elif kind is fw.Float64: + expressions.append((text.cast(pl.Int64) / 10**scale).alias(name)) + else: + expressions.append(text.cast(pl.Int64).alias(name)) + return lines.select(expressions) + + +def check(native: pd.DataFrame, expected: pd.DataFrame, exact: bool = True) -> None: + assert len(native) == len(expected), (len(native), len(expected)) + for name, _, _, kind, _ in cotahist.FIELDS: + left, right = native[name], expected[name] + if kind is fw.String: + assert left.fillna("").tolist() == right.fillna("").tolist(), name + elif kind is fw.Date: + assert [pd.Timestamp(v).date() for v in left] == [pd.Timestamp(v).date() for v in right], name + else: + same = np.array_equal if exact else np.allclose + assert same(left.to_numpy(dtype=float), right.to_numpy(dtype=float)), name + + +def best(runs: int, work) -> float: + times = [] + for _ in range(runs): + start = time.perf_counter() + work() + times.append(time.perf_counter() - start) + return min(times) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--lines", type=int, default=1_000_000) + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--fwf-runs", type=int, default=1, help="read_fwf is slow; one run is usually enough") + args = parser.parse_args() + + layout = cotahist.layout() + options = fw.ReadOptions(header_lines=1, skip_invalid=True) + sequential = fw.ReadOptions(header_lines=1, skip_invalid=True, degree_of_parallelism=1) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "COTAHIST_SYNTHETIC.TXT" + cotahist.generate(path, args.lines) + size = path.stat().st_size / 2**20 + print(f"{args.lines:,} lines, {size:.0f} MiB, Python {platform.python_version()}, pandas {pd.__version__}, polars {pl.__version__}") + + expected = read_fwf(path) + check(fw.read_pandas(path, layout, options), expected) + check(fw.read_pandas(path, layout, sequential), expected) + check(polars_slices(path).to_pandas(), expected, exact=False) + print("values agree with pandas.read_fwf (fixedwidth exactly; the polars reference within rounding)") + + rows = [ + ("pandas.read_fwf", best(args.fwf_runs, lambda: read_fwf(path))), + ("polars str.slice (reference)", best(args.runs, lambda: polars_slices(path))), + ("fw.read_pandas", best(args.runs, lambda: fw.read_pandas(path, layout, options))), + ("fw.read_pandas, 1 thread", best(args.runs, lambda: fw.read_pandas(path, layout, sequential))), + ("fw.read_polars", best(args.runs, lambda: fw.read_polars(path, layout, options))), + ("fw.read_arrow", best(args.runs, lambda: fw.read_arrow(path, layout, options))), + ("fw.read (NumPy views)", best(args.runs, lambda: fw.read(path, layout, options))), + ("fw.read, 1 thread", best(args.runs, lambda: fw.read(path, layout, sequential))), + ] + baseline = rows[0][1] + print(f"{'reader':<32}{'seconds':>10}{'vs read_fwf':>14}") + for name, seconds in rows: + print(f"{name:<32}{seconds:>10.3f}{baseline / seconds:>13.1f}x") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/benchmarks/cotahist.py b/python/benchmarks/cotahist.py new file mode 100644 index 0000000..f82caae --- /dev/null +++ b/python/benchmarks/cotahist.py @@ -0,0 +1,118 @@ +"""B3's COTAHIST quote history layout (record type 01) and a synthetic file generator. + +Positions follow B3's "Layout do Arquivo - Cotações Históricas": 26 fields that tile bytes 1..245 +exactly (checked in `check_layout`). The real files are B3's and are not committed; `generate` writes a +file of the same shape with random quotes, a 00 header and a 99 trailer. +""" + +from __future__ import annotations + +import random +from pathlib import Path + +import fixedwidth as fw + +LINE_LENGTH = 245 + +# (name, 1-based first byte, length, type, scale) +FIELDS = [ + ("tipreg", 1, 2, fw.String, 0), + ("data_pregao", 3, 8, fw.Date, 0), + ("codbdi", 11, 2, fw.String, 0), + ("codneg", 13, 12, fw.String, 0), + ("tpmerc", 25, 3, fw.Int64, 0), + ("nomres", 28, 12, fw.String, 0), + ("especi", 40, 10, fw.String, 0), + ("prazot", 50, 3, fw.String, 0), + ("modref", 53, 4, fw.String, 0), + ("preabe", 57, 13, fw.Float64, 2), + ("premax", 70, 13, fw.Float64, 2), + ("premin", 83, 13, fw.Float64, 2), + ("premed", 96, 13, fw.Float64, 2), + ("preult", 109, 13, fw.Float64, 2), + ("preofc", 122, 13, fw.Float64, 2), + ("preofv", 135, 13, fw.Float64, 2), + ("totneg", 148, 5, fw.Int64, 0), + ("quatot", 153, 18, fw.Int64, 0), + ("voltot", 171, 18, fw.Float64, 2), + ("preexe", 189, 13, fw.Float64, 2), + ("indopc", 202, 1, fw.Int64, 0), + ("datven", 203, 8, fw.Date, 0), + ("fatcot", 211, 7, fw.Int64, 0), + ("ptoexe", 218, 13, fw.Float64, 6), + ("codisi", 231, 12, fw.String, 0), + ("dismes", 243, 3, fw.Int64, 0), +] + + +def check_layout() -> None: + position = 1 + for name, first, length, _, _ in FIELDS: + assert first == position, f"{name} starts at {first}, expected {position}" + position += length + assert position - 1 == LINE_LENGTH + + +def layout() -> fw.Layout: + check_layout() + return fw.Layout( + [ + fw.Column( + name, first - 1, length, kind, + scale=scale, + format="yyyyMMdd" if kind is fw.Date else None, + padding="0" if kind in (fw.Int64, fw.Float64) else None, + ) + for name, first, length, kind, scale in FIELDS + ] + ) + + +def colspecs() -> list[tuple[int, int]]: + return [(first - 1, first - 1 + length) for _, first, length, _, _ in FIELDS] + + +TICKERS = [f"{a}{b}{c}{d}{n}" for a in "ABCPV" for b in "AEIO" for c in "TRLN" for d in "SAEB" for n in (3, 4, 11)] + + +def _line(rng: random.Random, day: int) -> str: + ticker = rng.choice(TICKERS) + open_ = rng.randint(100, 9_999_999) + prices = [open_ + rng.randint(-500, 500) for _ in range(6)] + trades = rng.randint(1, 99_999) + quantity = rng.randint(100, 10**12) + return ( + "01" + + f"202310{day:02d}" + + "02" + + f"{ticker:<12}" + + "010" + + f"{ticker[:4] + ' SA':<12}" + + f"{'ON NM':<10}" + + " " + + "R$ " + + "".join(f"{p:013d}" for p in [open_, *prices]) + + f"{trades:05d}" + + f"{quantity:018d}" + + f"{quantity * open_ // 100:018d}" + + f"{0:013d}" + + "0" + + "99991231" + + "0000001" + + f"{0:013d}" + + f"BR{ticker[:4]}ACNOR{rng.randint(0, 9)}" + + f"{rng.randint(0, 999):03d}" + ) + + +def generate(path: Path, lines: int, seed: int = 2023) -> None: + rng = random.Random(seed) + header = f"{'00COTAHIST.2023BOVESPA 20231229':<{LINE_LENGTH}}" + trailer = f"{'99COTAHIST.2023BOVESPA 20231229' + f'{lines + 2:011d}':<{LINE_LENGTH}}" + with path.open("w", encoding="ascii", newline="\n") as file: + file.write(header + "\n") + for i in range(lines): + line = _line(rng, 2 + i * 27 // max(1, lines)) + assert len(line) == LINE_LENGTH, len(line) + file.write(line + "\n") + file.write(trailer + "\n") From 9d30a6eaec18d4888f4f9017d4f13e6b17095b7b Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 01:48:12 -0300 Subject: [PATCH 44/85] feat(cpp): header-only C++ wrapper fixedwidth.hpp and fixedwidth_arrow.hpp wrap the C ABI in C++23 RAII types (Layout, Table, Reader, ArrowResult, ArrowStream) with defaulted options structs and std::expected results. cpp/ builds the fw::fixedwidth target against the published library, generating the Windows import library from include/fixedwidth.def, which a test keeps in step with the exports. --- cpp/CMakeLists.txt | 39 ++ cpp/README.md | 29 ++ cpp/cmake/NativeLib.cmake | 41 ++ cpp/tests/CMakeLists.txt | 13 + cpp/tests/arrow.cpp | 35 ++ cpp/tests/common.hpp | 43 ++ cpp/tests/read.cpp | 72 ++++ cpp/tests/write.cpp | 41 ++ .../include/fixedwidth.def | 20 + .../include/fixedwidth.hpp | 393 ++++++++++++++++++ .../include/fixedwidth_arrow.hpp | 123 ++++++ .../Native/NativeApiTests.cs | 15 + 12 files changed, 864 insertions(+) create mode 100644 cpp/CMakeLists.txt create mode 100644 cpp/README.md create mode 100644 cpp/cmake/NativeLib.cmake create mode 100644 cpp/tests/CMakeLists.txt create mode 100644 cpp/tests/arrow.cpp create mode 100644 cpp/tests/common.hpp create mode 100644 cpp/tests/read.cpp create mode 100644 cpp/tests/write.cpp create mode 100644 src/FixedWidthParser.Native/include/fixedwidth.def create mode 100644 src/FixedWidthParser.Native/include/fixedwidth.hpp create mode 100644 src/FixedWidthParser.Native/include/fixedwidth_arrow.hpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt new file mode 100644 index 0000000..6c883df --- /dev/null +++ b/cpp/CMakeLists.txt @@ -0,0 +1,39 @@ +# The C++ wrapper: a header-only INTERFACE target, fw::fixedwidth, linked against the published +# FixedWidthParser.Native library. +# +# python python/scripts/build_native.py +# cmake -S cpp -B build/cpp -DFIXEDWIDTH_NATIVE_LIB= +# cmake --build build/cpp && ctest --test-dir build/cpp --output-on-failure +cmake_minimum_required(VERSION 3.16) +project(FixedWidthParser LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 23) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +set(_include_dir "${CMAKE_CURRENT_SOURCE_DIR}/../src/FixedWidthParser.Native/include") +if(WIN32) + set(_lib_name "FixedWidthParser.Native.dll") +elseif(APPLE) + set(_lib_name "FixedWidthParser.Native.dylib") +else() + set(_lib_name "FixedWidthParser.Native.so") +endif() +set(FIXEDWIDTH_NATIVE_LIB "${CMAKE_CURRENT_SOURCE_DIR}/../python/src/fixedwidth/_lib/${_lib_name}" + CACHE FILEPATH "Path to the published FixedWidthParser.Native shared library") +if(NOT EXISTS "${FIXEDWIDTH_NATIVE_LIB}") + message(FATAL_ERROR "FixedWidthParser.Native not found at ${FIXEDWIDTH_NATIVE_LIB}; run python/scripts/build_native.py or set FIXEDWIDTH_NATIVE_LIB") +endif() + +include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/NativeLib.cmake") +fixedwidth_import_native_lib("${FIXEDWIDTH_NATIVE_LIB}" "${_include_dir}/fixedwidth.def") + +add_library(fw_fixedwidth INTERFACE) +target_include_directories(fw_fixedwidth INTERFACE "${_include_dir}") +target_compile_features(fw_fixedwidth INTERFACE cxx_std_23) +target_link_libraries(fw_fixedwidth INTERFACE fw::native) +add_library(fw::fixedwidth ALIAS fw_fixedwidth) + +if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) + enable_testing() + add_subdirectory(tests) +endif() diff --git a/cpp/README.md b/cpp/README.md new file mode 100644 index 0000000..3b966d6 --- /dev/null +++ b/cpp/README.md @@ -0,0 +1,29 @@ +# FixedWidthParser for C++ + +`fixedwidth.hpp` (and `fixedwidth_arrow.hpp`) wrap the C ABI in C++23: RAII `Layout`, `Table`, `Reader`, +`ArrowResult` and `ArrowStream`, options structs with defaults, and `std::expected` results. + +```cpp +#include + +std::vector columns{ + {.name = "codneg", .start = 12, .length = 12, .type = fw::Type::String}, + {.name = "preabe", .start = 56, .length = 13, .type = fw::Type::Float64, .scale = 2}, +}; +auto layout = fw::Layout::create(columns).value(); +auto table = fw::read_file("COTAHIST_A2023.TXT", layout, {.header_lines = 1, .skip_invalid = true}); +if (!table) { std::puts(table.error().message.c_str()); return 1; } +std::span prices = table->values(1); +``` + +## CMake + + python python/scripts/build_native.py + cmake -S cpp -B build/cpp + cmake --build build/cpp && ctest --test-dir build/cpp --output-on-failure + +`add_subdirectory(cpp)` gives the `fw::fixedwidth` target; `FIXEDWIDTH_NATIVE_LIB` points at the published +library (by default the copy in `python/src/fixedwidth/_lib`). On Windows an import library is generated from +`include/fixedwidth.def` with `lib.exe` or `dlltool`, and `fixedwidth_copy_native_library()` copies +the DLL next to an executable. GCC's `-Wextra` warns about designated initializers that leave defaulted +options out; that is the intended use, so build with `-Wno-missing-field-initializers`. diff --git a/cpp/cmake/NativeLib.cmake b/cpp/cmake/NativeLib.cmake new file mode 100644 index 0000000..fd2f3cc --- /dev/null +++ b/cpp/cmake/NativeLib.cmake @@ -0,0 +1,41 @@ +# fixedwidth_import_native_lib( ) creates the IMPORTED target fw::native. +# +# NativeAOT ships no import library on Windows, so one is generated from the checked-in .def (lib.exe +# under MSVC, dlltool under MinGW), with a LIBRARY line naming the actual DLL so the loader finds it. +function(fixedwidth_import_native_lib library def_file) + add_library(fixedwidth_native SHARED IMPORTED GLOBAL) + set_target_properties(fixedwidth_native PROPERTIES IMPORTED_LOCATION "${library}") + + if(WIN32) + get_filename_component(_dll_name "${library}" NAME) + set(_generated_def "${CMAKE_BINARY_DIR}/fixedwidth.generated.def") + set(_implib "${CMAKE_BINARY_DIR}/fixedwidth_native.lib") + file(READ "${def_file}" _exports) + file(WRITE "${_generated_def}" "LIBRARY ${_dll_name}\n${_exports}") + if(MSVC) + find_program(_lib_exe NAMES lib REQUIRED) + execute_process(COMMAND "${_lib_exe}" "/def:${_generated_def}" "/out:${_implib}" "/machine:x64" + RESULT_VARIABLE _result) + else() + find_program(_dlltool NAMES dlltool REQUIRED) + execute_process(COMMAND "${_dlltool}" -d "${_generated_def}" -l "${_implib}" RESULT_VARIABLE _result) + endif() + if(NOT _result EQUAL 0) + message(FATAL_ERROR "generating the import library for ${_dll_name} failed") + endif() + set_target_properties(fixedwidth_native PROPERTIES IMPORTED_IMPLIB "${_implib}") + endif() + + add_library(fw::native ALIAS fixedwidth_native) +endfunction() + +# Copies the native library next to an executable: on Windows a DLL beside neither the executable nor +# PATH is not found, and the process exits before main. A no-op elsewhere, where the rpath resolves it. +function(fixedwidth_copy_native_library target) + if(NOT WIN32) + return() + endif() + add_custom_command(TARGET ${target} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different "$" "$" + COMMENT "Copying FixedWidthParser.Native next to ${target}") +endfunction() diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt new file mode 100644 index 0000000..8ba89a5 --- /dev/null +++ b/cpp/tests/CMakeLists.txt @@ -0,0 +1,13 @@ +foreach(test read write arrow) + add_executable(fw_cpp_${test} ${test}.cpp) + target_link_libraries(fw_cpp_${test} PRIVATE fw::fixedwidth) + if(MSVC) + target_compile_options(fw_cpp_${test} PRIVATE /W4 /WX) + else() + # Designated initializers that leave defaulted options out are the intended use. + target_compile_options(fw_cpp_${test} PRIVATE -Wall -Wextra -Werror -Wno-missing-field-initializers) + endif() + add_test(NAME fw_cpp_${test} COMMAND fw_cpp_${test} WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + set_tests_properties(fw_cpp_${test} PROPERTIES PASS_REGULAR_EXPRESSION "${test} ok") + fixedwidth_copy_native_library(fw_cpp_${test}) +endforeach() diff --git a/cpp/tests/arrow.cpp b/cpp/tests/arrow.cpp new file mode 100644 index 0000000..b2b5726 --- /dev/null +++ b/cpp/tests/arrow.cpp @@ -0,0 +1,35 @@ +#include "common.hpp" + +#include + +#include + +int main() +{ + auto layout = cotahist_layout(); + + auto result = fw::read_arrow_memory(content, layout, skip_header_and_invalid(4)); + CHECK(result.has_value()); + const ArrowSchema& schema = result->schema(); + const ArrowArray& array = result->array(); + CHECK(std::strcmp(schema.format, "+s") == 0 && schema.n_children == 4); + CHECK(std::strcmp(schema.children[2]->name, "preabe") == 0); + CHECK(array.length == 3 && array.children[3]->null_count == 1); + CHECK(static_cast(array.children[2]->buffers[1])[1] == 678.9); + + auto stream = fw::read_arrow_stream_memory(content, layout, {.read = skip_header_and_invalid(), .batch_rows = 2}); + CHECK(stream.has_value()); + ArrowSchema stream_schema; + CHECK(stream->get()->get_schema(stream->get(), &stream_schema) == 0); + stream_schema.release(&stream_schema); + int64_t rows = 0; + ArrowArray batch; + while (stream->get()->get_next(stream->get(), &batch) == 0 && batch.release != nullptr) { + rows += batch.length; + batch.release(&batch); + } + CHECK(rows == 3); + + std::puts("arrow ok"); + return 0; +} diff --git a/cpp/tests/common.hpp b/cpp/tests/common.hpp new file mode 100644 index 0000000..8ba45d9 --- /dev/null +++ b/cpp/tests/common.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include + +#include +#include +#include +#include + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "%s:%d: check failed: %s\n", __FILE__, __LINE__, #cond); \ + std::exit(1); \ + } \ + } while (0) + +// A COTAHIST-like file: a header, three quotes (one with a blank quantity), an invalid line, a trailer. +inline constexpr std::string_view content = + "00COTAHIST.2023\n" + "01PETR4 0000000123456000000000000001000\n" + "01VALE3 0000000067890 \n" + "01ITUB4 0000000004321000000000000000050\n" + "01XXXXX 00000000ABCDE000000000000000001\n" + "99TRAILER\n"; + +inline fw::Layout cotahist_layout() +{ + std::vector columns{ + {.name = "tipreg", .start = 0, .length = 2, .type = fw::Type::String}, + {.name = "codneg", .start = 2, .length = 12, .type = fw::Type::String}, + {.name = "preabe", .start = 14, .length = 13, .type = fw::Type::Float64, .scale = 2, .padding = '0'}, + {.name = "quatot", .start = 27, .length = 18, .type = fw::Type::Int64, .nullable = true, .padding = '0'}, + }; + auto layout = fw::Layout::create(columns); + CHECK(layout.has_value()); + return std::move(*layout); +} + +inline fw::ReadOptions skip_header_and_invalid(int32_t degree = 1) +{ + return {.header_lines = 1, .skip_invalid = true, .degree_of_parallelism = degree}; +} diff --git a/cpp/tests/read.cpp b/cpp/tests/read.cpp new file mode 100644 index 0000000..8f1d340 --- /dev/null +++ b/cpp/tests/read.cpp @@ -0,0 +1,72 @@ +#include "common.hpp" + +#include +#include + +static void check_table(const fw::Table& table) +{ + CHECK(table.row_count() == 3); + CHECK(table.column_count() == 4); + CHECK(table.string(1, 0) == "PETR4"); + CHECK(table.string(1, 2) == "ITUB4"); + auto prices = table.values(2); + CHECK(prices.size() == 3 && prices[0] == 1234.56 && prices[2] == 43.21); + auto quantities = table.values(3); + CHECK(quantities[0] == 1000 && quantities[2] == 50); + CHECK(table.is_valid(3, 0) && !table.is_valid(3, 1)); + CHECK(table.is_valid(2, 1)); +} + +int main() +{ + auto layout = cotahist_layout(); + + auto memory = fw::read_memory(content, layout, skip_header_and_invalid()); + CHECK(memory.has_value()); + check_table(*memory); + CHECK(memory->lines_read() == 3 && memory->lines_skipped() == 3); + + auto parallel = fw::read_memory(content, layout, skip_header_and_invalid(4)); + CHECK(parallel.has_value()); + check_table(*parallel); + + const std::string path = "fw_cpp_read.txt"; + std::FILE* file = std::fopen(path.c_str(), "wb"); + CHECK(file != nullptr); + std::fwrite(content.data(), 1, content.size(), file); + std::fclose(file); + auto from_file = fw::read_file(path, layout, skip_header_and_invalid()); + CHECK(from_file.has_value()); + check_table(*from_file); + std::remove(path.c_str()); + + auto failed = fw::read_memory(content, layout, {.header_lines = 1}); + CHECK(!failed.has_value()); + CHECK(failed.error().status == FW_PARSE_ERROR); + CHECK(failed.error().line_number == 5 && failed.error().column == 2); + CHECK(failed.error().message.find("ABCDE") != std::string::npos); + + auto reader = fw::Reader::open_memory(content, layout, {.read = skip_header_and_invalid(), .batch_rows = 2}); + CHECK(reader.has_value()); + int64_t rows = 0; + while (true) { + auto batch = reader->next(); + CHECK(batch.has_value()); + if (!batch->has_value()) { + break; + } + rows += (*batch)->row_count(); + } + CHECK(rows == 3); + + std::vector overlap{ + {.name = "a", .start = 0, .length = 4, .type = fw::Type::String}, + {.name = "b", .start = 2, .length = 4, .type = fw::Type::String}, + }; + auto bad = fw::Layout::create(overlap); + CHECK(!bad.has_value() && bad.error().status == FW_INVALID_ARGUMENT); + CHECK(bad.error().message.find("Overlapping") != std::string::npos); + + std::puts("read ok"); + return 0; +} diff --git a/cpp/tests/write.cpp b/cpp/tests/write.cpp new file mode 100644 index 0000000..2be652f --- /dev/null +++ b/cpp/tests/write.cpp @@ -0,0 +1,41 @@ +#include "common.hpp" + +#include +#include + +int main() +{ + auto layout = cotahist_layout(); + auto table = fw::read_memory(content, layout, skip_header_and_invalid()); + CHECK(table.has_value()); + + auto serial = fw::write_memory(*table, layout, {.degree_of_parallelism = 1}); + auto parallel = fw::write_memory(*table, layout, {.degree_of_parallelism = 4, .batch_rows = 1}); + CHECK(serial.has_value() && parallel.has_value()); + CHECK(*serial == *parallel); + const std::string expected = + "01PETR4 0000000123456000000000000001000\n" + "01VALE3 0000000067890 \n" + "01ITUB4 0000000004321000000000000000050\n"; + CHECK(std::string(serial->begin(), serial->end()) == expected); + + auto crlf = fw::write_memory(*table, layout, {.newline = "\r\n"}); + CHECK(crlf.has_value() && crlf->size() == expected.size() + 3); + + const std::string path = "fw_cpp_write.txt"; + CHECK(fw::write_file(path, *table, layout).has_value()); + std::FILE* file = std::fopen(path.c_str(), "rb"); + CHECK(file != nullptr); + std::string written(expected.size(), '\0'); + CHECK(std::fread(written.data(), 1, written.size(), file) == written.size()); + std::fclose(file); + std::remove(path.c_str()); + CHECK(written == expected); + + std::vector narrow{{.name = "tipreg", .start = 0, .length = 2, .type = fw::Type::String}}; + auto mismatch = fw::write_memory(*table, *fw::Layout::create(narrow)); + CHECK(!mismatch.has_value() && mismatch.error().status == FW_INVALID_ARGUMENT); + + std::puts("write ok"); + return 0; +} diff --git a/src/FixedWidthParser.Native/include/fixedwidth.def b/src/FixedWidthParser.Native/include/fixedwidth.def new file mode 100644 index 0000000..89679f2 --- /dev/null +++ b/src/FixedWidthParser.Native/include/fixedwidth.def @@ -0,0 +1,20 @@ +EXPORTS + fw_abi_version + fw_free_buffer + fw_free_table + fw_last_error + fw_last_parse_error + fw_layout_create + fw_layout_free + fw_read_arrow_file + fw_read_arrow_memory + fw_read_arrow_stream_file + fw_read_arrow_stream_memory + fw_read_file + fw_read_memory + fw_reader_close + fw_reader_next + fw_reader_open_file + fw_reader_open_memory + fw_write_file + fw_write_memory diff --git a/src/FixedWidthParser.Native/include/fixedwidth.hpp b/src/FixedWidthParser.Native/include/fixedwidth.hpp new file mode 100644 index 0000000..bab3962 --- /dev/null +++ b/src/FixedWidthParser.Native/include/fixedwidth.hpp @@ -0,0 +1,393 @@ +// Header-only C++23 wrapper over fixedwidth.h: RAII handles, options structs with defaults, and +// std::expected results. Link against the native library (fw::fixedwidth in cpp/CMakeLists.txt). +#pragma once + +#include "fixedwidth.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fw +{ + enum class Type : int32_t + { + String = FW_T_STRING, + Int64 = FW_T_INT64, + Float64 = FW_T_FLOAT64, + Decimal = FW_T_DECIMAL, + Date = FW_T_DATE, + Time = FW_T_TIME, + Timestamp = FW_T_TIMESTAMP, + }; + + enum class Trim : int32_t { Both = FW_TRIM_BOTH, Trailing = FW_TRIM_TRAILING, Leading = FW_TRIM_LEADING }; + enum class Align : int32_t { Default = FW_ALIGN_DEFAULT, Left = FW_ALIGN_LEFT, Right = FW_ALIGN_RIGHT }; + enum class Overflow : int32_t { Throw = FW_OVERFLOW_THROW, Truncate = FW_OVERFLOW_TRUNCATE }; + + /// A failed call: its FW_* status and the library's message; line and column after FW_PARSE_ERROR. + struct Error + { + int32_t status = FW_ERROR; + std::string message; + int64_t line_number = 0; + int32_t column = -1; + }; + + template + using Result = std::expected; + + namespace detail + { + inline Error last_error(int32_t status) + { + Error error; + error.status = status; + int32_t length = 0; + std::string buffer(1024, '\0'); + if (fw_last_error(reinterpret_cast(buffer.data()), static_cast(buffer.size()), &length) == FW_BUFFER_TOO_SMALL) + { + buffer.resize(static_cast(length)); + fw_last_error(reinterpret_cast(buffer.data()), length, &length); + } + buffer.resize(static_cast(length)); + error.message = std::move(buffer); + if (status == FW_PARSE_ERROR) + { + fw_parse_error parse{.struct_size = sizeof(fw_parse_error), .line_number = 0, .column = -1}; + fw_last_parse_error(&parse); + error.line_number = parse.line_number; + error.column = parse.column; + } + return error; + } + + inline Result check(int32_t status) + { + if (status == FW_OK) + { + return {}; + } + return std::unexpected(last_error(status)); + } + + inline const uint8_t* bytes(std::string_view text) + { + return reinterpret_cast(text.data()); + } + } + + /// One column: name, zero-based byte start, byte length and type, plus everything optional. + struct ColumnSpec + { + std::string name; + int32_t start = 0; + int32_t length = 0; + Type type = Type::String; + bool nullable = false; + int32_t scale = 0; + std::optional format; + char trim_char = 0; + Trim trim_mode = Trim::Both; + Align alignment = Align::Default; + char padding = 0; + Overflow overflow = Overflow::Throw; + }; + + struct LayoutOptions + { + int32_t line_length = 0; + char filler = 0; + }; + + /// An immutable, validated layout; move-only, shareable across threads by reference. + class Layout + { + public: + static Result create(std::span columns, const LayoutOptions& options = {}) + { + std::vector specs; + specs.reserve(columns.size()); + for (const ColumnSpec& column : columns) + { + specs.push_back(fw_column_spec{ + .struct_size = sizeof(fw_column_spec), + .name = detail::bytes(column.name), + .name_len = static_cast(column.name.size()), + .start = column.start, + .length = column.length, + .type = static_cast(column.type), + .nullable = column.nullable ? 1 : 0, + .scale = column.scale, + .format = column.format ? detail::bytes(*column.format) : nullptr, + .format_len = column.format ? static_cast(column.format->size()) : 0, + .trim_char = static_cast(column.trim_char), + .trim_mode = static_cast(column.trim_mode), + .alignment = static_cast(column.alignment), + .padding = static_cast(column.padding), + .overflow = static_cast(column.overflow), + }); + } + const fw_layout_options raw{.struct_size = sizeof(fw_layout_options), .line_length = options.line_length, + .filler = static_cast(options.filler)}; + fw_layout* handle = nullptr; + if (auto ok = detail::check(fw_layout_create(specs.data(), static_cast(specs.size()), &raw, &handle)); !ok) + { + return std::unexpected(ok.error()); + } + return Layout(handle); + } + + Layout(Layout&& other) noexcept : handle_(std::exchange(other.handle_, nullptr)) {} + Layout& operator=(Layout&& other) noexcept + { + std::swap(handle_, other.handle_); + return *this; + } + Layout(const Layout&) = delete; + Layout& operator=(const Layout&) = delete; + ~Layout() + { + if (handle_ != nullptr) + { + fw_layout_free(handle_); + } + } + + const fw_layout* get() const noexcept { return handle_; } + + private: + explicit Layout(fw_layout* handle) : handle_(handle) {} + fw_layout* handle_; + }; + + struct ReadOptions + { + int64_t header_lines = 0; + bool skip_invalid = false; + int32_t degree_of_parallelism = 0; + int32_t encoding = 0; + int32_t max_line_length = 0; + std::optional skip_byte_order_mark; + + fw_read_options to_c() const + { + return { + .struct_size = sizeof(fw_read_options), + .header_lines = header_lines, + .skip_invalid = skip_invalid ? 1 : 0, + .degree_of_parallelism = degree_of_parallelism, + .encoding = encoding, + .max_line_length = max_line_length, + .skip_byte_order_mark = !skip_byte_order_mark ? FW_OPT_DEFAULT : (*skip_byte_order_mark ? FW_OPT_TRUE : FW_OPT_FALSE), + }; + } + }; + + struct ReaderOptions + { + ReadOptions read; + int64_t batch_rows = 0; + + fw_reader_options to_c() const + { + return {.struct_size = sizeof(fw_reader_options), .read = read.to_c(), .batch_rows = batch_rows}; + } + }; + + struct WriteOptions + { + int32_t encoding = 0; + std::string newline = "\n"; + int32_t degree_of_parallelism = 0; + int64_t batch_rows = 0; + }; + + /// A table the library published; move-only, freed on destruction. + class Table + { + public: + explicit Table(fw_table table, fw_read_result result = {}) noexcept : table_(table), result_(result) {} + Table(Table&& other) noexcept : table_(std::exchange(other.table_, {})), result_(other.result_) {} + Table& operator=(Table&& other) noexcept + { + std::swap(table_, other.table_); + std::swap(result_, other.result_); + return *this; + } + Table(const Table&) = delete; + Table& operator=(const Table&) = delete; + ~Table() { fw_free_table(&table_); } + + int64_t row_count() const noexcept { return table_.row_count; } + int32_t column_count() const noexcept { return table_.column_count; } + Type type(int32_t column) const noexcept { return static_cast(table_.columns[column].type); } + int64_t lines_read() const noexcept { return result_.lines_read; } + int64_t lines_skipped() const noexcept { return result_.lines_skipped; } + const fw_table& get() const noexcept { return table_; } + + /// The values of a non-string column as stored (see FW_T_*): int64_t, double or int32_t. + template + std::span values(int32_t column) const noexcept + { + const fw_column& raw = table_.columns[column]; + return {static_cast(raw.values), static_cast(raw.length)}; + } + + std::string_view string(int32_t column, int64_t row) const noexcept + { + const fw_column& raw = table_.columns[column]; + const int32_t* offsets = static_cast(raw.values); + const char* data = reinterpret_cast(raw.data); + return {data == nullptr ? "" : data + offsets[row], static_cast(offsets[row + 1] - offsets[row])}; + } + + bool is_valid(int32_t column, int64_t row) const noexcept + { + const uint8_t* bits = table_.columns[column].validity; + return bits == nullptr || (bits[row >> 3] & (1 << (row & 7))) != 0; + } + + private: + fw_table table_; + fw_read_result result_; + }; + + inline Result read_file(std::string_view path, const Layout& layout, const ReadOptions& options = {}) + { + fw_table table{}; + fw_read_result result{.struct_size = sizeof(fw_read_result), .lines_read = 0, .lines_skipped = 0}; + const fw_read_options raw = options.to_c(); + if (auto ok = detail::check(fw_read_file(detail::bytes(path), static_cast(path.size()), layout.get(), &raw, &table, &result)); !ok) + { + return std::unexpected(ok.error()); + } + return Table(table, result); + } + + inline Result
read_memory(std::string_view data, const Layout& layout, const ReadOptions& options = {}) + { + fw_table table{}; + fw_read_result result{.struct_size = sizeof(fw_read_result), .lines_read = 0, .lines_skipped = 0}; + const fw_read_options raw = options.to_c(); + if (auto ok = detail::check(fw_read_memory(detail::bytes(data), static_cast(data.size()), layout.get(), &raw, &table, &result)); !ok) + { + return std::unexpected(ok.error()); + } + return Table(table, result); + } + + /// A sequential read in tables of at most batch_rows rows. Use from one thread at a time. + class Reader + { + public: + static Result open_file(std::string_view path, const Layout& layout, const ReaderOptions& options = {}) + { + const fw_reader_options raw = options.to_c(); + fw_reader* reader = nullptr; + if (auto ok = detail::check(fw_reader_open_file(detail::bytes(path), static_cast(path.size()), layout.get(), &raw, &reader)); !ok) + { + return std::unexpected(ok.error()); + } + return Reader(reader); + } + + /// Reads data in place: it must outlive the reader. + static Result open_memory(std::string_view data, const Layout& layout, const ReaderOptions& options = {}) + { + const fw_reader_options raw = options.to_c(); + fw_reader* reader = nullptr; + if (auto ok = detail::check(fw_reader_open_memory(detail::bytes(data), static_cast(data.size()), layout.get(), &raw, &reader)); !ok) + { + return std::unexpected(ok.error()); + } + return Reader(reader); + } + + Reader(Reader&& other) noexcept : reader_(std::exchange(other.reader_, nullptr)) {} + Reader& operator=(Reader&& other) noexcept + { + std::swap(reader_, other.reader_); + return *this; + } + Reader(const Reader&) = delete; + Reader& operator=(const Reader&) = delete; + ~Reader() + { + if (reader_ != nullptr) + { + fw_reader_close(reader_); + } + } + + /// The next batch, or an empty optional at the end. + Result> next() + { + fw_table table{}; + const int32_t status = fw_reader_next(reader_, &table); + if (status == FW_EOF) + { + return std::optional
{}; + } + if (auto ok = detail::check(status); !ok) + { + return std::unexpected(ok.error()); + } + return std::optional
(Table(table)); + } + + private: + explicit Reader(fw_reader* reader) : reader_(reader) {} + fw_reader* reader_; + }; + + namespace detail + { + inline fw_write_options write_options(const WriteOptions& options) + { + return { + .struct_size = sizeof(fw_write_options), + .encoding = options.encoding, + .newline = bytes(options.newline), + .newline_len = static_cast(options.newline.size()), + .degree_of_parallelism = options.degree_of_parallelism, + .batch_rows = options.batch_rows, + }; + } + } + + inline Result write_file(std::string_view path, const fw_table& table, const Layout& layout, const WriteOptions& options = {}) + { + const fw_write_options raw = detail::write_options(options); + return detail::check(fw_write_file(detail::bytes(path), static_cast(path.size()), &table, layout.get(), &raw)); + } + + inline Result write_file(std::string_view path, const Table& table, const Layout& layout, const WriteOptions& options = {}) + { + return write_file(path, table.get(), layout, options); + } + + inline Result> write_memory(const fw_table& table, const Layout& layout, const WriteOptions& options = {}) + { + const fw_write_options raw = detail::write_options(options); + fw_buffer buffer{}; + if (auto ok = detail::check(fw_write_memory(&table, layout.get(), &raw, &buffer)); !ok) + { + return std::unexpected(ok.error()); + } + std::vector bytes(buffer.data, buffer.data + buffer.length); + fw_free_buffer(&buffer); + return bytes; + } + + inline Result> write_memory(const Table& table, const Layout& layout, const WriteOptions& options = {}) + { + return write_memory(table.get(), layout, options); + } +} diff --git a/src/FixedWidthParser.Native/include/fixedwidth_arrow.hpp b/src/FixedWidthParser.Native/include/fixedwidth_arrow.hpp new file mode 100644 index 0000000..b1c5a8c --- /dev/null +++ b/src/FixedWidthParser.Native/include/fixedwidth_arrow.hpp @@ -0,0 +1,123 @@ +// Arrow C Data Interface helpers for fixedwidth.hpp: RAII owners of the arrays and streams the library +// publishes. Hand get() to any Arrow consumer, which then takes ownership (set its release to null). +#pragma once + +#include "fixedwidth.hpp" +#include "fixedwidth_arrow.h" + +#include + +namespace fw +{ + /// A struct array and its schema, one child per layout column; released on destruction. + class ArrowResult + { + public: + ArrowResult() noexcept : array_{}, schema_{} {} + ArrowResult(ArrowResult&& other) noexcept : array_(std::exchange(other.array_, {})), schema_(std::exchange(other.schema_, {})) {} + ArrowResult& operator=(ArrowResult&& other) noexcept + { + std::swap(array_, other.array_); + std::swap(schema_, other.schema_); + return *this; + } + ArrowResult(const ArrowResult&) = delete; + ArrowResult& operator=(const ArrowResult&) = delete; + ~ArrowResult() + { + if (array_.release != nullptr) + { + array_.release(&array_); + } + if (schema_.release != nullptr) + { + schema_.release(&schema_); + } + } + + ArrowArray& array() noexcept { return array_; } + ArrowSchema& schema() noexcept { return schema_; } + + private: + ArrowArray array_; + ArrowSchema schema_; + }; + + inline Result read_arrow_file(std::string_view path, const Layout& layout, const ReadOptions& options = {}) + { + ArrowResult result; + const fw_read_options raw = options.to_c(); + if (auto ok = detail::check(fw_read_arrow_file(detail::bytes(path), static_cast(path.size()), layout.get(), &raw, + &result.array(), &result.schema(), nullptr)); + !ok) + { + return std::unexpected(ok.error()); + } + return result; + } + + inline Result read_arrow_memory(std::string_view data, const Layout& layout, const ReadOptions& options = {}) + { + ArrowResult result; + const fw_read_options raw = options.to_c(); + if (auto ok = detail::check(fw_read_arrow_memory(detail::bytes(data), static_cast(data.size()), layout.get(), &raw, + &result.array(), &result.schema(), nullptr)); + !ok) + { + return std::unexpected(ok.error()); + } + return result; + } + + /// An ArrowArrayStream over a sequential read; released on destruction. + class ArrowStream + { + public: + ArrowStream() noexcept : stream_{} {} + ArrowStream(ArrowStream&& other) noexcept : stream_(std::exchange(other.stream_, {})) {} + ArrowStream& operator=(ArrowStream&& other) noexcept + { + std::swap(stream_, other.stream_); + return *this; + } + ArrowStream(const ArrowStream&) = delete; + ArrowStream& operator=(const ArrowStream&) = delete; + ~ArrowStream() + { + if (stream_.release != nullptr) + { + stream_.release(&stream_); + } + } + + ArrowArrayStream* get() noexcept { return &stream_; } + + private: + ArrowArrayStream stream_; + }; + + inline Result read_arrow_stream_file(std::string_view path, const Layout& layout, const ReaderOptions& options = {}) + { + ArrowStream stream; + const fw_reader_options raw = options.to_c(); + if (auto ok = detail::check(fw_read_arrow_stream_file(detail::bytes(path), static_cast(path.size()), layout.get(), &raw, stream.get())); + !ok) + { + return std::unexpected(ok.error()); + } + return stream; + } + + /// Reads data lazily: it must outlive the stream. + inline Result read_arrow_stream_memory(std::string_view data, const Layout& layout, const ReaderOptions& options = {}) + { + ArrowStream stream; + const fw_reader_options raw = options.to_c(); + if (auto ok = detail::check(fw_read_arrow_stream_memory(detail::bytes(data), static_cast(data.size()), layout.get(), &raw, stream.get())); + !ok) + { + return std::unexpected(ok.error()); + } + return stream; + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeApiTests.cs b/tests/FixedWidthParser.Tests/Native/NativeApiTests.cs index 7313238..46381bf 100644 --- a/tests/FixedWidthParser.Tests/Native/NativeApiTests.cs +++ b/tests/FixedWidthParser.Tests/Native/NativeApiTests.cs @@ -72,6 +72,21 @@ public void HandleTable_IdIsNeverReissued() Assert.Null(NativeHandleTable.Resolve(first)); } + [Fact] + public void DefFile_ListsEveryExport() + { + var exports = typeof(NativeApi).Assembly.GetType("FixedWidthParser.Native.Exports")! + .GetMethods(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static) + .Select(m => m.GetCustomAttributes(typeof(System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute), false).Cast().FirstOrDefault()?.EntryPoint) + .OfType() + .Order(StringComparer.Ordinal); + string def = Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "src", "FixedWidthParser.Native", "include", "fixedwidth.def"); + + var listed = File.ReadAllLines(def).Skip(1).Select(l => l.Trim()).Where(l => l.Length > 0).Order(StringComparer.Ordinal); + + Assert.Equal(exports, listed); + } + [Fact] public void HandleTable_ResolvesOnlyItsOwnKind() { From d3cefe1bd9a00ba0fe6b5d513f725b67445cc7a7 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 01:51:05 -0300 Subject: [PATCH 45/85] ci: build and test the native library and its bindings native-bindings.yml publishes the library on Linux, Windows and macOS, then runs the Python tests and a benchmark smoke, the C smoke test and the C++ tests against it. The README, ARCHITECTURE, CHANGELOG and the native README describe the C ABI, the Python package and the C++ wrapper. --- .github/workflows/native-bindings.yml | 127 ++++++++++++++++++ ARCHITECTURE.md | 43 +++++- CHANGELOG.md | 9 ++ README.md | 29 +++- src/FixedWidthParser.Native/README.md | 39 ++++++ .../CMakeLists.txt | 3 +- 6 files changed, 243 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/native-bindings.yml diff --git a/.github/workflows/native-bindings.yml b/.github/workflows/native-bindings.yml new file mode 100644 index 0000000..92f49af --- /dev/null +++ b/.github/workflows/native-bindings.yml @@ -0,0 +1,127 @@ +name: Native bindings + +on: + push: + branches: [master] + pull_request: + branches: [master, develop] + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + +concurrency: + group: native-bindings-${{ github.ref }} + cancel-in-progress: true + +jobs: + changes: + name: Detect changed paths + if: github.event.pull_request.draft != true + runs-on: ubuntu-latest + outputs: + bindings: ${{ steps.filter.outputs.bindings }} + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Filter + id: filter + uses: dorny/paths-filter@v4 + with: + filters: | + bindings: + - 'src/FixedWidthParser/**' + - 'src/FixedWidthParser.Native/**' + - 'python/**' + - 'cpp/**' + - 'tests/FixedWidthParser.NativeSmoke/**' + - '.github/workflows/native-bindings.yml' + + build-native: + name: Build native library (${{ matrix.os }}) + needs: changes + if: needs.changes.outputs.bindings == 'true' + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup .NET + uses: actions/setup-dotnet@v6 + with: + dotnet-version: '10.0.x' + + - name: Publish the native library into the Python package + shell: bash + env: + RunAnalyzers: 'false' + run: python python/scripts/build_native.py + + - name: Upload native library + uses: actions/upload-artifact@v7 + with: + name: native-lib-${{ matrix.os }} + path: python/src/fixedwidth/_lib/* + if-no-files-found: error + + test-bindings: + name: Python + C + C++ (${{ matrix.os }}) + needs: [changes, build-native] + if: needs.changes.outputs.bindings == 'true' + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: '3.12' + cache: pip + cache-dependency-path: python/pyproject.toml + + - name: Download native library + uses: actions/download-artifact@v8 + with: + name: native-lib-${{ matrix.os }} + path: python/src/fixedwidth/_lib + + - name: Restore executable bit (Linux/macOS) + if: runner.os != 'Windows' + shell: bash + run: chmod +x python/src/fixedwidth/_lib/* + + - name: Install the Python package + run: pip install -e "python[dev,pandas,polars]" + + - name: Run the Python tests + run: pytest python/tests -v + + - name: Run the Python benchmark (smoke) + run: python python/benchmarks/bench_read.py --lines 2000 --runs 1 + + # lib.exe, which turns fixedwidth.def into an import library, is only on PATH in a developer prompt. + - name: MSVC developer environment (Windows) + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + + - name: Build and run the C smoke test + run: | + cmake -S tests/FixedWidthParser.NativeSmoke -B build/nativesmoke -DCMAKE_BUILD_TYPE=Release + cmake --build build/nativesmoke --config Release + ctest --test-dir build/nativesmoke --output-on-failure -C Release + + - name: Build and run the C++ tests + run: | + cmake -S cpp -B build/cpp -DCMAKE_BUILD_TYPE=Release + cmake --build build/cpp --config Release + ctest --test-dir build/cpp --output-on-failure -C Release diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 34c80c0..aac6124 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -111,11 +111,13 @@ stream is read once and disposed unless the caller passed `leaveOpen`. ### Parallel aggregation -`ParallelAggregator` (`Readers/ParallelAggregator.cs`) backs `AggregateFileAsync` on both paths. It -opens the file once as a `SafeFileHandle`, cuts it into byte ranges (four per worker, 1 MB to 64 MB -each), and lets workers pull ranges from a shared counter. Each worker reads its range with -`RandomAccess.Read`, through its own `LineBufferState`, into its own accumulator; the caller merges the -accumulators in range order and throws the first failure in file order. +`ParallelAggregator` (`Readers/ParallelAggregator.cs`) backs `AggregateFileAsync` on both paths, and the +native library's reads. It reads an `IRandomAccessSource` (a file handle, or native memory), cuts +`[dataStart, length)` into byte ranges (four per worker, 1 MB to 64 MB each), and lets workers pull ranges +from a shared counter. Each worker reads its range through its own `LineBufferState` into its own +accumulator — or, for the native library, its own `ILineSink` — and the caller merges them in range order +and throws the first failure in file order. The accumulator and sink loops are twins on purpose: folding +the accumulator into a generic sink cost one-call reads about 40% under dynamic PGO. A line belongs to the range its first byte falls in, the byte after the previous line break, so a byte order mark belongs to line 1. A worker starts one byte before its range, skips the line it lands @@ -192,6 +194,37 @@ for an invalid model. Otherwise it writes `TryParse` and `TryFormat` as straight `FixedWidthRuntime` call per column. Each column's settings become two `static readonly` fields, a `ColumnParseOptions` and a `ColumnFormatOptions`, passed to the helpers by `in`. +## The native library + +`src/FixedWidthParser.Native` is published with NativeAOT as a shared library with a C ABI +(`include/fixedwidth.h`), following ExcelReader.Native: `[UnmanagedCallersOnly]` wrappers in `Exports*.cs` +that only check pointers, over a span-based `*Api` layer the tests drive; opaque handles in a +`NativeHandleTable` that never reissues an id; a per-thread last error; every optional input in a +`struct_size`-versioned options struct. It uses the core through `InternalsVisibleTo` and adds nothing to +its public API. + +``` +fw_column_spec[] -> NativeLayout -> ParallelAggregator -> TableSink per partition -> fw_table + (validated, (byte ranges, (a ColumnBuilder per or Arrow + immutable) header skipped) column, native buffers) +``` + +There is no model type: a `NativeLayout` holds each column's `ColumnParseOptions` and +`ColumnFormatOptions`, and a `ColumnBuilder` parses its slice of each line with the same +`FixedWidthRuntime` helpers generated code calls, appending to `GrowableNativeBuffer`s (`int64`, +`double`, `int32` days or string offsets, UTF-8 bytes, a validity bitmap created on the first null). A line +rejected midway truncates the columns it already filled. A partitioned read merges by adopting the +following partition's builders; publishing concatenates each column once. The Arrow export moves those +buffers into `ArrowArray`s without a copy, except DECIMAL, widened to 128 bits. + +Writing formats a caller's `fw_table` through the layout. Every line has the same length, so the output +size and each batch's offset are known up front: batches format in parallel straight into the output +buffer, or into a pooled buffer written at its offset with `RandomAccess.Write`, and the bytes equal a +serial write. + +`python/` binds the C ABI with `ctypes`; `fixedwidth.hpp` wraps it for C++23. The NativeSmoke C program +pins every struct's layout with static asserts. + ## Public surface User-facing types live in the root `FixedWidthParser` namespace. `FixedWidthParser.Infrastructure` diff --git a/CHANGELOG.md b/CHANGELOG.md index fc32e2c..8668a8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,15 @@ Windows-1252 after registering `CodePagesEncodingProvider`). `ColumnLength` and `RawValue`. A short line says how long it is and how long the layout needs it to be. A line that parses pays nothing for this: a failed line is parsed again to find the column. - `FixedWidthReaderOptions.OnParseError`: return `true` to skip a line that fails to parse. +- `FixedWidthParser.Native`: the library published with NativeAOT as a shared library with a C ABI + (`fixedwidth.h`), for languages without a .NET runtime. Layouts are described at run time + (`fw_layout_create`), including implied decimals (`scale`) for vendor formats such as B3's COTAHIST. + Reads fill a columnar `fw_table` or Arrow C Data Interface arrays and streams, in parallel by byte range, + with `header_lines` and `skip_invalid`; writes take a columnar table and are byte-identical whatever the + thread count. Not on NuGet. +- The `fixedwidth` Python package over it (NumPy views, pyarrow, pandas, polars), 52 times faster than + `pandas.read_fwf` on a 1,000,000-line COTAHIST-shaped file, and a header-only C++23 wrapper + (`fixedwidth.hpp`). - `ref struct` models on `Read`, `ReadFile`, `ReadAsync`, `ReadFileAsync`, `AggregateFileAsync` and `AggregateAsync`, generated and through a layout. The enumerators parse a line when `Current` is read, so reading it twice parses twice; with `OnParseError` set, each line is parsed once more to diff --git a/README.md b/README.md index 70a4f46..d9f19d1 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Upgrading from 1.x? See [the 2.0 entry in the changelog](CHANGELOG.md#200). - **Culture-aware** numbers and dates, including `double`/`float` via csFastFloat and any `IUtf8SpanParsable`/`ISpanParsable` type. - **Layout validation** when a layout is built or generated: negative `Start`, non-positive `Length`, overlapping columns and non-ASCII padding fail clearly. - **`ref struct` model support** on single-line parsing and formatting, on every reader and on parallel aggregation. +- **Python and C/C++** through a NativeAOT library with a C ABI: columnar and Arrow reads, parallel reads and writes. See [Python and C/C++](#python-and-cc). ## Requirements @@ -360,13 +361,39 @@ dotnet run -c Release --project tests/Benchmarks/Benchmarks.csproj -- --filter " Benchmark reports are written to `tests/Benchmarks/BenchmarkDotNet.Artifacts/results`. +## Python and C/C++ + +`src/FixedWidthParser.Native` publishes the library as a NativeAOT shared library with a C ABI +(`include/fixedwidth.h`), so Python and C/C++ read and write fixed-width files without a .NET runtime. The +caller describes the columns at run time; a read fills one native buffer per column, or Arrow arrays, in +parallel by byte range. + +```python +import fixedwidth as fw + +layout = fw.Layout([ + fw.Column("codneg", 12, 12, fw.String), + fw.Column("preabe", 56, 13, fw.Float64, scale=2), # B3 N(11)V99: two implied decimals +]) +df = fw.read_pandas("COTAHIST_A2023.TXT", layout, fw.ReadOptions(header_lines=1, skip_invalid=True)) +``` + +On a 1,000,000-line COTAHIST-shaped file, `fw.read_pandas` is 52 times faster than `pandas.read_fwf` +([measurements](docs/python-benchmarks.md)). The [Python README](python/README.md), the +[C++ README](cpp/README.md) and the [native library README](src/FixedWidthParser.Native/README.md) cover +the rest. + ## Project Layout ```text src/FixedWidthParser/ The library src/FixedWidthParser.Generator/ Roslyn source generator -tests/FixedWidthParser.Tests/ Runtime, reader, writer and parity tests +src/FixedWidthParser.Native/ NativeAOT C ABI, its C and C++ headers +python/ The fixedwidth Python package +cpp/ CMake target and tests for the C++ wrapper +tests/FixedWidthParser.Tests/ Runtime, reader, writer, parity and native-layer tests tests/FixedWidthParser.Generator.Tests/Source generator tests +tests/FixedWidthParser.NativeSmoke/ C consumer of the native library tests/Benchmarks/ BenchmarkDotNet benchmarks tests/AotSmoke/ Fluent layout published with NativeAOT docs/ Measurements behind design decisions diff --git a/src/FixedWidthParser.Native/README.md b/src/FixedWidthParser.Native/README.md index b6fafd6..ec73340 100644 --- a/src/FixedWidthParser.Native/README.md +++ b/src/FixedWidthParser.Native/README.md @@ -25,3 +25,42 @@ library needs AVX2 (`x86-64-v3`). Every export catches everything and returns a status (`FW_OK` is 0, failures are negative). Every options struct starts with `struct_size`; `NULL` means all defaults. + +## Exports + +| Area | Functions | +|---|---| +| Errors | `fw_abi_version`, `fw_last_error`, `fw_last_parse_error` | +| Layouts | `fw_layout_create`, `fw_layout_free` | +| Columnar reads | `fw_read_file`, `fw_read_memory`, `fw_free_table` | +| Batches | `fw_reader_open_file`, `fw_reader_open_memory`, `fw_reader_next`, `fw_reader_close` | +| Arrow | `fw_read_arrow_file`, `fw_read_arrow_memory`, `fw_read_arrow_stream_file`, `fw_read_arrow_stream_memory` | +| Writes | `fw_write_file`, `fw_write_memory`, `fw_free_buffer` | + +`include/fixedwidth.def` lists them for import-library generation on Windows; a test fails when it drifts +from the `[UnmanagedCallersOnly]` entry points. + +## Consuming from C + +Check `fw_abi_version()` against `FW_ABI_VERSION` first. NativeAOT ships no import library on Windows: load +the library dynamically (`LoadLibraryA`/`dlopen`, as `tests/FixedWidthParser.NativeSmoke` does) or generate +one from `include/fixedwidth.def` (`cpp/cmake/NativeLib.cmake` does this for CMake). + +```c +fw_column_spec columns[2] = {0}; +columns[0] = (fw_column_spec){ .struct_size = sizeof(fw_column_spec), .name = (const uint8_t*)"codneg", + .name_len = 6, .start = 12, .length = 12, .type = FW_T_STRING }; +columns[1] = (fw_column_spec){ .struct_size = sizeof(fw_column_spec), .name = (const uint8_t*)"preabe", + .name_len = 6, .start = 56, .length = 13, .type = FW_T_FLOAT64, .scale = 2 }; +fw_layout* layout; +fw_layout_create(columns, 2, NULL, &layout); + +fw_read_options options = { .struct_size = sizeof(fw_read_options), .header_lines = 1, .skip_invalid = 1 }; +fw_table table; +if (fw_read_file((const uint8_t*)path, (int32_t)strlen(path), layout, &options, &table, NULL) == FW_OK) { + const double* preabe = table.columns[1].values; + /* ... */ + fw_free_table(&table); +} +fw_layout_free(layout); +``` diff --git a/tests/FixedWidthParser.NativeSmoke/CMakeLists.txt b/tests/FixedWidthParser.NativeSmoke/CMakeLists.txt index 8e7b8de..1a74195 100644 --- a/tests/FixedWidthParser.NativeSmoke/CMakeLists.txt +++ b/tests/FixedWidthParser.NativeSmoke/CMakeLists.txt @@ -26,7 +26,8 @@ add_executable(fw_smoke smoke.c) target_include_directories(fw_smoke PRIVATE "${FIXEDWIDTH_ROOT}/src/FixedWidthParser.Native/include") target_compile_definitions(fw_smoke PRIVATE FIXEDWIDTH_LIB_PATH_DEFAULT="${FIXEDWIDTH_LIB_PATH}") if(MSVC) - target_compile_options(fw_smoke PRIVATE /W4 /WX) + # C4152: the smoke test casts GetProcAddress results to function pointers on purpose. + target_compile_options(fw_smoke PRIVATE /W4 /WX /wd4152) else() target_compile_options(fw_smoke PRIVATE -Wall -Wextra -Werror) endif() From c130f5cb98a8d05994e6116d14f1dd25a4fd0f6f Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 02:08:17 -0300 Subject: [PATCH 46/85] fix(native): valid UTF-8, size limits and UTC timestamps From the final review: - String columns are always valid UTF-8: invalid bytes (another encoding read as UTF-8, a character split by a column boundary) are decoded with U+FFFD as the core does, instead of reaching Arrow utf8. - A table over 2^31-1 rows or a string column over 2 GiB fails with a clear message before anything is allocated, and a failed publish frees what it built; buffer growth and the decimal128 widening no longer overflow 32-bit sizes; batch_rows is capped at int.MaxValue. - TIMESTAMP text with a zone or offset is converted to UTC, and text without one is taken as UTC, independent of the machine. - DECIMAL refuses more than 18 digits; a scaled FLOAT64 too long for an int64 keeps its implied decimals. - Python write refuses a decimal with more digits than the column scale instead of truncating it. --- python/src/fixedwidth/writer.py | 12 ++- python/tests/test_write.py | 21 +++++ src/FixedWidthParser.Native/Arrow/ArrowApi.cs | 10 ++- .../GrowableNativeBuffer.cs | 20 +++-- .../Reading/ColumnBuilder.cs | 54 +++++++++--- .../Reading/NativeTable.cs | 2 +- .../Reading/NativeTableReader.cs | 2 +- .../Reading/ReadApi.cs | 15 ++-- .../Reading/ReaderApi.cs | 4 +- .../Reading/TableLimits.cs | 19 +++++ .../Reading/TableSink.cs | 52 +++++++++--- .../Native/NativeLimitsTests.cs | 85 +++++++++++++++++++ .../Native/NativeReadTests.cs | 54 ++++++++++++ 13 files changed, 311 insertions(+), 39 deletions(-) create mode 100644 src/FixedWidthParser.Native/Reading/TableLimits.cs create mode 100644 tests/FixedWidthParser.Tests/Native/NativeLimitsTests.cs diff --git a/python/src/fixedwidth/writer.py b/python/src/fixedwidth/writer.py index 35b4534..7f6486b 100644 --- a/python/src/fixedwidth/writer.py +++ b/python/src/fixedwidth/writer.py @@ -109,7 +109,7 @@ def _values(kind: ColumnType, spec: Column, array: Any, pa: Any, np: Any) -> Any return array.cast(pa.float64()).fill_null(0).to_numpy(zero_copy_only=False) if kind is ColumnType.DECIMAL: if pa.types.is_decimal(array.type): - return np.array([0 if v is None else int(v.scaleb(spec.scale)) for v in array.to_pylist()], dtype=np.int64) + return np.array([_unscaled(v, spec) for v in array.to_pylist()], dtype=np.int64) floats = array.cast(pa.float64()).fill_null(0).to_numpy(zero_copy_only=False) return np.round(floats * 10**spec.scale).astype(np.int64) if kind is ColumnType.DATE: @@ -121,6 +121,16 @@ def _values(kind: ColumnType, spec: Column, array: Any, pa: Any, np: Any) -> Any return array.cast(pa.timestamp("us")).cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) +def _unscaled(value: Any, spec: Column) -> int: + """A decimal as the integer the column stores; more decimals than its scale would be lost, so they fail.""" + if value is None: + return 0 + scaled = value.scaleb(spec.scale) + if scaled != scaled.to_integral_value(): + raise FixedWidthError(f"value {value} of column '{spec.name}' has more decimals than its scale {spec.scale}") + return int(scaled) + + def _fill_strings(target: _native.Column, array: Any, keep: list[object]) -> None: import numpy diff --git a/python/tests/test_write.py b/python/tests/test_write.py index be260cb..d036196 100644 --- a/python/tests/test_write.py +++ b/python/tests/test_write.py @@ -55,6 +55,27 @@ def test_write_values_from_python_types(): assert fw.write_bytes(frame, columns) == b"20231024-0001250 7\n" +def test_write_decimal_with_more_digits_than_the_scale_raises(): + import decimal + + columns = fw.Layout([fw.Column("v", 0, 8, fw.Decimal, scale=2)]) + exact = pa.table({"v": pa.array([decimal.Decimal("1.23"), decimal.Decimal("-0.5")], type=pa.decimal128(10, 3))}) + lossy = pa.table({"v": pa.array([decimal.Decimal("1.239")], type=pa.decimal128(10, 3))}) + + assert fw.write_bytes(exact, columns) == b" 123\n -50\n" + with pytest.raises(fw.FixedWidthError, match="1.239"): + fw.write_bytes(lossy, columns) + + +def test_read_invalid_utf8_reaches_polars_and_pandas_as_valid_text(): + columns = fw.Layout([fw.Column("name", 0, 6, fw.String), fw.Column("n", 6, 2, fw.Int64)]) + data = "AÇÃO 01\n".encode("latin-1") + + assert fw.read_arrow(data, columns).validate(full=True) is None + assert fw.read_polars(data, columns)["name"].to_list()[0].startswith("A") + assert fw.read_pandas(data, columns)["name"][0].startswith("A") + + def test_write_mismatched_columns_raise(): columns = fw.Layout([fw.Column("a", 0, 2, fw.String), fw.Column("b", 2, 2, fw.Int64)]) diff --git a/src/FixedWidthParser.Native/Arrow/ArrowApi.cs b/src/FixedWidthParser.Native/Arrow/ArrowApi.cs index 3c71ba8..902123c 100644 --- a/src/FixedWidthParser.Native/Arrow/ArrowApi.cs +++ b/src/FixedWidthParser.Native/Arrow/ArrowApi.cs @@ -212,8 +212,8 @@ private static IntPtr OrAllocate(IntPtr buffer, int minimum) private static IntPtr WidenDecimals(NativeColumnRaw* column) { - var values = new ReadOnlySpan(column->Values, (int)column->Length); - var wide = (long*)Marshal.AllocHGlobal((int)Math.Max(1, column->Length) * DecimalBytes); + var values = new ReadOnlySpan(column->Values, checked((int)column->Length)); + var wide = (long*)Marshal.AllocHGlobal((nint)DecimalBufferBytes(column->Length)); for (int i = 0; i < values.Length; i++) { wide[2 * i] = values[i]; @@ -224,6 +224,12 @@ private static IntPtr WidenDecimals(NativeColumnRaw* column) return (IntPtr)wide; } + /// The bytes of a decimal128 buffer for values, in 64-bit arithmetic. + internal static long DecimalBufferBytes(long rows) + { + return checked(Math.Max(1, rows) * DecimalBytes); + } + private static long CountUnset(IntPtr validity, long length) { if (validity == IntPtr.Zero) diff --git a/src/FixedWidthParser.Native/GrowableNativeBuffer.cs b/src/FixedWidthParser.Native/GrowableNativeBuffer.cs index 5d93c58..ee6deee 100644 --- a/src/FixedWidthParser.Native/GrowableNativeBuffer.cs +++ b/src/FixedWidthParser.Native/GrowableNativeBuffer.cs @@ -37,7 +37,7 @@ internal Span Append(int count) { if (count > _capacity - Count) { - Grow(checked(Count + count)); + Grow((long)Count + count); } var span = new Span(_data + Count, count); Count += count; @@ -49,7 +49,7 @@ internal void Add(T value) { if (Count == _capacity) { - Grow(Count + 1); + Grow(Count + 1L); } _data[Count++] = value; } @@ -58,7 +58,7 @@ internal void AddRange(ReadOnlySpan values) { if (values.Length > _capacity - Count) { - Grow(checked(Count + values.Length)); + Grow((long)Count + values.Length); } values.CopyTo(new Span(_data + Count, values.Length)); Count += values.Length; @@ -94,10 +94,20 @@ private void Reset() Count = 0; } + /// Doubles, at least to , clamped to int.MaxValue elements; past that the buffer cannot grow. + internal static int NextCapacity(int capacity, long minimum) + { + if (minimum > int.MaxValue) + { + throw new InvalidOperationException($"A column buffer cannot hold more than {int.MaxValue} elements."); + } + return (int)Math.Min(int.MaxValue, Math.Max(minimum, Math.Max(capacity * 2L, InitialLength))); + } + [MethodImpl(MethodImplOptions.NoInlining)] - private void Grow(int minimum) + private void Grow(long minimum) { - int capacity = Math.Max(minimum, Math.Max(_capacity * 2, InitialLength)); + int capacity = NextCapacity(_capacity, minimum); nint bytes = checked((nint)capacity * sizeof(T)); _block ??= new Block(); _data = (T*)_block.Resize(bytes); diff --git a/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs index 9d05ff6..1c51aa4 100644 --- a/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs +++ b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs @@ -2,6 +2,7 @@ using System.Globalization; using System.Runtime.InteropServices; using System.Text; +using System.Text.Unicode; using FixedWidthParser.Infrastructure; using FixedWidthParser.Native.Layouts; @@ -15,10 +16,13 @@ namespace FixedWidthParser.Native.Reading internal sealed unsafe class ColumnBuilder : IDisposable { private const int StackChars = 256; + private const long DecimalLimit = 1_000_000_000_000_000_000; + private const DateTimeStyles Utc = DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal; private static readonly int EpochDay = new DateOnly(1970, 1, 1).DayNumber; private static readonly double[] Pow10 = [.. Enumerable.Range(0, NativeType.MaxScale + 1).Select(p => Math.Pow(10, p))]; private readonly NativeColumn _column; + private readonly long _maxTextBytes; private readonly GrowableNativeBuffer _longs = new(); private readonly GrowableNativeBuffer _doubles = new(); private readonly GrowableNativeBuffer _ints = new(); @@ -27,9 +31,10 @@ internal sealed unsafe class ColumnBuilder : IDisposable private GrowableNativeBuffer? _validity; #pragma warning restore SS066 - internal ColumnBuilder(NativeColumn column) + internal ColumnBuilder(NativeColumn column, long maxTextBytes) { _column = column; + _maxTextBytes = maxTextBytes; if (column.Type == NativeType.String) { _ints.Add(0); @@ -38,6 +43,9 @@ internal ColumnBuilder(NativeColumn column) internal int Rows { get; private set; } + /// The UTF-8 bytes a STRING column holds. + internal long TextBytes => _bytes.Count; + internal bool TryAppend(ReadOnlySpan raw, in FixedWidthContext context) { var parse = _column.Parse; @@ -228,7 +236,8 @@ private bool Add(bool parsed, long value) private bool AppendFloat(ReadOnlySpan raw, in FixedWidthContext context, in ColumnParseOptions parse) { - if (_column.Scale > 0 && long.TryParse(FixedWidthRuntime.TrimColumn(raw, in parse), NumberStyles.Integer, CultureInfo.InvariantCulture, out long digits)) + var text = FixedWidthRuntime.TrimColumn(raw, in parse); + if (_column.Scale > 0 && long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out long digits)) { _doubles.Add(digits / Pow10[_column.Scale]); return true; @@ -237,7 +246,9 @@ private bool AppendFloat(ReadOnlySpan raw, in FixedWidthContext context, i { return false; } - _doubles.Add(value); + // Digits too many for a long still carry the implied decimals. + bool impliedDecimals = _column.Scale > 0 && text.IndexOfAny(".,eE"u8) < 0; + _doubles.Add(impliedDecimals ? value / Pow10[_column.Scale] : value); return true; } @@ -245,14 +256,14 @@ private static bool TryDecimal(ReadOnlySpan text, int scale, out long unsc { if (long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out unscaled)) { - return true; + return unscaled is > -DecimalLimit and < DecimalLimit; } if (!decimal.TryParse(text, NumberStyles.Number, CultureInfo.InvariantCulture, out decimal value)) { return false; } decimal scaled = value * (decimal)Pow10[scale]; - if (scaled != decimal.Truncate(scaled) || scaled is < long.MinValue or > long.MaxValue) + if (scaled != decimal.Truncate(scaled) || scaled <= -DecimalLimit || scaled >= DecimalLimit) { return false; } @@ -276,16 +287,32 @@ private bool AppendTemporal(ReadOnlySpan raw, in FixedWidthContext context bool timeOk = exact ? FixedWidthRuntime.TryTimeOnlyExact(raw, in context, in parse, out var time) : FixedWidthRuntime.TryParseDecoded(raw, in context, in parse, out time); return Add(timeOk, time.Ticks / TimeSpan.TicksPerMicrosecond); default: - bool stampOk = exact ? FixedWidthRuntime.TryDateTimeExact(raw, in context, in parse, out var stamp) : FixedWidthRuntime.TryParseDecoded(raw, in context, in parse, out stamp); + bool stampOk = TryTimestamp(FixedWidthRuntime.TrimColumn(raw, in parse), context.Encoding, parse.Format, out var stamp); return Add(stampOk, (stamp.Ticks - DateTime.UnixEpoch.Ticks) / TimeSpan.TicksPerMicrosecond); } } + /// A timestamp in UTC: a zone or offset in the text is applied, and text without one is taken as UTC. + private static bool TryTimestamp(ReadOnlySpan text, Encoding encoding, string? format, out DateTime value) + { + value = default; + if (encoding.GetMaxCharCount(text.Length) > StackChars && encoding.GetCharCount(text) > StackChars) + { + return false; + } + Span chars = stackalloc char[StackChars]; + chars = chars[..encoding.GetChars(text, chars)]; + return format is null + ? DateTime.TryParse(chars, CultureInfo.InvariantCulture, Utc, out value) + : DateTime.TryParseExact(chars, format, CultureInfo.InvariantCulture, Utc, out value); + } + private bool AppendString(ReadOnlySpan text, in FixedWidthContext context) { - if (!context.IsUtf8) + // Invalid UTF-8 (a split character, or another encoding read as UTF-8) is decoded with U+FFFD, as the core does. + if (!context.IsUtf8 || !Utf8.IsValid(text)) { - AppendTranscoded(text, context.Encoding); + AppendTranscoded(text, context.IsUtf8 ? Encoding.UTF8 : context.Encoding); _ints.Add(_bytes.Count); return true; } @@ -318,13 +345,18 @@ private void AppendTranscoded(ReadOnlySpan text, Encoding encoding) private void EnsureTextRoom(int bytes) { - if ((long)_bytes.Count + bytes > int.MaxValue) + if (_bytes.Count + (long)bytes > _maxTextBytes) { - throw new InvalidOperationException( - $"Column '{_column.Name}' holds more than 2 GiB of text; read in batches (fw_reader_open_*)."); + throw TooMuchText(_column.Name, _maxTextBytes); } } + internal static InvalidOperationException TooMuchText(string column, long limit) + { + return new InvalidOperationException( + $"Column '{column}' holds more than {limit} bytes of text; read in batches (fw_reader_open_*)."); + } + private void AppendNull() { switch (_column.Type) diff --git a/src/FixedWidthParser.Native/Reading/NativeTable.cs b/src/FixedWidthParser.Native/Reading/NativeTable.cs index c1ebb36..516ed05 100644 --- a/src/FixedWidthParser.Native/Reading/NativeTable.cs +++ b/src/FixedWidthParser.Native/Reading/NativeTable.cs @@ -62,7 +62,7 @@ internal static void Free(NativeTableRaw* table) Marshal.FreeHGlobal((IntPtr)column.Validity); Marshal.FreeHGlobal((IntPtr)column.Data); } - Marshal.FreeHGlobal((IntPtr)table->Columns); + NativeMemory.Free(table->Columns); *table = default; } } diff --git a/src/FixedWidthParser.Native/Reading/NativeTableReader.cs b/src/FixedWidthParser.Native/Reading/NativeTableReader.cs index a6e0c40..78a5d26 100644 --- a/src/FixedWidthParser.Native/Reading/NativeTableReader.cs +++ b/src/FixedWidthParser.Native/Reading/NativeTableReader.cs @@ -51,7 +51,7 @@ internal int Next(NativeTableRaw* table) { return NativeStatus.Eof; } - var sink = new TableSink(_layout); + var sink = new TableSink(_layout, TableLimits.Default); try { #pragma warning disable IDISP003 // Fill appends to the sink; it never replaces it. diff --git a/src/FixedWidthParser.Native/Reading/ReadApi.cs b/src/FixedWidthParser.Native/Reading/ReadApi.cs index a07fd33..85c1b39 100644 --- a/src/FixedWidthParser.Native/Reading/ReadApi.cs +++ b/src/FixedWidthParser.Native/Reading/ReadApi.cs @@ -9,12 +9,13 @@ namespace FixedWidthParser.Native.Reading /// fw_read_file and fw_read_memory: a whole source into one columnar table. internal static unsafe class ReadApi { - internal static int ReadFile(ReadOnlySpan path, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult, int chunkSize = 0) + internal static int ReadFile(ReadOnlySpan path, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult, + int chunkSize = 0, TableLimits? limits = null) { try { using var handle = File.OpenHandle(Encoding.UTF8.GetString(path), FileMode.Open, FileAccess.Read, FileShare.Read); - return Read(new FileSource(handle), layout, options, outTable, outResult, chunkSize); + return Read(new FileSource(handle), layout, options, outTable, outResult, chunkSize, limits ?? TableLimits.Default); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException) { @@ -24,12 +25,14 @@ internal static int ReadFile(ReadOnlySpan path, NativeLayout layout, Nativ } // chunkSize: bytes per partition, 0 sizes them from the source; tests set it to cut every boundary. - internal static int ReadMemory(byte* data, long length, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult, int chunkSize = 0) + internal static int ReadMemory(byte* data, long length, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult, + int chunkSize = 0, TableLimits? limits = null) { - return Read(new PointerSource(data, length), layout, options, outTable, outResult, chunkSize); + return Read(new PointerSource(data, length), layout, options, outTable, outResult, chunkSize, limits ?? TableLimits.Default); } - private static int Read(IRandomAccessSource source, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult, int chunkSize) + private static int Read(IRandomAccessSource source, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult, + int chunkSize, TableLimits limits) { *outTable = default; int status = CheckResult(outResult); @@ -48,7 +51,7 @@ private static int Read(IRandomAccessSource source, NativeLayout layout, NativeR var skipped = new StrongBox(headerLines); var readerOptions = settings.ReaderOptions(skipped, settings.DegreeOfParallelism, chunkSize); #pragma warning disable VSTHRD002 // An export is synchronous and runs with no synchronization context. - using var table = ParallelAggregator.RunAsync(source, dataStart, () => new TableSink(layout), readerOptions, CancellationToken.None) + using var table = ParallelAggregator.RunAsync(source, dataStart, () => new TableSink(layout, limits), readerOptions, CancellationToken.None) .GetAwaiter().GetResult(); #pragma warning restore VSTHRD002 table.Publish(outTable); diff --git a/src/FixedWidthParser.Native/Reading/ReaderApi.cs b/src/FixedWidthParser.Native/Reading/ReaderApi.cs index 8ea05d4..ee29b26 100644 --- a/src/FixedWidthParser.Native/Reading/ReaderApi.cs +++ b/src/FixedWidthParser.Native/Reading/ReaderApi.cs @@ -113,9 +113,9 @@ private static int TrySettings(NativeReaderOptionsRaw* options, out ReadSettings { return status; } - if (raw.BatchRows < 0 || (raw.Read.StructSize != 0 && raw.Read.StructSize != sizeof(NativeReadOptionsRaw))) + if (raw.BatchRows is < 0 or > int.MaxValue || (raw.Read.StructSize != 0 && raw.Read.StructSize != sizeof(NativeReadOptionsRaw))) { - NativeApi.SetLastError("fw_reader_options: batch_rows must not be negative, and read.struct_size must be 0 or sizeof(fw_read_options)."); + NativeApi.SetLastError("fw_reader_options: batch_rows must be 0..2147483647, and read.struct_size must be 0 or sizeof(fw_read_options)."); return NativeStatus.InvalidArgument; } batchRows = raw.BatchRows; diff --git a/src/FixedWidthParser.Native/Reading/TableLimits.cs b/src/FixedWidthParser.Native/Reading/TableLimits.cs new file mode 100644 index 0000000..85eb002 --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/TableLimits.cs @@ -0,0 +1,19 @@ +using System.Runtime.InteropServices; + +namespace FixedWidthParser.Native.Reading +{ + /// + /// What one published table may hold: its row count and each string column's bytes must fit the int32 + /// lengths and offsets of fw_table and Arrow utf8. Tests lower them to reach the limits. + /// + [StructLayout(LayoutKind.Auto)] + internal readonly record struct TableLimits(long MaxTextBytes, long MaxRows) + { + internal static TableLimits Default => new(int.MaxValue, int.MaxValue); + + internal InvalidOperationException TooManyRows() + { + return new InvalidOperationException($"The table has more than {MaxRows} rows; read in batches (fw_reader_open_*)."); + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/TableSink.cs b/src/FixedWidthParser.Native/Reading/TableSink.cs index 18b6b7f..43a42ca 100644 --- a/src/FixedWidthParser.Native/Reading/TableSink.cs +++ b/src/FixedWidthParser.Native/Reading/TableSink.cs @@ -10,13 +10,15 @@ internal struct TableSink : ILineSink, IDisposable private readonly NativeLayout _layout; private readonly ColumnBuilder[] _columns; private readonly int _requiredLength; + private readonly TableLimits _limits; private int _failedColumn; private List? _following; - internal TableSink(NativeLayout layout) + internal TableSink(NativeLayout layout, TableLimits limits) { _layout = layout; - _columns = [.. layout.Columns.Select(c => new ColumnBuilder(c))]; + _limits = limits; + _columns = [.. layout.Columns.Select(c => new ColumnBuilder(c, limits.MaxTextBytes))]; _requiredLength = layout.Columns.Max(c => c.Start + c.Length); _failedColumn = -1; _following = null; @@ -36,6 +38,10 @@ public bool TryAdd(ReadOnlySpan line, in FixedWidthContext context) _failedColumn = -1; return false; } + if (Rows >= _limits.MaxRows) + { + throw _limits.TooManyRows(); + } var columns = _layout.Columns; for (int i = 0; i < _columns.Length; i++) { @@ -74,22 +80,48 @@ public void Merge(TableSink following) internal readonly IEnumerable Chunks => _following is null ? [_columns] : [_columns, .. _following.Select(f => f._columns)]; /// Hands every column's buffers to ; the caller frees them with . + /// Fails before allocating when the merged partitions exceed the limits; frees what it built if anything else fails. internal readonly unsafe void Publish(NativeTableRaw* table) { var chunks = Chunks.ToArray(); - var columns = (NativeColumnRaw*)Marshal.AllocHGlobal(_columns.Length * sizeof(NativeColumnRaw)); - var parts = new ColumnBuilder[chunks.Length]; + CheckLimits(chunks); + var columns = (NativeColumnRaw*)NativeMemory.AllocZeroed((nuint)_columns.Length, (nuint)sizeof(NativeColumnRaw)); + table->Columns = columns; + table->ColumnCount = _columns.Length; + try + { + var parts = new ColumnBuilder[chunks.Length]; + for (int i = 0; i < _columns.Length; i++) + { + for (int c = 0; c < chunks.Length; c++) + { + parts[c] = chunks[c][i]; + } + ColumnBuilder.Concat(parts, columns + i); + } + table->RowCount = TotalRows; + } + catch + { + NativeTable.Free(table); + throw; + } + } + + private readonly void CheckLimits(ColumnBuilder[][] chunks) + { + if (TotalRows > _limits.MaxRows) + { + throw _limits.TooManyRows(); + } for (int i = 0; i < _columns.Length; i++) { - for (int c = 0; c < chunks.Length; c++) + long text = chunks.Sum(chunk => chunk[i].TextBytes); + if (text > _limits.MaxTextBytes) { - parts[c] = chunks[c][i]; + throw ColumnBuilder.TooMuchText(_layout.Columns[i].Name, _limits.MaxTextBytes); } - ColumnBuilder.Concat(parts, columns + i); } - table->ColumnCount = _columns.Length; - table->RowCount = TotalRows; - table->Columns = columns; } public readonly void Dispose() diff --git a/tests/FixedWidthParser.Tests/Native/NativeLimitsTests.cs b/tests/FixedWidthParser.Tests/Native/NativeLimitsTests.cs new file mode 100644 index 0000000..e3c986e --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeLimitsTests.cs @@ -0,0 +1,85 @@ +using System.Text; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Arrow; +using FixedWidthParser.Native.Reading; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; +using static FixedWidthParser.Tests.Native.NativeTestReads; + +namespace FixedWidthParser.Tests.Native +{ + /// The size limits of one table, exercised with small limits instead of gigabytes of input. + public sealed unsafe class NativeLimitsTests + { + private static int ReadWith(byte[] content, TableLimits limits, int degree, int chunkSize, out NativeTableRaw table) + { + var options = Options(degreeOfParallelism: degree); + NativeTableRaw result = default; + int status; + fixed (byte* data = content) + { + status = ReadApi.ReadMemory(data, content.Length, Build(new Col("s", 0, 4, TString)), &options, &result, null, chunkSize, limits); + } + table = result; + return status; + } + + [Fact] + public void PartitionedText_OverTheLimit_FailsBeforePublishing() + { + byte[] content = Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat("abcd\n", 10))); + + int status = ReadWith(content, new TableLimits(MaxTextBytes: 30, MaxRows: int.MaxValue), degree: 4, chunkSize: 5, out var table); + + Assert.Equal(NativeStatus.Error, status); + Assert.Contains("more than 30 bytes of text", LastError(), StringComparison.Ordinal); + Assert.True(table.Columns is null); + } + + [Fact] + public void SequentialText_OverTheLimit_Fails() + { + byte[] content = Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat("abcd\n", 10))); + + Assert.Equal(NativeStatus.Error, ReadWith(content, new TableLimits(MaxTextBytes: 30, MaxRows: int.MaxValue), degree: 1, chunkSize: 0, out _)); + } + + [Theory] + [InlineData(1, 0)] + [InlineData(4, 5)] + public void Rows_OverTheLimit_Fail(int degree, int chunkSize) + { + byte[] content = Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat("abcd\n", 10))); + + int status = ReadWith(content, new TableLimits(MaxTextBytes: int.MaxValue, MaxRows: 4), degree, chunkSize, out var table); + + Assert.Equal(NativeStatus.Error, status); + Assert.Contains("more than 4 rows", LastError(), StringComparison.Ordinal); + Assert.True(table.Columns is null); + } + + [Fact] + public void BufferGrowth_PastHalfOfInt_ClampsInsteadOfOverflowing() + { + Assert.Equal(int.MaxValue, GrowableNativeBuffer.NextCapacity(1 << 30, (1L << 30) + 1)); + Assert.Equal(512, GrowableNativeBuffer.NextCapacity(256, 257)); + Assert.Throws(() => GrowableNativeBuffer.NextCapacity(int.MaxValue, (long)int.MaxValue + 1)); + } + + [Fact] + public void DecimalBuffer_SizeIsComputedIn64Bits() + { + Assert.Equal(4_294_967_312L, ArrowApi.DecimalBufferBytes(268_435_457)); + Assert.Equal(16L, ArrowApi.DecimalBufferBytes(0)); + } + + [Fact] + public void Reader_BatchRowsOverInt_IsInvalidArgument() + { + var options = new NativeReaderOptionsRaw { StructSize = sizeof(NativeReaderOptionsRaw), BatchRows = (long)int.MaxValue + 1 }; + fixed (byte* data = "abcd\n"u8) + { + Assert.Equal(NativeStatus.InvalidArgument, ReaderApi.OpenMemory(data, 5, Build(new Col("s", 0, 4, TString)), &options, out _)); + } + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeReadTests.cs b/tests/FixedWidthParser.Tests/Native/NativeReadTests.cs index c262a8e..4e9f565 100644 --- a/tests/FixedWidthParser.Tests/Native/NativeReadTests.cs +++ b/tests/FixedWidthParser.Tests/Native/NativeReadTests.cs @@ -250,6 +250,60 @@ public void Read_Latin1Text_IsTranscodedToUtf8() } } + [Fact] + public void Read_InvalidUtf8_IsReplacedLikeTheCore() + { + var layout = Build(new Col("name", 0, 6, TString), new Col("n", 6, 2, TInt64)); + byte[] latin1 = Encoding.Latin1.GetBytes("AÇÃO 01\n"); + + var outcome = FromMemory(latin1, layout); + + Assert.Equal(NativeStatus.Ok, outcome.Status); + Assert.Equal([Encoding.UTF8.GetString(latin1.AsSpan(0, 4))], outcome.Column(0)); + } + + [Fact] + public void Read_SplitMultiByteCharacter_StaysValidUtf8() + { + var layout = Build(new Col("a", 0, 2, TString), new Col("b", 2, 3, TString)); + byte[] bytes = Encoding.UTF8.GetBytes("AÇBC\n"); + + var outcome = FromMemory(bytes, layout); + + Assert.Equal(["A�", "�BC"], outcome.Column(0).Concat(outcome.Column(1))); + } + + [Fact] + public void Read_TimestampWithZoneOrOffset_IsUtc() + { + var layout = Build(new Col("t", 0, 25, TTimestamp) { Nullable = true }); + long midnight = (new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc) - DateTime.UnixEpoch).Ticks / 10; + + foreach (var outcome in Ok("2024-01-01T00:00:00Z \n2024-01-01T00:00:00-03:00\n2024-01-01T00:00:00 \n", layout)) + { + Assert.Equal([midnight, midnight + (3L * 3600 * 1_000_000), midnight], outcome.Column(0)); + } + } + + [Fact] + public void Read_DecimalOver18Digits_IsParseError() + { + var layout = Build(new Col("d", 0, 19, TDecimal) { Scale = 2 }); + + Assert.All(Both("1234567890123456789\n", layout), o => Assert.Equal(NativeStatus.ParseError, o.Status)); + } + + [Fact] + public void Read_ScaledFloatBeyondInt64_KeepsTheScale() + { + var layout = Build(new Col("v", 0, 20, TFloat64) { Scale = 2 }); + + foreach (var outcome in Ok("99999999999999999999\n", layout)) + { + Assert.Equal([1e18], outcome.Column(0)); + } + } + [Fact] public void Read_EmptyFile_ReturnsZeroRows() { From 251d7864120cdea2ae5d47a5d8bb271e1fe22f37 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 02:08:55 -0300 Subject: [PATCH 47/85] docs(native): document UTF-8 replacement and UTC timestamps in the header --- src/FixedWidthParser.Native/include/fixedwidth.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/FixedWidthParser.Native/include/fixedwidth.h b/src/FixedWidthParser.Native/include/fixedwidth.h index c7d93ef..bddba5f 100644 --- a/src/FixedWidthParser.Native/include/fixedwidth.h +++ b/src/FixedWidthParser.Native/include/fixedwidth.h @@ -36,13 +36,15 @@ extern "C" { #define FW_PARSE_ERROR -6 /* Column types and their fw_column values. */ -#define FW_T_STRING 0 /* int32 offsets (length + 1) into data, UTF-8 */ +#define FW_T_STRING 0 /* int32 offsets (length + 1) into data, valid UTF-8: bytes invalid in the read + encoding become U+FFFD */ #define FW_T_INT64 1 /* int64 */ #define FW_T_FLOAT64 2 /* double */ #define FW_T_DECIMAL 3 /* int64 unscaled value: 10^-scale per unit; at most 18 digits */ #define FW_T_DATE 4 /* int32 days since 1970-01-01 */ #define FW_T_TIME 5 /* int64 microseconds since midnight */ -#define FW_T_TIMESTAMP 6 /* int64 microseconds since 1970-01-01T00:00:00, no time zone */ +#define FW_T_TIMESTAMP 6 /* int64 microseconds since 1970-01-01T00:00:00Z: a zone or offset in the text is + applied, text without one is taken as UTC */ #define FW_TRIM_BOTH 0 #define FW_TRIM_TRAILING 1 From 2b94e058f46a176fc7189deea7912f783ff0b99a Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 02:59:12 -0300 Subject: [PATCH 48/85] fix: add IDE1006 to WarningsNotAsErrors in Directory.Build.props --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index 4c6b11c..92a410e 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -9,7 +9,7 @@ $(NoWarn);CA1062;CA1008;1591;RS0016;RS0037;RS0026;S1244;SS002;SS008;SS019;MA0016;CA1819;S6964;NU1902;NU1901;IDE0008;IDE0046;IDE0058;CA1515;CA1002 $(NoWarn);CA1848;CA3003;AD0001;S6960;CA1873;MA0049;CA1724;SS017;MA0048 true - HLQ005;HLQ012 + HLQ005;HLQ012;IDE1006 true From b7baca04d7b54501eeaee5ba948106ee39c12eea Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 03:07:06 -0300 Subject: [PATCH 49/85] ci: publish and run the NativeAOT smoke app Ordinary builds never link with NativeAOT, so trimming/AOT regressions in the fluent layout went unchecked. The new job publishes tests/AotSmoke for win-x64 with IlcTreatWarningsAsErrors, failing on any IL2xxx/IL3xxx warning, and runs the binary. --- .github/workflows/ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 149ec48..09ce15d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,6 +91,26 @@ jobs: files: ./TestResults/*.cobertura.xml fail_ci_if_error: false + # Ordinary builds never link with NativeAOT. Publishing tests/AotSmoke does, and turns any trimming/AOT + # warning (IL2xxx/IL3xxx) into a failure; running the binary proves the fluent layout round-trips. + aot-smoke: + name: NativeAOT smoke (win-x64) + runs-on: windows-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup .NET + uses: actions/setup-dotnet@v6 + with: + dotnet-version: '10.0.x' + + - name: Publish AotSmoke (PublishAot) + run: dotnet publish tests/AotSmoke/AotSmoke.csproj --configuration Release --runtime win-x64 -p:IlcTreatWarningsAsErrors=true + + - name: Run the published binary + run: ./tests/AotSmoke/bin/Release/net10.0/win-x64/publish/AotSmoke.exe + # Surfaces vulnerable transitive/direct packages regardless of NoWarn suppressions # in Directory.Build.props (NU190x warnings are silenced there). audit: From 68b4b8b4e3ad70f4f224f732bd6349d9a939542f Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 03:07:06 -0300 Subject: [PATCH 50/85] fix(python): exact, range-checked decimal writes Non-decimal input to a Decimal column went through float64: large integers lost digits and out-of-range values wrapped to int64 min. Integers now scale exactly in int64; NaN, infinity and values outside int64 once scaled raise FixedWidthError before anything is written. Floats keep rounding half to even to the scale, now documented. --- python/README.md | 4 ++++ python/src/fixedwidth/writer.py | 40 ++++++++++++++++++++++++++++----- python/tests/test_write.py | 32 ++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 5 deletions(-) diff --git a/python/README.md b/python/README.md index 3fd5fa9..6c9994c 100644 --- a/python/README.md +++ b/python/README.md @@ -55,6 +55,10 @@ Takes pandas, polars, pyarrow or anything with `__arrow_c_stream__` (needs pyarr `padding="0"` writes zero-padded numbers with the sign leftmost; a null writes as its column filled with `trim_char`. Parallel writes are byte-identical to serial ones. +A `fw.Decimal` column stores its value times `10**scale` as an int64. Decimals and integers are scaled +exactly, and a decimal with more digits than `scale` raises. Floats round half to even to `scale`. +A value that is NaN, infinite or outside int64 once scaled raises before anything is written. + ## Building from source python python/scripts/build_native.py # publishes the library into src/fixedwidth/_lib diff --git a/python/src/fixedwidth/writer.py b/python/src/fixedwidth/writer.py index 7f6486b..87a5640 100644 --- a/python/src/fixedwidth/writer.py +++ b/python/src/fixedwidth/writer.py @@ -108,10 +108,7 @@ def _values(kind: ColumnType, spec: Column, array: Any, pa: Any, np: Any) -> Any if kind is ColumnType.FLOAT64: return array.cast(pa.float64()).fill_null(0).to_numpy(zero_copy_only=False) if kind is ColumnType.DECIMAL: - if pa.types.is_decimal(array.type): - return np.array([_unscaled(v, spec) for v in array.to_pylist()], dtype=np.int64) - floats = array.cast(pa.float64()).fill_null(0).to_numpy(zero_copy_only=False) - return np.round(floats * 10**spec.scale).astype(np.int64) + return _decimals(spec, array, pa, np) if kind is ColumnType.DATE: return array.cast(pa.date32()).cast(pa.int32()).fill_null(0).to_numpy(zero_copy_only=False) if kind is ColumnType.TIME: @@ -121,6 +118,33 @@ def _values(kind: ColumnType, spec: Column, array: Any, pa: Any, np: Any) -> Any return array.cast(pa.timestamp("us")).cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) +def _decimals(spec: Column, array: Any, pa: Any, np: Any) -> Any: + """The unscaled integers a DECIMAL column stores, each of which must fit an int64. + + Decimals and integers scale exactly, and a decimal with more digits than the scale fails. Floats, binary + approximations to begin with, round half to even to the scale; NaN and infinity fail.""" + if pa.types.is_decimal(array.type): + return np.array([_unscaled(v, spec) for v in array.to_pylist()], dtype=np.int64) + factor = 10**spec.scale + if pa.types.is_integer(array.type): + import pyarrow.compute + + bounds = pyarrow.compute.min_max(array).as_py() + for value in (bounds["min"], bounds["max"]): + if value is not None: + _check_int64(value * factor, value, spec) + return array.cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) * factor + floats = array.cast(pa.float64()).fill_null(0).to_numpy(zero_copy_only=False) + with np.errstate(over="ignore", invalid="ignore"): + scaled = np.round(floats * factor) + # Written so NaN fails too; 2**63 is exact in a float64, the int64 range is not. + outside = ~((scaled >= -(2.0**63)) & (scaled < 2.0**63)) + if outside.any(): + value = floats[int(np.argmax(outside))] + raise FixedWidthError(f"value {value} of column '{spec.name}' is not finite or does not fit an int64 at scale {spec.scale}") + return scaled.astype(np.int64) + + def _unscaled(value: Any, spec: Column) -> int: """A decimal as the integer the column stores; more decimals than its scale would be lost, so they fail.""" if value is None: @@ -128,7 +152,13 @@ def _unscaled(value: Any, spec: Column) -> int: scaled = value.scaleb(spec.scale) if scaled != scaled.to_integral_value(): raise FixedWidthError(f"value {value} of column '{spec.name}' has more decimals than its scale {spec.scale}") - return int(scaled) + return _check_int64(int(scaled), value, spec) + + +def _check_int64(unscaled: int, value: Any, spec: Column) -> int: + if not -(2**63) <= unscaled < 2**63: + raise FixedWidthError(f"value {value} of column '{spec.name}' does not fit an int64 at scale {spec.scale}") + return unscaled def _fill_strings(target: _native.Column, array: Any, keep: list[object]) -> None: diff --git a/python/tests/test_write.py b/python/tests/test_write.py index d036196..7b61c2b 100644 --- a/python/tests/test_write.py +++ b/python/tests/test_write.py @@ -67,6 +67,38 @@ def test_write_decimal_with_more_digits_than_the_scale_raises(): fw.write_bytes(lossy, columns) +def test_write_decimal_from_integers_is_exact(): + columns = fw.Layout([fw.Column("v", 0, 20, fw.Decimal, scale=1)]) + data = pa.table({"v": pa.array([9007199254740991, None, -3], type=pa.int64())}) + + assert fw.write_bytes(data, columns) == b" 90071992547409910\n \n -30\n" + + +@pytest.mark.parametrize( + "values", + [ + pa.array([1e30]), + pa.array([float("nan")]), + pa.array([float("-inf")]), + pa.array([2**62], type=pa.int64()), + pa.array([2**64 - 1], type=pa.uint64()), + pa.array([10**17], type=pa.decimal128(38, 0)), + ], +) +def test_write_decimal_out_of_range_or_not_finite_raises(values): + columns = fw.Layout([fw.Column("v", 0, 20, fw.Decimal, scale=2)]) + + with pytest.raises(fw.FixedWidthError, match="column 'v'"): + fw.write_bytes(pa.table({"v": values}), columns) + + +def test_write_decimal_from_floats_rounds_half_to_even_to_the_scale(): + columns = fw.Layout([fw.Column("v", 0, 6, fw.Decimal, scale=2)]) + data = pa.table({"v": pa.array([1.234, 0.125, -2.5, None])}) + + assert fw.write_bytes(data, columns) == b" 123\n 12\n -250\n \n" + + def test_read_invalid_utf8_reaches_polars_and_pandas_as_valid_text(): columns = fw.Layout([fw.Column("name", 0, 6, fw.String), fw.Column("n", 6, 2, fw.Int64)]) data = "AÇÃO 01\n".encode("latin-1") From 3c55352a46b0fd23ab0f8c970488dedafb058bb0 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 03:07:06 -0300 Subject: [PATCH 51/85] fix: merge stream aggregation results incrementally with bounded pending work Workers kept every chunk's accumulator until EOF, so memory grew with the source despite the bounded queue. Results now merge left to right as soon as every earlier chunk has merged, and each chunk holds one of four slots per worker until then, so a slow early chunk stalls the producer instead of letting later results pile up. An OperationCanceledException that is not the caller's cancellation now fails its chunk instead of leaving the producer waiting on its slot. --- ARCHITECTURE.md | 6 +- .../Readers/ParallelStreamAggregator.cs | 139 ++++++++++++------ .../ParallelAggregateTests.cs | 73 +++++++++ 3 files changed, 174 insertions(+), 44 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index aac6124..6fbbfab 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -141,8 +141,10 @@ source -> producer (1 thread) -> bounded queue -> N workers -> ordered merge The producer carries the bytes after the last line break into the next chunk and counts the line breaks it hands over, so each chunk knows the number of its first line and workers report sequential -line numbers directly. The queue holds about two chunks per worker, which bounds memory whatever the -source's size. A line longer than a chunk grows the chunk, up to `MaxLineLength`. A failure in the +line numbers directly. The queue holds about two chunks per worker, and results are merged in chunk +order as soon as every earlier chunk is merged. Each chunk holds one of four slots per worker from the +time it is read until it is merged, so a slow early chunk stalls the producer instead of letting later +results pile up: chunks and unmerged accumulators stay bounded whatever the source's size. A line longer than a chunk grows the chunk, up to `MaxLineLength`. A failure in the producer lies past every chunk it emitted, so a worker's failure is thrown first. With `DegreeOfParallelism = 1` the stream is read by the ordinary async enumerator. diff --git a/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs b/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs index d20b11b..7cf1fe6 100644 --- a/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs +++ b/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs @@ -10,13 +10,15 @@ namespace FixedWidthParser.Readers /// Parallel aggregation over a sequential source. One producer reads the source into pooled chunks, /// cuts each after its last line break, carries the rest into the next chunk, and counts the line /// breaks it hands over, so every chunk knows the number of its first line. Workers fold the chunks - /// from a bounded queue into one accumulator each; the caller merges them in chunk order. A queue - /// about twice the worker count bounds memory to roughly that many chunks, whatever the source's size. + /// from a bounded queue into one accumulator each, merged in chunk order as they complete. Every chunk + /// holds one of a few slots per worker from the moment it is produced until it is merged, which bounds + /// memory, chunks and unmerged accumulators alike, whatever the source's size. /// internal static class ParallelStreamAggregator { private const int DefaultChunkSize = 256 * 1024; private const int LinesPerCancellationCheck = 4096; + private const int SlotsPerWorker = 4; [StructLayout(LayoutKind.Auto)] private readonly record struct Chunk(int Index, byte[] Buffer, int Length, long LinesBefore); @@ -70,20 +72,25 @@ private static async Task PartitionedAsync, new() { int workerCount = options.Workers; + // Two chunks queued and one folding per worker leave one slot per worker for results waiting on an earlier chunk. + using var slots = new SemaphoreSlim(workerCount * SlotsPerWorker); var queue = Channel.CreateBounded(new BoundedChannelOptions(workerCount * 2) { SingleWriter = true }); - var workers = new Task>>[workerCount]; + var merge = new OrderedMerge(slots); + var workers = new Task[workerCount]; foreach (ref var worker in workers.AsSpan()) { - worker = Task.Run(() => ConsumeAsync(parser, queue.Reader, options, ct), ct); +#pragma warning disable IDISP013 // Every worker is awaited below, before the slots are disposed. + worker = Task.Run(() => ConsumeAsync(parser, queue.Reader, merge, options, ct), ct); +#pragma warning restore IDISP013 } ExceptionDispatchInfo? producerFailure = null; try { - await ProduceAsync(source, queue.Writer, options, options.ChunkSize > 0 ? options.ChunkSize : DefaultChunkSize, ct).ConfigureAwait(false); + await ProduceAsync(source, queue.Writer, slots, options, options.ChunkSize > 0 ? options.ChunkSize : DefaultChunkSize, ct).ConfigureAwait(false); } // Cancellation too: the workers must finish before the chunks left in the queue are returned. -#pragma warning disable CA1031 // Rethrown by Merge once the workers have finished. +#pragma warning disable CA1031 // Rethrown by merge.Result once the workers have finished. catch (Exception ex) #pragma warning restore CA1031 { @@ -96,8 +103,8 @@ private static async Task PartitionedAsync o).OrderBy(o => o.Index); - return Merge(outcomes, producerFailure); + await Task.WhenAll(workers).ConfigureAwait(false); + return merge.Result(producerFailure); } finally { @@ -108,33 +115,8 @@ private static async Task PartitionedAsync(IEnumerable> outcomes, ExceptionDispatchInfo? producerFailure) - where TModel : allows ref struct - where TAccumulator : IRecordAccumulator, new() - { - TAccumulator? merged = default; - bool first = true; - foreach (var outcome in outcomes) - { - outcome.Failure?.Throw(); - if (first) - { - merged = outcome.Accumulator; - first = false; - continue; - } - merged!.Merge(outcome.Accumulator!); - } - producerFailure?.Throw(); - if (first) - { - return new TAccumulator(); - } - return merged!; - } - - private static async Task ProduceAsync(Stream source, ChannelWriter queue, FixedWidthReaderOptions options, int chunkSize, CancellationToken ct) + private static async Task ProduceAsync( + Stream source, ChannelWriter queue, SemaphoreSlim slots, FixedWidthReaderOptions options, int chunkSize, CancellationToken ct) { byte[] buffer = ArrayPool.Shared.Rent(chunkSize); int filled = 0; @@ -171,6 +153,7 @@ private static async Task ProduceAsync(Stream source, ChannelWriter queue long lines = buffer.AsSpan(0, cut).Count((byte)'\n'); try { + await slots.WaitAsync(ct).ConfigureAwait(false); await queue.WriteAsync(new Chunk(index++, buffer, cut, linesBefore), ct).ConfigureAwait(false); } catch @@ -206,26 +189,28 @@ private static byte[] GrowUnterminated(byte[] buffer, long linesBefore, int maxL return bigger; } - private static async Task>> ConsumeAsync( - TParser parser, ChannelReader queue, FixedWidthReaderOptions options, CancellationToken ct) + private static async Task ConsumeAsync( + TParser parser, ChannelReader queue, OrderedMerge merge, FixedWidthReaderOptions options, CancellationToken ct) where TModel : allows ref struct where TParser : struct, IRecordLineParser where TAccumulator : IRecordAccumulator, new() { - var outcomes = new List>(); var context = options.Context; await foreach (var chunk in queue.ReadAllAsync(ct).ConfigureAwait(false)) { + Outcome outcome; try { - outcomes.Add(new(chunk.Index, Fold(parser, chunk, options, in context, ct), null)); + outcome = new(chunk.Index, Fold(parser, chunk, options, in context, ct), null); } - catch (Exception ex) when (ex is not OperationCanceledException) + // Anything but the caller's cancellation, which stops everything, becomes this chunk's result: a chunk + // that never completes would hold its slot, and the producer would wait for it forever. + catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested) { - outcomes.Add(new(chunk.Index, default, ExceptionDispatchInfo.Capture(ex))); + outcome = new(chunk.Index, default, ExceptionDispatchInfo.Capture(ex)); } + merge.Complete(outcome); } - return outcomes; } private static TAccumulator Fold( @@ -258,5 +243,75 @@ private static TAccumulator Fold( } return accumulator; } + + /// + /// Merges the workers' accumulators left to right as soon as every earlier chunk has been merged, and + /// frees each chunk's slot then: a slow early chunk stalls the producer instead of letting later + /// results pile up. + /// + private sealed class OrderedMerge(SemaphoreSlim slots) + where TModel : allows ref struct + where TAccumulator : IRecordAccumulator, new() + { + private readonly Lock _gate = new(); + private readonly Dictionary> _pending = []; + private int _next; + private bool _any; + private TAccumulator? _merged; + private ExceptionDispatchInfo? _failure; + + public void Complete(Outcome outcome) + { + int released = 0; + lock (_gate) + { + _pending.Add(outcome.Index, outcome); + while (_pending.Remove(_next, out var ready)) + { + _next++; + released++; + // Past the first failure the results are dropped; only their slots still matter. + _failure ??= ready.Failure; + if (_failure is null) + { + Accept(ready.Accumulator!); + } + } + } + if (released > 0) + { + slots.Release(released); + } + } + + // Every chunk precedes the point where the producer failed, so a worker's failure comes first in the source. + public TAccumulator Result(ExceptionDispatchInfo? producerFailure) + { + _failure?.Throw(); + producerFailure?.Throw(); + return _any ? _merged! : new TAccumulator(); + } + + private void Accept(TAccumulator accumulator) + { + if (!_any) + { + _merged = accumulator; + _any = true; + return; + } + try + { + _merged!.Merge(accumulator); + } + // Thrown by Result instead, so the slots keep flowing and the producer is never left waiting. +#pragma warning disable CA1031 + catch (Exception ex) +#pragma warning restore CA1031 + { + _failure = ExceptionDispatchInfo.Capture(ex); + } + } + } } } diff --git a/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs b/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs index a34c346..5510bf3 100644 --- a/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs +++ b/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs @@ -50,6 +50,54 @@ public void Merge(GenAges following) } } + /// Stalls on the first line and counts the accumulators not yet merged into another. + public sealed class SlowFirstChunk : IRecordAccumulator + { + public static int Unmerged; + public static int MaxUnmerged; + + public SlowFirstChunk() + { + int unmerged = Interlocked.Increment(ref Unmerged); + int max; + while (unmerged > (max = Volatile.Read(ref MaxUnmerged)) && Interlocked.CompareExchange(ref MaxUnmerged, unmerged, max) != max) + { + } + } + + public List Values { get; } = []; + + public void Add(in NameAgeModel model) + { + if (model.Name == "Name1" && model.Age == 1) + { + Thread.Sleep(300); + } + Values.Add(model.Age); + } + + public void Merge(SlowFirstChunk following) + { + Values.AddRange(following.Values); + Interlocked.Decrement(ref Unmerged); + } + } + + public sealed class CancelsItself : IRecordAccumulator + { + public void Add(in NameAgeModel model) + { + if (model.Age == 7) + { + throw new OperationCanceledException(); + } + } + + public void Merge(CancelsItself following) + { + } + } + public static TheoryData SourcesAndChunkSizes() { var data = new TheoryData(); @@ -130,6 +178,31 @@ public async Task EveryPartitionSize_EqualsTheSequentialRead(Source source, int Assert.Equal(expected, generated.Values); } + // Hundreds of chunks finish while the first one stalls; only a few slots' worth may wait for it. + [Fact] + public async Task SlowEarlyChunk_BoundsTheResultsWaitingToMerge() + { + File.WriteAllText(_path, Content(2000)); + var options = new FixedWidthReaderOptions { DegreeOfParallelism = 4, ChunkSize = 64 }; + + var result = await FixedWidth.AggregateAsync( + File.OpenRead(_path), FixedWidthLayout.FromAttributes(), leaveOpen: false, options, TestContext.Current.CancellationToken); + + Assert.Equal(Sequential(), result.Values); + Assert.InRange(SlowFirstChunk.MaxUnmerged, 1, 4 * 4 + 1); + } + + // Not the caller's cancellation: it fails its chunk like any exception instead of stalling the producer. + [Fact] + public async Task AccumulatorThrowingOperationCanceled_FailsTheAggregation() + { + File.WriteAllText(_path, Content(2000)); + var options = new FixedWidthReaderOptions { DegreeOfParallelism = 4, ChunkSize = 64 }; + + await Assert.ThrowsAsync(() => FixedWidth.AggregateAsync( + File.OpenRead(_path), FixedWidthLayout.FromAttributes(), leaveOpen: false, options, TestContext.Current.CancellationToken)); + } + [Theory] [MemberData(nameof(Sources))] public async Task OneWorker_EqualsTheSequentialRead(Source source) From e9a5ad86be20fc716e1143dbbd56713db5e0c9c3 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 03:07:06 -0300 Subject: [PATCH 52/85] fix(generator): include inherited mapped columns The generator read only members declared on the model, so a derived model silently skipped base-class columns when parsing and wrote filler over them. It now walks the base types as reflection does: the first public member of a name wins, so overrides and 'new' members replace what they hide, and an override inherits [FixedColumn] from the property it overrides. Inherited columns take part in overlap validation. --- .../FixedWidthModelGenerator.cs | 36 ++++++++++++++++--- .../FixedWidthModelGeneratorTests.cs | 35 ++++++++++++++++++ .../FixedWidthParser.Tests/GeneratedModels.cs | 21 +++++++++++ .../SourceGenParityTests.cs | 27 ++++++++++++++ 4 files changed, 114 insertions(+), 5 deletions(-) diff --git a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs index 8585ea3..5fd3cfe 100644 --- a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs +++ b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics.CodeAnalysis; using System.Linq; @@ -100,12 +101,23 @@ public void Initialize(IncrementalGeneratorInitializationContext context) return null; } + // Inherited columns too, as reflection's GetProperties/GetFields see them: walking from the + // model to its bases, the first public member of a name wins, so an override or a 'new' + // member replaces the base member it hides. var columns = ImmutableArray.CreateBuilder(); - foreach (var member in symbol.GetMembers()) + var seen = new HashSet(StringComparer.Ordinal); + for (var type = symbol; type is not null; type = type.BaseType) { - if (ExtractColumn(member, columnAttribute, compilation) is { } column) + foreach (var member in type.GetMembers()) { - columns.Add(column); + if (member is not (IPropertySymbol or IFieldSymbol) || member.DeclaredAccessibility != Accessibility.Public || !seen.Add(member.Name)) + { + continue; + } + if (ExtractColumn(member, columnAttribute, compilation) is { } column) + { + columns.Add(column); + } } } @@ -143,8 +155,7 @@ public void Initialize(IncrementalGeneratorInitializationContext context) return null; } - var attribute = member.GetAttributes().FirstOrDefault(a => - SymbolEqualityComparer.Default.Equals(a.AttributeClass, columnAttribute)); + var attribute = ColumnAttribute(member, columnAttribute); if (attribute is null || attribute.ConstructorArguments.Length < 2) { return null; @@ -205,6 +216,21 @@ public void Initialize(IncrementalGeneratorInitializationContext context) isNullable, writeKind, alignment, padding, format, resolvedOverflow, trimChar, trimMode); } + // [FixedColumn] is inherited: an override without one takes it from the property it overrides, as reflection does. + private static AttributeData? ColumnAttribute(ISymbol member, INamedTypeSymbol columnAttribute) + { + for (ISymbol? current = member; current is not null; current = (current as IPropertySymbol)?.OverriddenProperty) + { + var attribute = current.GetAttributes().FirstOrDefault(a => + SymbolEqualityComparer.Default.Equals(a.AttributeClass, columnAttribute)); + if (attribute is not null) + { + return attribute; + } + } + return null; + } + // Implements IFixedWidthModel? private static bool ImplementsMarker(INamedTypeSymbol symbol, INamedTypeSymbol marker) { diff --git a/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs b/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs index 056ce1e..6ceb25b 100644 --- a/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs +++ b/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs @@ -190,6 +190,30 @@ public partial class IgnoringModel : IFixedWidthModel Assert.DoesNotContain("Ignored2", generated, StringComparison.Ordinal); } + [Fact] + public void Model_IncludesInheritedColumns() + { + var result = Run(""" + public class BaseModel + { + [FixedColumn(0, 5)] public int Id { get; set; } + [FixedColumn(5, 5)] public string Code; + } + + public partial class DerivedModel : BaseModel, IFixedWidthModel + { + [FixedColumn(10, 5)] public string Name { get; set; } + } + """); + + Assert.Empty(result.GeneratorDiagnostics); + Assert.Empty(result.CompilationErrors); + var generated = Assert.Single(result.GeneratedSources); + Assert.Contains("Id = __v", generated, StringComparison.Ordinal); + Assert.Contains("Code = __v", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.FormatString(model.Code,", generated, StringComparison.Ordinal); + } + [Theory] [InlineData(""" public readonly record struct BadModel : IFixedWidthModel @@ -231,6 +255,17 @@ public partial class Outer [FixedColumn(2, 5)] public string Right { get; init; } } """, "FWP006")] + [InlineData(""" + public class BadBase + { + [FixedColumn(0, 5)] public string Left { get; set; } + } + + public partial class BadModel : BadBase, IFixedWidthModel + { + [FixedColumn(2, 5)] public string Right { get; set; } + } + """, "FWP006")] public void InvalidModel_ReportsExpectedDiagnostic(string modelSource, string diagnosticId) { var result = Run(modelSource); diff --git a/tests/FixedWidthParser.Tests/GeneratedModels.cs b/tests/FixedWidthParser.Tests/GeneratedModels.cs index 6d17eaf..1c94964 100644 --- a/tests/FixedWidthParser.Tests/GeneratedModels.cs +++ b/tests/FixedWidthParser.Tests/GeneratedModels.cs @@ -172,4 +172,25 @@ public partial struct GenProductModel : IFixedWidthModel { [FixedColumn(0, 70_000)] public string Name { get; init; } } + + /// Base class of : a mapped property, a field, a virtual and a hidden column. + public class GenBaseModel + { + [FixedColumn(0, 4)] public int Id { get; set; } + [FixedColumn(4, 3)] public string Code = ""; + [FixedColumn(7, 5)] public virtual string Tag { get; set; } = ""; + [FixedColumn(12, 5)] public string Label { get; set; } = ""; + } + + /// Generated model whose columns are partly inherited; its own attribute layout is the reflection counterpart. + public sealed partial class GenDerivedModel : GenBaseModel, IFixedWidthModel + { + [FixedColumn(17, 6)] public string Name { get; set; } = ""; + + // Inherits [FixedColumn(7, 5)] from the property it overrides. + public override string Tag { get; set; } = ""; + + // Hides the base column, so only this narrower one is mapped. + [FixedColumn(12, 2)] public new string Label { get; set; } = ""; + } } diff --git a/tests/FixedWidthParser.Tests/SourceGenParityTests.cs b/tests/FixedWidthParser.Tests/SourceGenParityTests.cs index f86b4b9..3b942bd 100644 --- a/tests/FixedWidthParser.Tests/SourceGenParityTests.cs +++ b/tests/FixedWidthParser.Tests/SourceGenParityTests.cs @@ -144,5 +144,32 @@ public void StringPool_InternsAndMatchesReflection() Assert.Same(r.Name, g.Name); Assert.Same(pool.GetOrAdd("John Doe".AsSpan()), g.Name); } + + [Theory] + [InlineData("0042ABCtag01XYzzzJohn ")] + [InlineData(" 7 Z Q Ann ")] + [InlineData("xxxxABCtag01XYzzzJohn ")] + public void InheritedColumns_GeneratedMatchesReflection(string line) + { + bool okR = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var r); + bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); + + Assert.Equal(okR, okG); + if (okR) + { + Assert.Equal((r.Id, r.Code, r.Tag, r.Label, r.Name), (g.Id, g.Code, g.Tag, g.Label, g.Name)); + } + } + + [Fact] + public void InheritedColumns_ParseAndFormatThroughGeneratedModel() + { + Assert.True(FixedWidth.TryParse("0042ABCtag01XYzzzJohn "u8, Ctx, out var g)); + Assert.Equal((42, "ABC", "tag01", "XY", "John"), (g.Id, g.Code, g.Tag, g.Label, g.Name)); + + var model = new GenDerivedModel { Id = 42, Code = "ABC", Tag = "tag01", Label = "XY", Name = "John" }; + Assert.Equal(WriteOne(FixedWidthLayout.FromAttributes(), model), Format(model)); + Assert.Equal("42 ABCtag01XY John ", Format(model)); + } } } From 26e56727f225bc6b4a3a55e67af980d5f17fe5f5 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 03:07:06 -0300 Subject: [PATCH 53/85] fix: return the rented buffer when strict decoding throws Decode rented a char array for wide columns before Encoding.GetChars, and its callers' try/finally began only after it returned, so invalid bytes under a strict encoding lost the array to the pool. --- .../Infrastructure/FixedWidthRuntime.cs | 13 ++++++++++- tests/FixedWidthParser.Tests/CoverageTests.cs | 23 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs index 23b9c3d..9f61486 100644 --- a/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs +++ b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs @@ -436,7 +436,18 @@ private static ReadOnlySpan Decode(ReadOnlySpan bytes, Encoding enco rented = ArrayPool.Shared.Rent(max); chars = rented; } - int written = encoding.GetChars(bytes, chars); + int written; + try + { + written = encoding.GetChars(bytes, chars); + } + catch + { + // A strict decoder throws on invalid bytes, before the caller owns the rental. + Return(rented); + rented = null; + throw; + } return chars[..written]; } diff --git a/tests/FixedWidthParser.Tests/CoverageTests.cs b/tests/FixedWidthParser.Tests/CoverageTests.cs index a95ca9e..4a1b96a 100644 --- a/tests/FixedWidthParser.Tests/CoverageTests.cs +++ b/tests/FixedWidthParser.Tests/CoverageTests.cs @@ -1,3 +1,4 @@ +using System.Buffers; using System.Globalization; using System.Text; using FixedWidthParser.Infrastructure; @@ -36,6 +37,28 @@ public void TimeExact_InvalidField_ReturnsFalse() Assert.False(FixedWidth.TryParse(bad, Ctx, out _)); } + [Fact] + public void StrictDecoding_InvalidWideColumn_ReturnsTheRentedBuffer() + { + // Wider than the 256-char stack buffer, so decoding rents; the trailing 0xFF makes the strict decoder throw. + var strict = new UTF8Encoding(false, true); + byte[] column = new byte[300]; + column.AsSpan().Fill((byte)'1'); + column[^1] = 0xFF; + var options = new ColumnParseOptions(' ', TrimMode.Trailing, "yyyyMMdd"); + int size = strict.GetMaxCharCount(column.Length); + + // The shared pool hands a thread back the array it last returned, so a lost rental shows as a different array. + char[] cached = ArrayPool.Shared.Rent(size); + ArrayPool.Shared.Return(cached); + + Assert.Throws(() => FixedWidthRuntime.TryDateOnlyExact(column, CtxWith(encoding: strict), in options, out _)); + + char[] next = ArrayPool.Shared.Rent(size); + ArrayPool.Shared.Return(next); + Assert.Same(cached, next); + } + [Fact] public void WideNumericField_CommaCulture_DecodesThroughArrayPool() { From 3cf71300750bef717761ce9c3a8fd62edba65065 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 03:07:06 -0300 Subject: [PATCH 54/85] fix(cpp): import library for the target architecture lib.exe was always given /machine:x64, so ARM64 builds got an x64 import library. The machine now comes from the target compiler's architecture ID, which holds when cross-compiling. The MSVC tests also define _CRT_SECURE_NO_WARNINGS: /WX turned the std::fopen deprecation warning into an error. --- cpp/cmake/NativeLib.cmake | 8 +++++++- cpp/tests/CMakeLists.txt | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cpp/cmake/NativeLib.cmake b/cpp/cmake/NativeLib.cmake index fd2f3cc..76f8008 100644 --- a/cpp/cmake/NativeLib.cmake +++ b/cpp/cmake/NativeLib.cmake @@ -13,8 +13,14 @@ function(fixedwidth_import_native_lib library def_file) file(READ "${def_file}" _exports) file(WRITE "${_generated_def}" "LIBRARY ${_dll_name}\n${_exports}") if(MSVC) + # The target's architecture, as the compiler reports it (x64, ARM64, X86), not the host's: + # an x64 machine building for ARM64 (-A ARM64, or an ARM64 developer prompt) needs an ARM64 import library. + set(_machine "${CMAKE_CXX_COMPILER_ARCHITECTURE_ID}") + if(NOT _machine) + message(FATAL_ERROR "cannot tell the MSVC target architecture for the import library") + endif() find_program(_lib_exe NAMES lib REQUIRED) - execute_process(COMMAND "${_lib_exe}" "/def:${_generated_def}" "/out:${_implib}" "/machine:x64" + execute_process(COMMAND "${_lib_exe}" "/def:${_generated_def}" "/out:${_implib}" "/machine:${_machine}" RESULT_VARIABLE _result) else() find_program(_dlltool NAMES dlltool REQUIRED) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 8ba89a5..e3d0693 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -3,6 +3,8 @@ foreach(test read write arrow) target_link_libraries(fw_cpp_${test} PRIVATE fw::fixedwidth) if(MSVC) target_compile_options(fw_cpp_${test} PRIVATE /W4 /WX) + # The tests use standard std::fopen, which the CRT deprecates (C4996) in favour of fopen_s. + target_compile_definitions(fw_cpp_${test} PRIVATE _CRT_SECURE_NO_WARNINGS) else() # Designated initializers that leave defaulted options out are the intended use. target_compile_options(fw_cpp_${test} PRIVATE -Wall -Wextra -Werror -Wno-missing-field-initializers) From 1236724a59f05cb61127b1d4e2a820fbdf7657dc Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 03:08:25 -0300 Subject: [PATCH 55/85] refactor(ci): clean up comments and improve workflow clarity --- .github/workflows/ci.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09ce15d..c3b075d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,14 +1,11 @@ name: CI -# Roda em PRs direcionados à master (o status check que bloqueia o merge) e em -# pushes na master (mantém o branch padrão verde e registra o check no histórico). on: pull_request: branches: [ master, develop ] push: branches: [ master ] -# Cancela runs antigas do mesmo PR/branch quando um novo push chega. concurrency: group: ci-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -34,9 +31,6 @@ jobs: with: dotnet-version: '10.0.x' - # Cache restored NuGet packages keyed on the project/props files. - # (setup-dotnet's built-in cache needs a packages.lock.json, which this repo - # does not use, so we cache the global packages folder directly.) - name: Cache NuGet packages uses: actions/cache@v6 with: @@ -48,16 +42,9 @@ jobs: - name: Restore run: dotnet restore FixedWidthParser.slnx - # DeterministicSourcePaths is disabled here so the Microsoft code-coverage - # collector can map IL back to source. With it on (it is enabled on CI via - # ContinuousIntegrationBuild in FixedWidthParser.csproj), source paths get - # remapped and coverage for the library comes out empty. This build is only - # used for test/coverage; the pack/publish pipeline lives in release.yml. - name: Build (Release) run: dotnet build FixedWidthParser.slnx --configuration Release --no-restore -p:DeterministicSourcePaths=false - # Packing runs package validation, which fails on any binary break against the last release. - # No --no-build: the generator's analyzer ProjectReference trips NETSDK1085 otherwise (see release.yml). - name: Validate package against the last release if: matrix.os == 'ubuntu-latest' run: dotnet pack src/FixedWidthParser/FixedWidthParser.csproj --configuration Release --no-restore --output ./package-validation @@ -82,7 +69,6 @@ jobs: path: ./TestResults/**/*.trx if-no-files-found: ignore - # Coverage is uploaded once, from the Linux leg only, to avoid duplicate reports. - name: Upload coverage to Codecov if: matrix.os == 'ubuntu-latest' uses: codecov/codecov-action@v7 @@ -91,8 +77,6 @@ jobs: files: ./TestResults/*.cobertura.xml fail_ci_if_error: false - # Ordinary builds never link with NativeAOT. Publishing tests/AotSmoke does, and turns any trimming/AOT - # warning (IL2xxx/IL3xxx) into a failure; running the binary proves the fluent layout round-trips. aot-smoke: name: NativeAOT smoke (win-x64) runs-on: windows-latest @@ -111,8 +95,6 @@ jobs: - name: Run the published binary run: ./tests/AotSmoke/bin/Release/net10.0/win-x64/publish/AotSmoke.exe - # Surfaces vulnerable transitive/direct packages regardless of NoWarn suppressions - # in Directory.Build.props (NU190x warnings are silenced there). audit: name: Dependency audit runs-on: ubuntu-latest From 6ec4caa6c268afe455457bbd198af71e663e0b95 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 03:24:40 -0300 Subject: [PATCH 56/85] perf: parse doubles with csFastFloat in comma cultures csFastFloat ran only when the decimal separator was '.'; every other culture decoded the column and used the BCL parser, ~40 ns more per value. It now runs with any one-ASCII-character separator when the negative sign is '-', reusing FixedWidthContext.DecimalSeparator from the writer instead of a separate cache, and still falls back unless every byte was consumed. A pt-BR read goes from 98.4 to 58.5 ns/line. --- src/FixedWidthParser/FixedWidthContext.cs | 2 +- .../Processors/CultureHelpers.cs | 59 +++++-------------- tests/FixedWidthParser.Tests/CultureTests.cs | 24 ++++++++ 3 files changed, 39 insertions(+), 46 deletions(-) diff --git a/src/FixedWidthParser/FixedWidthContext.cs b/src/FixedWidthParser/FixedWidthContext.cs index 0d8b1fd..4bf6d9f 100644 --- a/src/FixedWidthParser/FixedWidthContext.cs +++ b/src/FixedWidthParser/FixedWidthContext.cs @@ -32,7 +32,7 @@ public IFormatProvider FormatProvider /// /// The provider's decimal separator as one ASCII byte when its negative sign is "-", so a plain - /// number can be written without going through the provider; 0 otherwise. + /// number can be written or parsed without going through the provider; 0 otherwise. /// // ponytail: read once, so a NumberFormatInfo mutated after this context was built is not seen. internal byte DecimalSeparator { get => _formatProvider is null ? (byte)'.' : field; private init; } diff --git a/src/FixedWidthParser/Processors/CultureHelpers.cs b/src/FixedWidthParser/Processors/CultureHelpers.cs index 7dde005..0db245b 100644 --- a/src/FixedWidthParser/Processors/CultureHelpers.cs +++ b/src/FixedWidthParser/Processors/CultureHelpers.cs @@ -10,46 +10,15 @@ namespace FixedWidthParser.Processors { /// - /// Culture-aware double/float parsing. csFastFloat is only trusted for the '.' decimal - /// separator, and even then only after checking every byte of the (trimmed) field was consumed — - /// csFastFloat does not fail on trailing content it doesn't recognize, it silently stops and returns - /// the leading prefix it did parse. Any other separator, and a dot-culture value using thousands - /// separators, falls back to the -aware BCL parser over the decoded text. + /// Culture-aware double/float parsing. csFastFloat is trusted only when the provider's + /// decimal separator is one ASCII character and its negative sign is "-" + /// (), and even then only after checking every byte of the + /// (trimmed) field was consumed: csFastFloat does not fail on trailing content it doesn't recognize, it + /// silently stops and returns the leading prefix it did parse. Any other provider, and a value using + /// thousands separators, falls back to the -aware BCL parser over the decoded text. /// internal static class CultureHelpers { - // Decimal separator cache keyed by IFormatProvider identity. A ConditionalWeakTable ties each - // entry's lifetime to its provider's own lifetime: no manual eviction, no unbounded growth if a - // caller passes many distinct (e.g. per-request) providers, and — unlike a single-entry memo — - // no thrash when two providers are used alternately in the same process. - private static readonly ConditionalWeakTable> _separatorCache = new(); - - private sealed class CacheEntry(IFormatProvider provider, char separator) - { - public readonly IFormatProvider Provider = provider; - public readonly char Separator = separator; - } - private static CacheEntry? _cache; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static char GetDecimalSeparator(IFormatProvider formatProvider) - { - var cache = _cache; - if (cache is not null && ReferenceEquals(formatProvider, cache.Provider)) - { - return cache.Separator; - } - char separator = _separatorCache.GetValue(formatProvider, static fp => new StrongBox(Resolve(fp))).Value; - _cache = new CacheEntry(formatProvider, separator); - return separator; - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private static char Resolve(IFormatProvider formatProvider) - { - return NumberFormatInfo.GetInstance(formatProvider).NumberDecimalSeparator[0]; - } - /// Parses a (not yet trimmed) column as , honoring the context's provider. public static bool TryParseDouble(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out double value) { @@ -57,28 +26,28 @@ public static bool TryParseDouble(ReadOnlySpan column, in FixedWidthContex // skipped leading whitespace, so a right-aligned column would otherwise fail the // full-consumption check below. var trimmed = FixedWidthRuntime.TrimColumn(column, in options).TrimStart((byte)' '); - var formatProvider = context.FormatProvider; - if (GetDecimalSeparator(formatProvider) == '.' - && FastDoubleParser.TryParseDouble(trimmed, out int consumed, out value, decimal_separator: (byte)'.') + byte separator = context.DecimalSeparator; + if (separator != 0 + && FastDoubleParser.TryParseDouble(trimmed, out int consumed, out value, decimal_separator: separator) && consumed == trimmed.Length) { return true; } - return TryParseDecoded(trimmed, formatProvider, context.Encoding, out value); + return TryParseDecoded(trimmed, context.FormatProvider, context.Encoding, out value); } /// Parses a (not yet trimmed) column as , honoring the context's provider. public static bool TryParseFloat(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out float value) { var trimmed = FixedWidthRuntime.TrimColumn(column, in options).TrimStart((byte)' '); - var formatProvider = context.FormatProvider; - if (GetDecimalSeparator(formatProvider) == '.' - && FastFloatParser.TryParseFloat(trimmed, out int consumed, out value, decimal_separator: (byte)'.') + byte separator = context.DecimalSeparator; + if (separator != 0 + && FastFloatParser.TryParseFloat(trimmed, out int consumed, out value, decimal_separator: separator) && consumed == trimmed.Length) { return true; } - return TryParseDecoded(trimmed, formatProvider, context.Encoding, out value); + return TryParseDecoded(trimmed, context.FormatProvider, context.Encoding, out value); } [SkipLocalsInit] diff --git a/tests/FixedWidthParser.Tests/CultureTests.cs b/tests/FixedWidthParser.Tests/CultureTests.cs index 82d9a31..4f0acf0 100644 --- a/tests/FixedWidthParser.Tests/CultureTests.cs +++ b/tests/FixedWidthParser.Tests/CultureTests.cs @@ -64,6 +64,30 @@ public void Parse_DoubleColumn_DeDeThousandsSeparator_ParsesFullValue() Assert.Equal(1234.50, model.Salary, 2); } + // The fast path takes any one-ASCII-character separator, so every accepted value must match the BCL's. + [Theory] + [InlineData("de-DE")] + [InlineData("pt-BR")] + [InlineData("fr-FR")] + public void Parse_Double_CommaCulture_MatchesTheBcl(string cultureName) + { + var culture = CultureInfo.GetCultureInfo(cultureName); + var context = CtxWith(culture); + var options = new Infrastructure.ColumnParseOptions(' ', TrimMode.Both, format: null); + string[] edges = ["-0,5", "+1,5", "1,5e3", "1,5E-3", "1.234,5", "0,1", ",5", "5,", "007,10", "1,7976931348623157E+308", "4,9E-324", "1,2,3", "--1", "1e", "1,5x", "1 000,5"]; + var random = new Random(42); + var values = edges.Concat(Enumerable.Range(0, 2000).Select(_ => (random.NextDouble() * Math.Pow(10, random.Next(-8, 12)) * (random.Next(2) * 2 - 1)).ToString("R", culture))); + + foreach (string text in values) + { + bool expected = double.TryParse(text, NumberStyles.Float | NumberStyles.AllowThousands, culture, out double bcl); + bool actual = Infrastructure.FixedWidthRuntime.TryDouble(Bytes(text), in context, in options, out double parsed); + + Assert.True(expected == actual, $"'{text}': BCL {expected}, parser {actual}"); + Assert.True(!expected || bcl.Equals(parsed), $"'{text}': BCL {bcl:R}, parser {parsed:R}"); + } + } + [Fact] public void Parse_DoubleColumn_TrailingGarbageAfterNumber_Fails() { From 6b40b406492529968e88ef410e2ecb565df94244 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 03:24:40 -0300 Subject: [PATCH 57/85] perf: cache UTF-8 columns in front of StringPool A StringPool lookup cost several times a fresh GetString, so a pooled read ran at 137.8 ns/line against 57.1 without. A per-thread, 4-way set-associative cache, as in ExcelReader's Utf8StringCache, now answers repeated UTF-8 values; every hit is checked against the bytes and a miss still goes to the pool. Pooled reads run at ~70 ns/line. docs/reader-performance.md records both changes and why larger reads in the file engine were not kept. --- docs/README.md | 2 + docs/parallel-aggregate.md | 3 +- docs/reader-performance.md | 50 +++++++++ .../Infrastructure/FixedWidthRuntime.cs | 2 +- .../Infrastructure/Utf8StringCache.cs | 100 ++++++++++++++++++ tests/FixedWidthParser.Tests/ParserTests.cs | 35 ++++++ 6 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 docs/reader-performance.md create mode 100644 src/FixedWidthParser/Infrastructure/Utf8StringCache.cs diff --git a/docs/README.md b/docs/README.md index 1e074e5..09448c4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,6 +6,8 @@ a constraint; the numbers that justified it live here, so they can be re-run and - [writer-performance.md](writer-performance.md) — where a written line spends its time, and the formatting shortcuts that came out of it. +- [reader-performance.md](reader-performance.md) — `double` parsing in comma cultures, the cache in + front of `StringPool`, and the read size the file engine kept. - [parallel-aggregate.md](parallel-aggregate.md) — how `AggregateFileAsync` and `AggregateAsync` scale with the number of workers. - [pipe-reader.md](pipe-reader.md) — why a `PipeReader` is read through `AsStream()` rather than diff --git a/docs/parallel-aggregate.md b/docs/parallel-aggregate.md index 730b218..4840bf8 100644 --- a/docs/parallel-aggregate.md +++ b/docs/parallel-aggregate.md @@ -41,7 +41,8 @@ the pipe's own buffer size, not the engine, sets the ceiling. The stream engine beats the file engine from `dop 2` up on this cached file. The file engine's ranges are 3.25 MB at `dop 8` against the stream engine's 256 KB chunks, and each file worker refills a 64 KB -buffer with one `RandomAccess.Read` call at a time. Which of the two costs more is unmeasured. +buffer with one `RandomAccess.Read` call at a time. Measured warm, the read size is not it, and the +gap is nearer 10 % than these figures suggest ([reader-performance.md](reader-performance.md)). ## Not measured diff --git a/docs/reader-performance.md b/docs/reader-performance.md new file mode 100644 index 0000000..5877878 --- /dev/null +++ b/docs/reader-performance.md @@ -0,0 +1,50 @@ +# Reader performance + +## Machine and method + +AMD Ryzen 7 5700X (8C/16T), Windows 10 Pro 19045, .NET SDK 11.0.100-rc.2, `net10.0`, Release. A +Stopwatch console app reads 2,000,000 lines of a generated model (a 10-byte string, a 5-byte `int`, a +10-byte `double`) with `ReadFile`, summing every column. Each figure is the best of five runs after one +warm-up run, file in the OS cache. Before and after were measured in the same position of the same +program: the first configuration of each kind a run measures comes out slower while the JIT is still +tiering up, and comparing across positions misled the file-engine experiment below. + +## `double` columns in a comma culture + +csFastFloat parsed only when the decimal separator was `.`; any other culture decoded the column and +ran the BCL parser. It now runs with any one-ASCII-character separator when the negative sign is `-` +(`FixedWidthContext.DecimalSeparator`, which the writer already used), still falling back unless every +byte was consumed, so thousands separators keep going through the BCL. `CultureTests` checks the +fast path against `double.TryParse` in de-DE, pt-BR and fr-FR. + +| Read | Before | After | +|---|--:|--:| +| `.` separator, invariant | 57.1 ns/line | 58.5 | +| `,` separator, pt-BR | 98.4 ns/line | 58.5 | + +One value: 42.3 ns decoded and BCL-parsed, 7.7 ns through csFastFloat. + +## `StringPool` + +CommunityToolkit's `StringPool.GetOrAdd(bytes, encoding)` costs 87 ns per 6-byte value against 19 ns +for `Encoding.UTF8.GetString` (76 ns even decoding on the stack first): hashing, a lock and its +priority map. A per-thread, 4-way set-associative cache of UTF-8 columns now sits in front of it, as +ExcelReader's `Utf8StringCache` does; a miss still goes to the pool. + +| Read | No pool | Pool, before | Pool, after | +|---|--:|--:|--:| +| 997 distinct names | 57.1 ns/line | 137.8 | 69.8 | +| 50 distinct names | 58.6 ns/line | — | 69.7 | + +A 256-set cache (ExcelReader's size) left the 997-name column at 119.5 ns/line: four ways overflowed +in many sets and the cyclic order missed every time. 1,024 sets (48 KB per thread that reads with a +pool) holds it. A pool still costs ~11 ns/line over allocating: short-lived strings are cheap, and +the pool exists to save memory, not time. + +## Not done: larger reads in `AggregateFileAsync` + +The file engine trailed the stream engine by 29–35 % in the +[parallel-aggregate](parallel-aggregate.md) runs, and each file worker refills a 64 KB buffer one +`RandomAccess.Read` at a time. Measured warm and in either order, reading 256 KB or 1 MB at a time +moved dop 8 from 21.2 to 19.5 ms at best, within run-to-run spread, and the gap to the stream engine +was ~10 % (19–21 ms against 17.4–18.3 ms), not 30 %. Not changed. diff --git a/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs index 9f61486..6806f58 100644 --- a/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs +++ b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs @@ -44,7 +44,7 @@ public static bool IsBlank(ReadOnlySpan column, in ColumnParseOptions opti public static string String(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options) { var trimmed = TrimColumn(column, in options); - return context.StringPool is { } pool ? pool.GetOrAdd(trimmed, context.Encoding) : context.Encoding.GetString(trimmed); + return context.StringPool is { } pool ? Utf8StringCache.GetOrAdd(pool, trimmed, context.Encoding) : context.Encoding.GetString(trimmed); } /// Parses a column, honoring the provider's decimal separator. diff --git a/src/FixedWidthParser/Infrastructure/Utf8StringCache.cs b/src/FixedWidthParser/Infrastructure/Utf8StringCache.cs new file mode 100644 index 0000000..86c8752 --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/Utf8StringCache.cs @@ -0,0 +1,100 @@ +using System.Runtime.CompilerServices; +using System.Text; +using CommunityToolkit.HighPerformance.Buffers; + +namespace FixedWidthParser.Infrastructure +{ + /// + /// A lock-free cache of UTF-8 columns in front of a , one per thread, because the + /// pool's own lookup is slow (docs/reader-performance.md). + /// Fixed-size and 4-way set-associative, as in ExcelReader's Utf8StringCache: a full set evicts its + /// oldest entry, and every hit is re-verified against the bytes, so a collision costs a lookup in the pool, + /// never a wrong string. A miss goes to the pool, so every string returned was handed out by it. + /// + internal sealed class Utf8StringCache + { + private const int SetCount = 1024; + private const int Ways = 4; + private const int MaxKeyLength = 64; + + [ThreadStatic] + private static Utf8StringCache? t_current; + + private readonly StringPool _pool; + private readonly string?[] _values = new string[SetCount * Ways]; + private readonly int[] _hashes = new int[SetCount * Ways]; + + private Utf8StringCache(StringPool pool) + { + _pool = pool; + } + + /// The string holds for , decoded with . + internal static string GetOrAdd(StringPool pool, ReadOnlySpan column, Encoding encoding) + { + if (column.Length > MaxKeyLength || encoding.CodePage != Encoding.UTF8.CodePage) + { + return pool.GetOrAdd(column, encoding); + } + var cache = t_current; + if (cache is null || !ReferenceEquals(cache._pool, pool)) + { + // ponytail: one pool per thread; a thread alternating between pools rebuilds its cache each switch. + t_current = cache = new Utf8StringCache(pool); + } + return cache.GetOrAdd(column, encoding); + } + + private string GetOrAdd(ReadOnlySpan utf8, Encoding encoding) + { + if (utf8.IsEmpty) + { + return string.Empty; + } + int hash = ComputeHash(utf8); + int start = (hash & (SetCount - 1)) * Ways; + for (int i = start; i < start + Ways; i++) + { + string? cached = _values[i]; + if (cached is null) + { + break; + } + if (_hashes[i] == hash && Matches(cached, utf8)) + { + return cached; + } + } + // Decoded with the context's encoding, so a strict decoder still rejects invalid bytes. + string value = _pool.GetOrAdd(utf8, encoding); + Array.Copy(_values, start, _values, start + 1, Ways - 1); + Array.Copy(_hashes, start, _hashes, start + 1, Ways - 1); + _values[start] = value; + _hashes[start] = hash; + return value; + } + + [SkipLocalsInit] + private static bool Matches(string cached, ReadOnlySpan utf8) + { + if (cached.Length > utf8.Length) + { + return false; + } + if (cached.Length == utf8.Length && Ascii.Equals(cached, utf8)) + { + return true; + } + Span buffer = stackalloc byte[MaxKeyLength]; + return Encoding.UTF8.TryGetBytes(cached, buffer, out int written) + && buffer[..written].SequenceEqual(utf8); + } + + private static int ComputeHash(ReadOnlySpan utf8) + { + HashCode hash = default; + hash.AddBytes(utf8); + return hash.ToHashCode(); + } + } +} diff --git a/tests/FixedWidthParser.Tests/ParserTests.cs b/tests/FixedWidthParser.Tests/ParserTests.cs index 10d1a6b..0074eaa 100644 --- a/tests/FixedWidthParser.Tests/ParserTests.cs +++ b/tests/FixedWidthParser.Tests/ParserTests.cs @@ -73,6 +73,41 @@ public void TryParse_WithStringPool_ReturnsInternedInstances() Assert.Same(first.Name, second.Name); } + // More values than the per-thread cache in front of the pool holds, so sets fill, evict and collide: + // every value must still decode right. (The pool evicts too, so instances are not compared here.) + [Fact] + public void TryParse_WithStringPool_ManyValues_DecodeRight() + { + var layout = FixedWidthLayout.FromAttributes(); + var pool = new StringPool(); + var context = CtxWith(pool: pool); + var names = Enumerable.Range(0, 5000).Select(i => i % 3 == 0 ? "é" + i : "N" + i).ToArray(); + + for (int round = 0; round < 2; round++) + { + foreach (string name in names) + { + byte[] line = [.. Bytes(name), .. Enumerable.Repeat((byte)' ', 10 - Bytes(name).Length), .. "25 100.00 "u8]; + Assert.True(FixedWidth.TryParse(line, layout, context, out var model)); + Assert.Equal(name, model.Name); + } + } + } + + [Fact] + public void TryParse_WithStringPool_OtherEncodingsAndStrictUtf8_DecodeAsWithoutIt() + { + var layout = FixedWidthLayout.FromAttributes(); + var latin1 = CtxWith(pool: new StringPool(), encoding: System.Text.Encoding.Latin1); + var strict = CtxWith(pool: new StringPool(), encoding: new System.Text.UTF8Encoding(false, true)); + + Assert.True(FixedWidth.TryParse([0xC9, 0x41, 0x42], layout, latin1, out var model)); + Assert.Equal("ÉAB", model.Code); + Assert.True(FixedWidth.TryParse("éA"u8, layout, strict, out model)); + Assert.Equal("éA", model.Code); + Assert.Throws(() => FixedWidth.TryParse([0xC3, 0x41, 0x42], layout, strict, out _)); + } + [Fact] public void TryParse_WithoutPool_ReturnsDistinctInstances() { From ef287ed12988e192f770f2c9085c9f290fa16e8c Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 03:33:59 -0300 Subject: [PATCH 58/85] feat: enhance documentation for FixedWidthMapper and add example usage --- ARCHITECTURE.md | 4 +- cpp/README.md | 43 ++ cpp/tests/CMakeLists.txt | 2 +- cpp/tests/mapped.cpp | 155 +++++ .../include/fixedwidth.hpp | 559 +++++++++++++++++- 5 files changed, 759 insertions(+), 4 deletions(-) create mode 100644 cpp/tests/mapped.cpp diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6fbbfab..8b1de80 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -224,7 +224,9 @@ size and each batch's offset are known up front: batches format in parallel stra buffer, or into a pooled buffer written at its offset with `RandomAccess.Write`, and the bytes equal a serial write. -`python/` binds the C ABI with `ctypes`; `fixedwidth.hpp` wraps it for C++23. The NativeSmoke C program +`python/` binds the C ABI with `ctypes`; `fixedwidth.hpp` wraps it for C++23, and maps structs through +`FixedWidthMapper`, as ExcelReader's C++ header does: the bindings build the layout, a read converts +the columnar table to `T` row by row, and a write builds columns from the structs. The NativeSmoke C program pins every struct's layout with static asserts. ## Public surface diff --git a/cpp/README.md b/cpp/README.md index 3b966d6..40d2795 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -16,6 +16,49 @@ if (!table) { std::puts(table.error().message.c_str()); return 1; } std::span prices = table->values(1); ``` +## Structs + +Specialize `fw::FixedWidthMapper` once, as with ExcelReader's `ExcelMapper`, and the layout comes from the +bindings: `read_file`, `read_memory` and `TypedReader` read rows as `T`, and `write_file`/`write_memory` +write a range of them. + +```cpp +struct Quote +{ + std::string ticker; + double open = 0; + std::optional volume; +}; + +template <> +struct fw::FixedWidthMapper +{ + static constexpr auto get_bindings() + { + return std::tuple{ + fw::make_field("codneg", 12, 12, &Quote::ticker), + fw::make_field("preabe", 56, 13, &Quote::open, {.scale = 2}), + fw::make_field("quatot", 152, 18, &Quote::volume), + }; + } +}; + +auto quotes = fw::read_file("COTAHIST_A2023.TXT", {.header_lines = 1, .skip_invalid = true}); +for (const Quote& quote : *quotes) { /* ... */ } +auto bytes = fw::write_memory(std::vector{/* ... */}); +``` + +A field's type picks the column type: `std::string` or `std::string_view` (String), an integer (Int64, or +Decimal holding the unscaled value when the field sets a `scale`), `float` or `double` (Float64), +`std::chrono::sys_days` or `year_month_day` (Date), `std::chrono::microseconds` or `hh_mm_ss` +(Time), `std::chrono::system_clock::time_point` (Timestamp). `std::optional` makes the column nullable; a +null read leaves a plain field at its default. `make_field`'s last argument, `fw::FieldOptions`, takes the +rest of `ColumnSpec`: `scale`, `format`, `trim_char`, `trim_mode`, `alignment`, `padding`, `overflow`. + +`TableView` keeps the library's columns and converts a row on access; a `std::string_view` field points +into it. An integer field narrower than `int64_t` that a value does not fit fails the read with +`FW_INVALID_ARGUMENT` instead of wrapping. + ## CMake python python/scripts/build_native.py diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index e3d0693..9d6bedf 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -foreach(test read write arrow) +foreach(test read write arrow mapped) add_executable(fw_cpp_${test} ${test}.cpp) target_link_libraries(fw_cpp_${test} PRIVATE fw::fixedwidth) if(MSVC) diff --git a/cpp/tests/mapped.cpp b/cpp/tests/mapped.cpp new file mode 100644 index 0000000..f7d8d18 --- /dev/null +++ b/cpp/tests/mapped.cpp @@ -0,0 +1,155 @@ +#include "common.hpp" + +#include +#include +#include +#include +#include +#include +#include + +using namespace std::chrono; + +// Maps part of the COTAHIST-like content in common.hpp: the record type and gaps stay unmapped. +struct Quote +{ + std::string ticker; + double open = 0; + std::optional volume; +}; + +template <> +struct fw::FixedWidthMapper +{ + static constexpr auto get_bindings() + { + return std::tuple{ + fw::make_field("codneg", 2, 12, &Quote::ticker), + fw::make_field("preabe", 14, 13, &Quote::open, {.scale = 2, .padding = '0'}), + fw::make_field("quatot", 27, 18, &Quote::volume, {.padding = '0'}), + }; + } +}; + +// Every other column type. +struct Trade +{ + std::string code; + int32_t quantity = 0; + int64_t cents = 0; + sys_days day; + std::optional> time; + system_clock::time_point at; + + // hh_mm_ss has no operator==. + bool operator==(const Trade& other) const + { + auto duration = [](const std::optional>& t) { return t.transform([](auto v) { return v.to_duration(); }); }; + return code == other.code && quantity == other.quantity && cents == other.cents && day == other.day && + duration(time) == duration(other.time) && at == other.at; + } +}; + +template <> +struct fw::FixedWidthMapper +{ + static constexpr auto get_bindings() + { + return std::tuple{ + fw::make_field("code", 0, 6, &Trade::code), + fw::make_field("quantity", 6, 5, &Trade::quantity), + fw::make_field("cents", 11, 9, &Trade::cents, {.scale = 2}), + fw::make_field("day", 20, 8, &Trade::day, {.format = "yyyyMMdd"}), + fw::make_field("time", 28, 6, &Trade::time, {.format = "HHmmss"}), + fw::make_field("at", 34, 14, &Trade::at, {.format = "yyyyMMddHHmmss"}), + }; + } +}; + +struct Small +{ + int8_t value = 0; +}; + +template <> +struct fw::FixedWidthMapper +{ + static constexpr auto get_bindings() { return std::tuple{fw::make_field("value", 0, 4, &Small::value)}; } +}; + +static_assert(fw::mapped_range>); +static_assert(!fw::mapped_range>); +static_assert(std::ranges::random_access_range>); + +static void check_quotes(const fw::TableView& quotes) +{ + CHECK(quotes.size() == 3); + const Quote first = quotes[0]; + CHECK(first.ticker == "PETR4" && first.open == 1234.56 && first.volume == 1000); + CHECK(quotes[1].ticker == "VALE3" && !quotes[1].volume.has_value()); + CHECK(quotes.at(2)->open == 43.21 && quotes.at(2)->volume == 50); + CHECK(!quotes.at(3).has_value()); + std::vector tickers; + for (const Quote& quote : quotes) { + tickers.push_back(quote.ticker); + } + CHECK((tickers == std::vector{"PETR4", "VALE3", "ITUB4"})); + CHECK(std::ranges::distance(quotes) == 3 && (*(quotes.end() - 1)).ticker == "ITUB4"); +} + +int main() +{ + auto quotes = fw::read_memory(content, skip_header_and_invalid()); + CHECK(quotes.has_value()); + check_quotes(*quotes); + CHECK(quotes->table().lines_skipped() == 3); + + auto parallel = fw::read_memory(content, skip_header_and_invalid(4)); + CHECK(parallel.has_value()); + check_quotes(*parallel); + + auto reader = fw::TypedReader::open_memory(content, {.read = skip_header_and_invalid(), .batch_rows = 2}); + CHECK(reader.has_value()); + std::vector batched; + while (true) { + auto batch = reader->next(); + CHECK(batch.has_value()); + if (!batch->has_value()) { + break; + } + for (const Quote& quote : **batch) { + batched.push_back(quote.ticker); + } + } + CHECK((batched == std::vector{"PETR4", "VALE3", "ITUB4"})); + + auto failed = fw::read_memory(content, {.header_lines = 1}); + CHECK(!failed.has_value() && failed.error().status == FW_PARSE_ERROR && failed.error().line_number == 5); + + const std::vector trades{ + {"PETR4", 100, 123456, sys_days(2023y / October / 24), hh_mm_ss(10h + 30min + 5s), + sys_days(2023y / October / 24) + 10h + 30min + 5s}, + {"VALE3", -7, -5, sys_days(1999y / December / 31), std::nullopt, sys_days(2000y / January / 1)}, + }; + auto written = fw::write_memory(trades); + CHECK(written.has_value()); + const std::string text(written->begin(), written->end()); + // The decimal's two decimals are implied; the empty time is blank. + CHECK(text == "PETR4 100 1234562023102410300520231024103005\n" + "VALE3 -7 -519991231 20000101000000\n"); + + auto path = std::string("fw_cpp_mapped.txt"); + CHECK(fw::write_file(path, trades).has_value()); + auto from_file = fw::read_file(path, {.degree_of_parallelism = 1}); + std::remove(path.c_str()); + CHECK(from_file.has_value()); + CHECK(from_file->size() == 2); + CHECK((*from_file)[0] == trades[0] && (*from_file)[1] == trades[1]); + + auto too_big = fw::read_memory(" 12\n 300\n"); + CHECK(!too_big.has_value() && too_big.error().status == FW_INVALID_ARGUMENT); + CHECK(too_big.error().message.find("'value' holds 300 at row 1") != std::string::npos); + + std::puts("mapped ok"); + return 0; +} diff --git a/src/FixedWidthParser.Native/include/fixedwidth.hpp b/src/FixedWidthParser.Native/include/fixedwidth.hpp index bab3962..2f77a80 100644 --- a/src/FixedWidthParser.Native/include/fixedwidth.hpp +++ b/src/FixedWidthParser.Native/include/fixedwidth.hpp @@ -1,16 +1,26 @@ -// Header-only C++23 wrapper over fixedwidth.h: RAII handles, options structs with defaults, and -// std::expected results. Link against the native library (fw::fixedwidth in cpp/CMakeLists.txt). +// Header-only C++23 wrapper over fixedwidth.h: RAII handles, options structs with defaults, +// std::expected results, and structs mapped to columns (FixedWidthMapper). Link against the native +// library (fw::fixedwidth in cpp/CMakeLists.txt). #pragma once #include "fixedwidth.h" +#include +#include +#include +#include #include #include #include +#include +#include #include +#include #include #include #include +#include +#include #include #include @@ -390,4 +400,549 @@ namespace fw { return write_memory(table.get(), layout, options); } + + // ---- Struct mapping ---------------------------------------------------------------------------- + // + // Specialize FixedWidthMapper with a `static constexpr auto get_bindings()` returning a std::tuple of + // make_field results; read_file, read_memory, TypedReader, write_file and write_memory all use + // that one mapping. A field's type picks its column type: std::string or std::string_view -> String, + // an integer -> Int64 (Decimal, unscaled, when the field sets a scale), float or double -> Float64, + // std::chrono::sys_days or year_month_day -> Date, std::chrono::microseconds or + // hh_mm_ss -> Time, std::chrono::system_clock::time_point -> Timestamp. std::optional + // makes the column nullable. + + /// The optional parts of a mapped column; see ColumnSpec. + struct FieldOptions + { + int32_t scale = 0; + const char* format = nullptr; + char trim_char = 0; + Trim trim_mode = Trim::Both; + Align alignment = Align::Default; + char padding = 0; + Overflow overflow = Overflow::Throw; + }; + + namespace detail + { + template + struct FieldBinding + { + const char* name; + int32_t start; + int32_t length; + T Class::*member; + FieldOptions options; + using FieldType = T; + }; + } + + /// Binds a member to the column at bytes [start, start + length). + template + constexpr detail::FieldBinding make_field(const char* name, int32_t start, int32_t length, T Class::*member, + FieldOptions options = {}) + { + return {name, start, length, member, options}; + } + + /// Maps a row type to columns: specialize with `static constexpr auto get_bindings()`. + template + struct FixedWidthMapper; + + namespace detail + { + template + struct IsOptional : std::false_type + { + using Inner = T; + }; + + template + struct IsOptional> : std::true_type + { + using Inner = T; + }; + + template + using unwrap_optional_t = typename IsOptional::Inner; + + template + inline constexpr bool always_false = false; + + template + using standard_integer_t = std::conditional_t, std::make_signed_t, std::make_unsigned_t>; + + using micros = std::chrono::microseconds; + + template + constexpr Type type_of(int32_t scale) + { + if constexpr (std::is_same_v || std::is_same_v) + { + return Type::String; + } + else if constexpr (std::is_integral_v && !std::is_same_v) + { + return scale > 0 ? Type::Decimal : Type::Int64; + } + else if constexpr (std::is_floating_point_v) + { + return Type::Float64; + } + else if constexpr (std::is_same_v || std::is_same_v) + { + return Type::Date; + } + else if constexpr (std::is_same_v || std::is_same_v>) + { + return Type::Time; + } + else if constexpr (std::is_same_v) + { + return Type::Timestamp; + } + else + { + static_assert(always_false, "no fixed-width column type for this field type"); + } + } + + template + constexpr auto bindings_of() + { + return FixedWidthMapper::get_bindings(); + } + + template + inline constexpr size_t field_count = std::tuple_size_v())>; + + template + ColumnSpec spec_of(const FieldBinding& binding) + { + const FieldOptions& options = binding.options; + return ColumnSpec{ + .name = binding.name, + .start = binding.start, + .length = binding.length, + .type = type_of>(options.scale), + .nullable = IsOptional::value, + .scale = options.scale, + .format = options.format == nullptr ? std::nullopt : std::optional(options.format), + .trim_char = options.trim_char, + .trim_mode = options.trim_mode, + .alignment = options.alignment, + .padding = options.padding, + .overflow = options.overflow, + }; + } + + template + Field value_at(const Table& table, int32_t column, int64_t row) + { + if constexpr (std::is_same_v || std::is_same_v) + { + return Field(table.string(column, row)); + } + else if constexpr (std::is_integral_v) + { + return static_cast(table.values(column)[row]); + } + else if constexpr (std::is_floating_point_v) + { + return static_cast(table.values(column)[row]); + } + else if constexpr (std::is_same_v || std::is_same_v) + { + return Field(std::chrono::sys_days(std::chrono::days(table.values(column)[row]))); + } + else if constexpr (std::is_same_v || std::is_same_v>) + { + return Field(micros(table.values(column)[row])); + } + else + { + return std::chrono::time_point_cast( + std::chrono::sys_time(micros(table.values(column)[row]))); + } + } + + template + T row_at(const Table& table, int64_t row, std::index_sequence) + { + static constexpr auto bindings = bindings_of(); + T instance{}; + // A null leaves a plain field at its default and an optional one empty. + (..., (table.is_valid(static_cast(Is), row) + ? void(instance.*(std::get(bindings).member) = + value_at>::FieldType>>( + table, static_cast(Is), row)) + : void())); + return instance; + } + + /// The int64 a narrower or unsigned integer field cannot hold is refused once, when the table is + /// handed over, instead of wrapping on every read. + template + Result check_ranges(const Table& table, std::index_sequence) + { + static constexpr auto bindings = bindings_of(); + Result result; + auto check = [&](auto index) + { + constexpr size_t I = decltype(index)::value; + using Field = unwrap_optional_t>::FieldType>; + if constexpr (std::is_integral_v && !std::is_same_v) + { + const auto values = table.values(static_cast(I)); + for (int64_t row = 0; row < table.row_count(); ++row) + { + if (table.is_valid(static_cast(I), row) && !std::in_range>(values[row])) + { + result = std::unexpected(Error{ + .status = FW_INVALID_ARGUMENT, + .message = "column '" + std::string(std::get(bindings).name) + "' holds " + std::to_string(values[row]) + + " at row " + std::to_string(row) + ", which its field's type cannot represent", + }); + return false; + } + } + } + return true; + }; + (void)(... && check(std::integral_constant{})); + return result; + } + } + + /// The layout a FixedWidthMapper describes. + template + Result layout_for(const LayoutOptions& options = {}) + { + static constexpr auto bindings = detail::bindings_of(); + const auto columns = std::apply([](const auto&... binding) { return std::array{detail::spec_of(binding)...}; }, + bindings); + return Layout::create(columns, options); + } + + /// The rows of a read, held in the library's columnar memory and converted to T on access; a + /// random-access range of T. A std::string_view field points into this view. + template + class TableView + { + public: + class iterator + { + public: + using iterator_category = std::random_access_iterator_tag; + using iterator_concept = std::random_access_iterator_tag; + using value_type = T; + using difference_type = std::ptrdiff_t; + using pointer = void; + using reference = T; + + iterator() = default; + + T operator*() const { return (*view_)[row_]; } + T operator[](difference_type n) const { return (*view_)[row_ + n]; } + + iterator& operator++() { ++row_; return *this; } + iterator operator++(int) { iterator copy = *this; ++row_; return copy; } + iterator& operator--() { --row_; return *this; } + iterator operator--(int) { iterator copy = *this; --row_; return copy; } + iterator& operator+=(difference_type n) { row_ += n; return *this; } + iterator& operator-=(difference_type n) { row_ -= n; return *this; } + + friend iterator operator+(iterator it, difference_type n) { return it += n; } + friend iterator operator+(difference_type n, iterator it) { return it += n; } + friend iterator operator-(iterator it, difference_type n) { return it -= n; } + friend difference_type operator-(const iterator& lhs, const iterator& rhs) { return lhs.row_ - rhs.row_; } + friend bool operator==(const iterator& lhs, const iterator& rhs) { return lhs.row_ == rhs.row_; } + friend std::strong_ordering operator<=>(const iterator& lhs, const iterator& rhs) { return lhs.row_ <=> rhs.row_; } + + private: + friend class TableView; + iterator(const TableView* view, int64_t row) : view_(view), row_(row) {} + + const TableView* view_ = nullptr; + int64_t row_ = 0; + }; + + /// Takes over a table read through layout_for(), checking that every integer fits its field. + static Result adopt(Table table) + { + if (auto ok = detail::check_ranges(table, std::make_index_sequence>{}); !ok) + { + return std::unexpected(ok.error()); + } + return TableView(std::move(table)); + } + + iterator begin() const { return iterator(this, 0); } + iterator end() const { return iterator(this, size()); } + int64_t size() const noexcept { return table_.row_count(); } + bool empty() const noexcept { return size() == 0; } + + /// Row `row`, unchecked; at() is the checked read. + T operator[](int64_t row) const { return detail::row_at(table_, row, std::make_index_sequence>{}); } + + std::optional at(int64_t row) const + { + if (row < 0 || row >= size()) + { + return std::nullopt; + } + return (*this)[row]; + } + + /// The columns underneath, with the read's line counts. + const Table& table() const noexcept { return table_; } + + private: + explicit TableView(Table table) noexcept : table_(std::move(table)) {} + Table table_; + }; + + /// Reads a whole file as T. + template + Result> read_file(std::string_view path, const ReadOptions& options = {}) + { + return layout_for() + .and_then([&](const Layout& layout) { return read_file(path, layout, options); }) + .and_then([](Table table) { return TableView::adopt(std::move(table)); }); + } + + /// Reads a whole buffer as T. data need not outlive the call, but std::string_view fields point into the view. + template + Result> read_memory(std::string_view data, const ReadOptions& options = {}) + { + return layout_for() + .and_then([&](const Layout& layout) { return read_memory(data, layout, options); }) + .and_then([](Table table) { return TableView::adopt(std::move(table)); }); + } + + /// A sequential read as T in views of at most batch_rows rows. Use from one thread at a time. + template + class TypedReader + { + public: + static Result open_file(std::string_view path, const ReaderOptions& options = {}) + { + return layout_for() + .and_then([&](const Layout& layout) { return Reader::open_file(path, layout, options); }) + .transform([](Reader reader) { return TypedReader(std::move(reader)); }); + } + + /// Reads data in place: it must outlive the reader. + static Result open_memory(std::string_view data, const ReaderOptions& options = {}) + { + return layout_for() + .and_then([&](const Layout& layout) { return Reader::open_memory(data, layout, options); }) + .transform([](Reader reader) { return TypedReader(std::move(reader)); }); + } + + /// The next batch, or an empty optional at the end. + Result>> next() + { + auto batch = reader_.next(); + if (!batch) + { + return std::unexpected(std::move(batch.error())); + } + if (!batch->has_value()) + { + return std::optional>{}; + } + return TableView::adopt(std::move(**batch)).transform([](TableView view) { return std::optional>(std::move(view)); }); + } + + private: + explicit TypedReader(Reader reader) : reader_(std::move(reader)) {} + Reader reader_; + }; + + namespace detail + { + /// One field's column on the way out: the values as the library stores them, and a validity + /// bitmap once a null appears. + template + struct ColumnBuilder + { + using Field = unwrap_optional_t; + static constexpr bool is_string = std::is_same_v || std::is_same_v; + using Stored = std::conditional_t, double, + std::conditional_t || + std::is_same_v, + int32_t, int64_t>>; + + std::vector values; + std::vector offsets{0}; + std::string data; + std::vector validity; + int64_t rows = 0; + bool any_null = false; + bool out_of_range = false; + + void push(const T& value) + { + if (validity.size() * 8 <= static_cast(rows)) + { + validity.push_back(0); + } + if constexpr (IsOptional::value) + { + if (!value.has_value()) + { + any_null = true; + if constexpr (is_string) + { + offsets.push_back(offsets.back()); + } + else + { + values.push_back(Stored{}); + } + ++rows; + return; + } + append(*value); + } + else + { + append(value); + } + validity[static_cast(rows >> 3)] |= static_cast(1 << (rows & 7)); + ++rows; + } + + void append(const Field& value) + { + if constexpr (is_string) + { + data.append(value); + offsets.push_back(static_cast(std::min(data.size(), std::numeric_limits::max()))); + } + else if constexpr (std::is_integral_v) + { + out_of_range |= !std::in_range(static_cast>(value)); + values.push_back(static_cast(value)); + } + else if constexpr (std::is_floating_point_v) + { + values.push_back(static_cast(value)); + } + else if constexpr (std::is_same_v || std::is_same_v) + { + values.push_back(static_cast(std::chrono::sys_days(value).time_since_epoch().count())); + } + else if constexpr (std::is_same_v) + { + values.push_back(value.count()); + } + else if constexpr (std::is_same_v>) + { + values.push_back(value.to_duration().count()); + } + else + { + values.push_back(std::chrono::time_point_cast(value).time_since_epoch().count()); + } + } + + bool overflowed() const { return is_string && data.size() > static_cast(std::numeric_limits::max()); } + + fw_column column(const ColumnSpec& spec) const + { + return fw_column{ + .type = static_cast(spec.type), + .scale = spec.scale, + .length = rows, + .values = is_string ? static_cast(offsets.data()) : static_cast(values.data()), + .validity = any_null ? validity.data() : nullptr, + .data = is_string && !data.empty() ? reinterpret_cast(data.data()) : nullptr, + .data_len = is_string ? static_cast(data.size()) : 0, + }; + } + }; + + template + auto make_builders(std::index_sequence) + { + static constexpr auto bindings = bindings_of(); + return std::tuple>::FieldType>...>{}; + } + + /// Builds the columns of rows, then hands write the table and its layout; both live only for the call. + template + Result write_rows(R&& rows, Write&& write) + { + using T = std::remove_cvref_t>; + static constexpr auto bindings = bindings_of(); + constexpr auto indices = std::make_index_sequence>{}; + auto layout = layout_for(); + if (!layout) + { + return std::unexpected(std::move(layout.error())); + } + auto builders = make_builders(indices); + for (const auto& row : rows) + { + std::apply([&](auto&... builder) + { std::apply([&](const auto&... binding) { (..., builder.push(row.*(binding.member))); }, bindings); }, + builders); + } + const bool overflowed = std::apply([](const auto&... builder) { return (... || builder.overflowed()); }, builders); + if (overflowed) + { + return std::unexpected(Error{.status = FW_INVALID_ARGUMENT, .message = "a string column holds more than 2 GiB, which int32 offsets cannot address"}); + } + const bool out_of_range = std::apply([](const auto&... builder) { return (... || builder.out_of_range); }, builders); + if (out_of_range) + { + return std::unexpected(Error{.status = FW_INVALID_ARGUMENT, .message = "an integer field holds a value above INT64_MAX, which an int64 column cannot represent"}); + } + auto columns = std::apply( + [&](const auto&... builder) + { + return std::apply([&](const auto&... binding) { return std::array{builder.column(spec_of(binding))...}; }, + bindings); + }, + builders); + const fw_table table{.column_count = static_cast(columns.size()), .row_count = std::get<0>(builders).rows, .columns = columns.data()}; + return write(table, *layout); + } + } + + template + concept mapped_range = std::ranges::input_range && requires { FixedWidthMapper>>::get_bindings(); }; + + /// Writes rows, a range of a type with a FixedWidthMapper, to a file. An empty std::optional writes as + /// its column filled with trim_char. + template + Result write_file(std::string_view path, R&& rows, const WriteOptions& options = {}) + { + return detail::write_rows(std::forward(rows), [&](const fw_table& table, const Layout& layout) { return write_file(path, table, layout, options); }); + } + + /// write_file into memory. + template + Result> write_memory(R&& rows, const WriteOptions& options = {}) + { + std::vector bytes; + auto written = detail::write_rows(std::forward(rows), + [&](const fw_table& table, const Layout& layout) -> Result + { + auto result = write_memory(table, layout, options); + if (!result) + { + return std::unexpected(std::move(result.error())); + } + bytes = std::move(*result); + return {}; + }); + if (!written) + { + return std::unexpected(std::move(written.error())); + } + return bytes; + } } From a03957c69cebe2cb7477f09e0bca6f4ffe06e0ae Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 04:13:02 -0300 Subject: [PATCH 59/85] docs: split the README into a usage guide and a benchmarks page The README was the NuGet landing page, the whole usage manual and a benchmark report at once. Following ExcelReader, it keeps the overview, installation and a quick start, and links to docs/guide/ (models and layouts, reading, writing, versioning), whose text moved unchanged. The comparison tables were 1.1 numbers from another machine; docs/performance/benchmarks.md has 2.0 numbers, the machine, and why the comparison is not like-for-like. --- CONTRIBUTING.md | 2 +- README.md | 304 ++------------------------------- docs/README.md | 7 +- docs/guide/models.md | 105 ++++++++++++ docs/guide/reading.md | 118 +++++++++++++ docs/guide/versioning.md | 7 + docs/guide/writing.md | 17 ++ docs/performance/benchmarks.md | 81 +++++++++ 8 files changed, 353 insertions(+), 288 deletions(-) create mode 100644 docs/guide/models.md create mode 100644 docs/guide/reading.md create mode 100644 docs/guide/versioning.md create mode 100644 docs/guide/writing.md create mode 100644 docs/performance/benchmarks.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 54ff55c..b143725 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,7 +21,7 @@ rewrites — see [ARCHITECTURE.md](ARCHITECTURE.md) for the shape of the codebas you. At a release, the unshipped entries move to `PublicAPI.Shipped.txt`. - **Minor versions don't break callers.** The policy is in the README's - [Versioning](README.md#versioning) section; in practice: + [Versioning](docs/guide/versioning.md) section; in practice: - `IFixedWidthModel` and `IFixedWidthConverter` are implemented by users, so a minor version never adds a member to them. New information reaches them as a new property on `FixedWidthContext`. diff --git a/README.md b/README.md index d9f19d1..dff2ad2 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,10 @@ dotnet add package FixedWidthParser.NET The package ships the Roslyn source generator bundled as an analyzer. Models that implement `IFixedWidthModel` get a generated parser and writer automatically; no extra package or setup is required. -## Defining a Model +## Quick Start -An attribute model needs a public parameterless constructor and mapped fields or properties; `FixedWidthLayout.FromAttributes()` turns it into a layout, built once per type by reflection: +Describe a line with `[FixedColumn(start, length)]`, both in bytes, then read and write through the +model's layout: ```csharp using FixedWidthParser; @@ -64,57 +65,6 @@ public readonly record struct Person [FixedColumn(10, 5)] public int Age { get; init; } [FixedColumn(15, 10)] public double Salary { get; init; } } -``` - -`start` is the 0-based offset and `length` is the column width, both **in bytes**. In UTF-8 a character such as `é` takes two bytes, so a layout that counts characters only matches a UTF-8 file whose text is ASCII; for accented text, read the file in the single-byte encoding it was written in (see [Encodings](#encodings)). - -For source generation, make the model `partial` and implement the marker interface: - -```csharp -using FixedWidthParser; - -public readonly partial record struct GeneratedPerson : IFixedWidthModel -{ - [FixedColumn(0, 10)] public string Name { get; init; } - [FixedColumn(10, 5)] public int Age { get; init; } - [FixedColumn(15, 10)] public double Salary { get; init; } -} -``` - -## Layouts without attributes - -For a type you cannot annotate, or a layout known only at run time, describe the columns in code. Each column binds through a getter (used to write) and a setter (used to read), so nothing is reflected over: - -```csharp -using FixedWidthParser; - -var layout = FixedWidthLayout.Build(b => b - .Factory(() => new Order()) - .Column("Id", 0, 6, (in Order o) => o.Id, (ref Order o, int v) => o.Id = v, - new FixedWidthColumnOptions { Alignment = Alignment.Right, Padding = '0' }) - .Column("Name", 6, 20, (in Order o) => o.Name, (ref Order o, string v) => o.Name = v) - .NullableColumn("Discount", 26, 10, (in Order o) => o.Discount, (ref Order o, decimal? v) => o.Discount = v) - .DecodedColumn("Date", 36, 8, (in Order o) => o.Date, (ref Order o, DateOnly v) => o.Date = v, - new FixedWidthColumnOptions { Format = "yyyyMMdd" }) - .ConvertedColumn("Total", 44, 12, (in Order o) => o.Total, (ref Order o, Money v) => o.Total = v, new MoneyConverter())); -``` - -| Method | Column type | -|---|---| -| `Column` | `string` | -| `Column` | a type that parses from and formats to UTF-8: numbers, `Guid`, … | -| `DecodedColumn` | a type that parses only from characters (decoded first) | -| `NullableColumn` | `T?` of a `Column` type; a blank column is `null` | -| `NullableDecodedColumn` | `T?` of a `DecodedColumn` type, such as `DateOnly?` | -| `ConvertedColumn` | any type, through an `IFixedWidthConverter` instance | -| `NullableConvertedColumn` | `T?` of a value type, through an `IFixedWidthConverter` instance | - -`FixedWidthColumnOptions` carries the same settings as `[FixedColumn]`, with the same defaults (see [Formatting Options](#formatting-options)). A reference-type model needs `Factory` to be read; a value type without one starts from `default`. Pass `null` as the getter or the setter to build a layout that only reads or only writes; using it the other way throws `InvalidOperationException` before any line is touched. The column name appears in error messages. - -## Reading - -```csharp -using FixedWidthParser; var layout = FixedWidthLayout.FromAttributes(); @@ -123,243 +73,25 @@ foreach (var person in FixedWidth.ReadFile("people.txt", layout)) // ... } -foreach (var person in FixedWidth.ReadFile("people.txt")) -{ - // generated TryParse for each line -} -``` - -`Read(Stream, …, leaveOpen)` reads a stream once; `ReadFile` reopens the file on each enumeration. Reading is lazy and reuses a pooled buffer; each line is parsed in place, without a string per line. A layout is immutable: build it once and share it across reads and threads. - -Empty lines (including a trailing newline at end of file) are skipped: they are counted toward the line number but not yielded. Lines end at `\n`, with a preceding `\r` dropped, and a leading UTF-8 byte order mark is skipped. - -### Async - -```csharp -await foreach (var person in FixedWidth.ReadFileAsync("people.txt", layout)) -{ - // ... -} -``` - -`ReadAsync(Stream, leaveOpen)` mirrors the synchronous overload, and cancellation is honored via `WithCancellation`. A [`PipeReader`](https://learn.microsoft.com/dotnet/standard/io/pipelines) — a Kestrel request body, a socket — is read through `pipeReader.AsStream()`. - -### Parallel aggregation - -To fold a large file into one result — totals, counts, groups — read it on several threads: - -```csharp -public sealed class Payroll : IRecordAccumulator -{ - public double Total { get; private set; } - - public void Add(in Person person) - { - Total += person.Salary; - } - - public void Merge(Payroll following) - { - Total += following.Total; - } -} - -var payroll = await FixedWidth.AggregateFileAsync("people.txt", layout); -// GeneratedPayroll: the same accumulator over GeneratedPerson. -var generated = await FixedWidth.AggregateFileAsync("people.txt"); -``` - -The file is cut into byte ranges, each folded into its own accumulator on a worker thread, and the -accumulators are merged left to right in file order. `Add` must touch only its own instance; `Merge` -runs on one thread. Every line is parsed exactly once, and a failing line reports the same line number -as a sequential read. `FixedWidthReaderOptions.DegreeOfParallelism` caps the threads (`0`, the -default, means one per processor; `1` reads the file in one pass); other reads ignore it. `OnParseError`, when set, runs on -the worker threads. - -`AggregateAsync` does the same over a `Stream`, which is read once: one thread reads it into chunks -and the others parse. - -### Options - -Every read takes an optional `FixedWidthReaderOptions`: - -```csharp -var options = new FixedWidthReaderOptions -{ - FormatProvider = CultureInfo.GetCultureInfo("pt-BR"), - Encoding = Encoding.Latin1, - StringPool = new StringPool(), - BufferSize = 64 * 1024, - MaxLineLength = 4096, - OnParseError = ex => { log.Warning(ex.Message); return true; }, // skip the line -}; -var people = FixedWidth.ReadFile("people.txt", layout, options); -``` - -| Option | Default | Meaning | -|---|---|---| -| `FormatProvider` | invariant culture | Numbers and dates. | -| `Encoding` | UTF-8 | Text columns. Must be ASCII-compatible. | -| `StringPool` | none | Interns repeated string values. | -| `BufferSize` | 65,536 | Bytes read from the source at a time. | -| `MaxLineLength` | 1,048,576 | Longest line accepted; a longer one throws `InvalidDataException`. | -| `OnParseError` | throw | Return `true` to skip a line that fails to parse. | -| `DegreeOfParallelism` | processor count | Threads for `AggregateFileAsync`/`AggregateAsync`; `1` reads in one pass. | - -### Errors - -A line that fails to parse throws `FixedWidthParseException`, a `FormatException` that says where: - -```text -Line 2 could not be parsed into Person: column 'Age' (start 10, length 5) holds "XX ". -``` - -`LineNumber`, `ColumnName`, `ColumnStart`, `ColumnLength` and `RawValue` carry the same information. A line shorter than the layout reports how long it is and how long the layout needs it to be. The parser pays nothing for this on lines that parse: a failed line is parsed a second time to find the column. - -### Untrusted input - -A reader buffers a line until it finds its line break. `MaxLineLength` bounds that buffer, so a request body or file without line breaks throws `InvalidDataException` instead of growing the buffer until memory runs out. Set it close to your layout's line length when the source is untrusted. - -## Single Lines - -```csharp -FixedWidth.TryParse("John Doe 30 60000.00 "u8, layout, default, out Person person); - -FixedWidth.TryParse(lineBytes, new FixedWidthContext { FormatProvider = culture }, out var generated); -``` - -`FixedWidthContext` carries the format provider, the encoding and the string pool for one line; `default` means invariant culture, UTF-8 and no pool. - -## Writing - -```csharp -var people = new[] -{ - new Person { Name = "John Doe", Age = 30, Salary = 60000 }, - new Person { Name = "Jane", Age = 28, Salary = 55000 }, -}; - +var people = new[] { new Person { Name = "John Doe", Age = 30, Salary = 60000 } }; using var stream = File.Create("out.txt"); -FixedWidth.WriteMany(stream, people.AsSpan(), layout); -FixedWidth.WriteMany(stream, generatedPeople); // generated models need no layout -``` - -`Write`, `WriteMany` (span or enumerable) and their async versions write each record followed by `NewLine`, batched into one pooled buffer. `FixedWidthWriterOptions` sets the `FormatProvider`, the `Encoding` and the `NewLine` (default `Environment.NewLine`). `FixedWidth.TryFormat(in model, layout, Span, in FixedWidthContext, out int)`, or its generated overload without the layout, formats one line into your own buffer. - -## Formatting Options - -Each column can be tuned through named attribute arguments: - -```csharp -[FixedColumn(0, 8, Alignment = Alignment.Right, Padding = '0')] public int Id { get; init; } // "00000042" -[FixedColumn(8, 10, Format = "F2")] public double Amount { get; init; } // "1234.50 " -[FixedColumn(18, 5, Overflow = OverflowBehavior.Truncate)] public string Code { get; init; } -[FixedColumn(23, 5, TrimChar = '0', TrimMode = TrimMode.Leading)] public int Count { get; init; } -``` - -- **`Alignment`**: `Left` (default) or `Right`. -- **`Padding`**: fill byte when writing (default space; for example `'0'` for zero-padding). Must be ASCII. -- **`TrimChar`** / **`TrimMode`**: what to strip when parsing, and from which side. Must be ASCII. -- **`Format`**: format string for `ISpanFormattable` when writing, and the exact format for `DateTime`, `DateOnly`, `TimeOnly` and `DateTimeOffset` when parsing. -- **`Overflow`**: `Default`, `Truncate` or `Throw`. `Default` resolves per type: strings truncate, everything else throws. A truncated string keeps whole characters, so a multi-byte character is never split. -- **`Converter`**: a type implementing `IFixedWidthConverter`, used instead of the built-in parsing and formatting. - -## Column Types - -`string`, `double` and `float`, any `IUtf8SpanParsable` (parsed straight from bytes), any other `ISpanParsable` (decoded first), `T?` of any of them (a blank column is `null`), and anything with a `Converter`. Writing needs `string`, `ISpanFormattable` or a converter. - -## Encodings - -Column positions are bytes, so the encoding only decides how text inside a column is read and written. UTF-8 files with ASCII text and single-byte encodings (Latin-1, Windows-1252, …) keep one byte per character. UTF-16 and EBCDIC are not supported. - -```csharp -var options = new FixedWidthReaderOptions { Encoding = Encoding.Latin1 }; -var records = FixedWidth.ReadFile("legacy.txt", options); -``` - -On .NET, Windows-1252 and other code pages need `Encoding.RegisterProvider(CodePagesEncodingProvider.Instance)` first. - -## Validation - -Invalid layouts fail fast with an `InvalidOperationException` from `FixedWidthLayout.Build` or `FromAttributes`, or a generator diagnostic on generated models: negative `Start`, `Length < 1`, overlapping columns, a non-ASCII `Padding` or `TrimChar` (FWP012), an unsupported column type and a converter for the wrong type. Adjacent columns are valid. An attribute model that can be read but not written (or the reverse), such as one with a column type that only formats, gets a layout anyway; the side it cannot do throws on first use. - -## Versioning - -The package follows [Semantic Versioning](https://semver.org/). A minor version may add types, members, options and enum values, but does not remove or change anything; every pull request checks the package, binary-compatibly, against the last release. - -- **Settings** arrive as new properties on the options objects (`FixedWidthReaderOptions`, `FixedWidthWriterOptions`, `FixedWidthColumnOptions`) or on `FixedWidthContext`, never as a new parameter on an existing method. -- **Interfaces you implement** — `IFixedWidthModel` (normally through the generator), `IFixedWidthConverter` and `IRecordAccumulator` — never gain a required member. Anything added comes with a default implementation (`static virtual` or a default interface method). -- **`FixedWidthParser.Infrastructure`** holds what is public only because generated code calls it (`FixedWidthRuntime`, `ColumnParseOptions`, `ColumnFormatOptions`) or because a method returns it so `foreach` allocates nothing (the `*RecordEnumerable`/`*RecordEnumerator` types and the `*LineParser` strategies). These types are hidden from IntelliSense and not meant for direct use, but they follow the same rules as everything else: code compiled against one 2.x version, generated code included, keeps working with every later 2.x version. A new setting for generated code is a new constructor overload on the options structs. - -## Performance - -Measured with BenchmarkDotNet (`MemoryDiagnoser`) on .NET 10 with 1.1, whose reader decoded text to UTF-16 first; the 2.0 numbers are yet to be measured. Highlights: - -- Parsing a line is allocation-light on the runtime path and reflection-free on the generated path. -- Readers parse each line in place, without a string per line. -- Writing a span of records is zero-alloc per line. - -### Comparison with other libraries - -#### Reading - -Batched read of N fixed-width records (10-char string + 5-char int + 10-char double), compared against [FileHelpers](https://www.filehelpers.net/), [FlatFiles](https://github.com/jehugaleahsa/FlatFiles), and [RecordParser](https://github.com/leandromoh/RecordParser). - -``` -BenchmarkDotNet v0.15.8, Windows 11 (10.0.26200.8655/25H2/2025Update/HudsonValley2) -13th Gen Intel Core i7-1355U 1.70GHz, 1 CPU, 12 logical and 10 physical cores -.NET SDK 10.0.301 - [Host] : .NET 10.0.9 (10.0.9, 10.0.926.27113), X64 RyuJIT x86-64-v3 - DefaultJob : .NET 10.0.9 (10.0.9, 10.0.926.27113), X64 RyuJIT x86-64-v3 +FixedWidth.WriteMany(stream, people, layout); ``` -| Method | Count | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Gen1 | Allocated | Alloc Ratio | -|--------------------------- |------:|-----------:|---------:|----------:|------:|--------:|---------:|--------:|----------:|------------:| -| FixedWidthParser_Generated | 100 | 2.661 μs | 0.052 μs | 0.062 μs | 0.10 | 0.02 | 0.6294 | - | 3.86 KB | 0.09 | -| FixedWidthParser_Read | 100 | 4.460 μs | 0.074 μs | 0.065 μs | 0.16 | 0.04 | 0.6256 | - | 3.86 KB | 0.09 | -| RecordParser_Read | 100 | 7.047 μs | 0.091 μs | 0.080 μs | 0.26 | 0.06 | 0.6866 | - | 4.23 KB | 0.10 | -| FileHelpers_Read | 100 | 28.860 μs | 2.630 μs | 7.671 μs | 1.07 | 0.39 | 6.7444 | 0.2136 | 41.49 KB | 1.00 | -| FlatFiles_Read | 100 | 40.173 μs | 0.747 μs | 0.699 μs | 1.49 | 0.36 | 16.7847 | 0.5493 | 103.13 KB | 2.49 | -| | | | | | | | | | | | -| FixedWidthParser_Generated | 1000 | 26.273 μs | 0.347 μs | 0.325 μs | 0.13 | 0.00 | 6.2256 | - | 38.31 KB | 0.09 | -| FixedWidthParser_Read | 1000 | 43.935 μs | 0.467 μs | 0.390 μs | 0.21 | 0.00 | 6.2256 | - | 38.31 KB | 0.09 | -| RecordParser_Read | 1000 | 72.391 μs | 1.392 μs | 1.429 μs | 0.35 | 0.01 | 6.2256 | - | 38.69 KB | 0.10 | -| FileHelpers_Read | 1000 | 205.683 μs | 3.662 μs | 3.058 μs | 1.00 | 0.02 | 66.1621 | 17.0898 | 406.42 KB | 1.00 | -| FlatFiles_Read | 1000 | 402.237 μs | 6.329 μs | 5.610 μs | 1.96 | 0.04 | 155.2734 | 4.8828 | 953.2 KB | 2.35 | - -The reflection-based reader is **~5× faster** and allocates **11× less** than FileHelpers. The source-generated reader reaches **~8× faster** at the same allocation footprint. - -#### Writing - -Write of N fixed-width records to `Stream.Null` (formatting cost only, no I/O). - -| Method | Count | Mean | Error | StdDev | Median | Ratio | RatioSD | Gen0 | Gen1 | Allocated | Alloc Ratio | -|----------------------- |------:|--------------:|-------------:|-------------:|--------------:|------:|--------:|----------:|----------:|----------:|------------:| -| FixedWidthParser_Write | 1 | 121.4 ns | 2.01 ns | 1.57 ns | 120.9 ns | 0.71 | 0.01 | - | - | - | 0.00 | -| FileHelpers_Write | 1 | 170.2 ns | 1.40 ns | 1.17 ns | 170.3 ns | 1.00 | 0.01 | 0.0572 | - | 360 B | 1.00 | -| FlatFiles_Write | 1 | 357.2 ns | 5.71 ns | 5.07 ns | 355.9 ns | 2.10 | 0.03 | 0.1135 | - | 712 B | 1.98 | -| RecordParser_Write | 1 | 29,258.0 ns | 542.81 ns | 507.75 ns | 29,232.1 ns | 171.96 | 3.11 | 7.9956 | 1.0376 | 49,640 B | 137.89 | -| | | | | | | | | | | | | -| FixedWidthParser_Write | 100 | 8,665.8 ns | 169.30 ns | 424.73 ns | 8,519.2 ns | 0.77 | 0.04 | - | - | - | 0.00 | -| FileHelpers_Write | 100 | 11,283.8 ns | 187.38 ns | 166.11 ns | 11,255.9 ns | 1.00 | 0.02 | 5.2338 | - | 32,832 B | 1.00 | -| FlatFiles_Write | 100 | 25,250.0 ns | 493.33 ns | 411.95 ns | 25,367.7 ns | 2.24 | 0.05 | 11.3220 | - | 71,200 B | 2.17 | -| RecordParser_Write | 100 | 87,556.0 ns | 1,820.98 ns | 5,224.72 ns | 86,343.9 ns | 7.76 | 0.47 | 8.3008 | 0.9766 | 51,953 B | 1.58 | -| | | | | | | | | | | | | -| FixedWidthParser_Write | 1000 | 86,454.8 ns | 1,695.66 ns | 1,952.72 ns | 86,407.9 ns | 0.71 | 0.03 | - | - | - | 0.00 | -| FileHelpers_Write | 1000 | 122,427.3 ns | 2,439.24 ns | 5,797.11 ns | 120,769.0 ns | 1.00 | 0.06 | 52.2461 | - | 328,032 B | 1.00 | -| FlatFiles_Write | 1000 | 258,895.1 ns | 5,051.42 ns | 4,725.10 ns | 258,890.0 ns | 2.12 | 0.10 | 113.2813 | - | 712,000 B | 2.17 | -| RecordParser_Write | 1000 | 754,218.4 ns | 14,792.11 ns | 29,880.79 ns | 749,527.8 ns | 6.17 | 0.37 | 173.8281 | 171.8750 | 1,076,624 B | 3.28 | - -The writer is **~30% faster** than FileHelpers and allocates **nothing** per batch, regardless of record count. - -Run benchmarks: +Make the model `partial` and implement `IFixedWidthModel` to have the generator write its +parser and writer instead; the generated overloads take no layout: `FixedWidth.ReadFile("people.txt")`. -```bash -dotnet run -c Release --project tests/Benchmarks/Benchmarks.csproj -- --filter "*Comparison*" -dotnet run -c Release --project tests/Benchmarks/Benchmarks.csproj -- --filter "*ReaderBenchmarks*" -``` +## Guide -Benchmark reports are written to `tests/Benchmarks/BenchmarkDotNet.Artifacts/results`. +- [Models and layouts](docs/guide/models.md) — attribute and generated models, fluent layouts for types you + cannot annotate, formatting options, column types, encodings and validation. +- [Reading](docs/guide/reading.md) — files and streams, async, parallel aggregation, options, errors, + untrusted input and single lines. +- [Writing](docs/guide/writing.md) — records and batches, synchronous and asynchronous, and formatting into + your own buffer. +- [Versioning](docs/guide/versioning.md) — what a minor version may change, and the `Infrastructure` namespace. +- [Benchmarks](docs/performance/benchmarks.md) — reading and writing against FileHelpers, FlatFiles and + RecordParser. [Live results](https://gabrielmarquezmatte.github.io/FixedWidthParser/dev/bench/). ## Python and C/C++ @@ -396,7 +128,7 @@ tests/FixedWidthParser.Generator.Tests/Source generator tests tests/FixedWidthParser.NativeSmoke/ C consumer of the native library tests/Benchmarks/ BenchmarkDotNet benchmarks tests/AotSmoke/ Fluent layout published with NativeAOT -docs/ Measurements behind design decisions +docs/ Usage guide and the measurements behind design decisions ``` [ARCHITECTURE.md](ARCHITECTURE.md) maps the code; [CONTRIBUTING.md](CONTRIBUTING.md) and diff --git a/docs/README.md b/docs/README.md index 09448c4..f95643c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,6 +16,11 @@ a constraint; the numbers that justified it live here, so they can be re-run and the classes it replaced. - [python-benchmarks.md](python-benchmarks.md) — the Python package reading a COTAHIST-shaped file against `pandas.read_fwf`. +- [performance/benchmarks.md](performance/benchmarks.md) — reading and writing against FileHelpers, + FlatFiles and RecordParser. -Usage documentation is the [README](../README.md); the shape of the code is +Usage documentation lives in [guide/](guide/), split out of the [README](../README.md) so the NuGet +landing page stays short: [models and layouts](guide/models.md), [reading](guide/reading.md), +[writing](guide/writing.md) and [versioning](guide/versioning.md). It is prose for a caller deciding how +to use the library; what each member does is in its `///` comments. The shape of the code is [ARCHITECTURE.md](../ARCHITECTURE.md). diff --git a/docs/guide/models.md b/docs/guide/models.md new file mode 100644 index 0000000..40904d9 --- /dev/null +++ b/docs/guide/models.md @@ -0,0 +1,105 @@ +# Models and layouts + +A layout maps byte ranges of a line to the members of a type. It comes from `[FixedColumn]` attributes, from the source generator, or from a fluent description in code; every read and write goes through one. + +## Defining a Model + +An attribute model needs a public parameterless constructor and mapped fields or properties; `FixedWidthLayout.FromAttributes()` turns it into a layout, built once per type by reflection: + +```csharp +using FixedWidthParser; + +public readonly record struct Person +{ + public Person() + { + Name = string.Empty; + } + + [FixedColumn(0, 10)] public string Name { get; init; } + [FixedColumn(10, 5)] public int Age { get; init; } + [FixedColumn(15, 10)] public double Salary { get; init; } +} +``` + +`start` is the 0-based offset and `length` is the column width, both **in bytes**. In UTF-8 a character such as `é` takes two bytes, so a layout that counts characters only matches a UTF-8 file whose text is ASCII; for accented text, read the file in the single-byte encoding it was written in (see [Encodings](#encodings)). + +For source generation, make the model `partial` and implement the marker interface: + +```csharp +using FixedWidthParser; + +public readonly partial record struct GeneratedPerson : IFixedWidthModel +{ + [FixedColumn(0, 10)] public string Name { get; init; } + [FixedColumn(10, 5)] public int Age { get; init; } + [FixedColumn(15, 10)] public double Salary { get; init; } +} +``` + +## Layouts without attributes + +For a type you cannot annotate, or a layout known only at run time, describe the columns in code. Each column binds through a getter (used to write) and a setter (used to read), so nothing is reflected over: + +```csharp +using FixedWidthParser; + +var layout = FixedWidthLayout.Build(b => b + .Factory(() => new Order()) + .Column("Id", 0, 6, (in Order o) => o.Id, (ref Order o, int v) => o.Id = v, + new FixedWidthColumnOptions { Alignment = Alignment.Right, Padding = '0' }) + .Column("Name", 6, 20, (in Order o) => o.Name, (ref Order o, string v) => o.Name = v) + .NullableColumn("Discount", 26, 10, (in Order o) => o.Discount, (ref Order o, decimal? v) => o.Discount = v) + .DecodedColumn("Date", 36, 8, (in Order o) => o.Date, (ref Order o, DateOnly v) => o.Date = v, + new FixedWidthColumnOptions { Format = "yyyyMMdd" }) + .ConvertedColumn("Total", 44, 12, (in Order o) => o.Total, (ref Order o, Money v) => o.Total = v, new MoneyConverter())); +``` + +| Method | Column type | +|---|---| +| `Column` | `string` | +| `Column` | a type that parses from and formats to UTF-8: numbers, `Guid`, … | +| `DecodedColumn` | a type that parses only from characters (decoded first) | +| `NullableColumn` | `T?` of a `Column` type; a blank column is `null` | +| `NullableDecodedColumn` | `T?` of a `DecodedColumn` type, such as `DateOnly?` | +| `ConvertedColumn` | any type, through an `IFixedWidthConverter` instance | +| `NullableConvertedColumn` | `T?` of a value type, through an `IFixedWidthConverter` instance | + +`FixedWidthColumnOptions` carries the same settings as `[FixedColumn]`, with the same defaults (see [Formatting Options](#formatting-options)). A reference-type model needs `Factory` to be read; a value type without one starts from `default`. Pass `null` as the getter or the setter to build a layout that only reads or only writes; using it the other way throws `InvalidOperationException` before any line is touched. The column name appears in error messages. + +## Formatting Options + +Each column can be tuned through named attribute arguments: + +```csharp +[FixedColumn(0, 8, Alignment = Alignment.Right, Padding = '0')] public int Id { get; init; } // "00000042" +[FixedColumn(8, 10, Format = "F2")] public double Amount { get; init; } // "1234.50 " +[FixedColumn(18, 5, Overflow = OverflowBehavior.Truncate)] public string Code { get; init; } +[FixedColumn(23, 5, TrimChar = '0', TrimMode = TrimMode.Leading)] public int Count { get; init; } +``` + +- **`Alignment`**: `Left` (default) or `Right`. +- **`Padding`**: fill byte when writing (default space; for example `'0'` for zero-padding). Must be ASCII. +- **`TrimChar`** / **`TrimMode`**: what to strip when parsing, and from which side. Must be ASCII. +- **`Format`**: format string for `ISpanFormattable` when writing, and the exact format for `DateTime`, `DateOnly`, `TimeOnly` and `DateTimeOffset` when parsing. +- **`Overflow`**: `Default`, `Truncate` or `Throw`. `Default` resolves per type: strings truncate, everything else throws. A truncated string keeps whole characters, so a multi-byte character is never split. +- **`Converter`**: a type implementing `IFixedWidthConverter`, used instead of the built-in parsing and formatting. + +## Column Types + +`string`, `double` and `float`, any `IUtf8SpanParsable` (parsed straight from bytes), any other `ISpanParsable` (decoded first), `T?` of any of them (a blank column is `null`), and anything with a `Converter`. Writing needs `string`, `ISpanFormattable` or a converter. + +## Encodings + +Column positions are bytes, so the encoding only decides how text inside a column is read and written. UTF-8 files with ASCII text and single-byte encodings (Latin-1, Windows-1252, …) keep one byte per character. UTF-16 and EBCDIC are not supported. + +```csharp +var options = new FixedWidthReaderOptions { Encoding = Encoding.Latin1 }; +var records = FixedWidth.ReadFile("legacy.txt", options); +``` + +On .NET, Windows-1252 and other code pages need `Encoding.RegisterProvider(CodePagesEncodingProvider.Instance)` first. + +## Validation + +Invalid layouts fail fast with an `InvalidOperationException` from `FixedWidthLayout.Build` or `FromAttributes`, or a generator diagnostic on generated models: negative `Start`, `Length < 1`, overlapping columns, a non-ASCII `Padding` or `TrimChar` (FWP012), an unsupported column type and a converter for the wrong type. Adjacent columns are valid. An attribute model that can be read but not written (or the reverse), such as one with a column type that only formats, gets a layout anyway; the side it cannot do throws on first use. diff --git a/docs/guide/reading.md b/docs/guide/reading.md new file mode 100644 index 0000000..d06dd32 --- /dev/null +++ b/docs/guide/reading.md @@ -0,0 +1,118 @@ +# Reading + +```csharp +using FixedWidthParser; + +var layout = FixedWidthLayout.FromAttributes(); + +foreach (var person in FixedWidth.ReadFile("people.txt", layout)) +{ + // ... +} + +foreach (var person in FixedWidth.ReadFile("people.txt")) +{ + // generated TryParse for each line +} +``` + +`Read(Stream, …, leaveOpen)` reads a stream once; `ReadFile` reopens the file on each enumeration. Reading is lazy and reuses a pooled buffer; each line is parsed in place, without a string per line. A layout is immutable: build it once and share it across reads and threads. + +Empty lines (including a trailing newline at end of file) are skipped: they are counted toward the line number but not yielded. Lines end at `\n`, with a preceding `\r` dropped, and a leading UTF-8 byte order mark is skipped. + +## Async + +```csharp +await foreach (var person in FixedWidth.ReadFileAsync("people.txt", layout)) +{ + // ... +} +``` + +`ReadAsync(Stream, leaveOpen)` mirrors the synchronous overload, and cancellation is honored via `WithCancellation`. A [`PipeReader`](https://learn.microsoft.com/dotnet/standard/io/pipelines) — a Kestrel request body, a socket — is read through `pipeReader.AsStream()`. + +## Parallel aggregation + +To fold a large file into one result — totals, counts, groups — read it on several threads: + +```csharp +public sealed class Payroll : IRecordAccumulator +{ + public double Total { get; private set; } + + public void Add(in Person person) + { + Total += person.Salary; + } + + public void Merge(Payroll following) + { + Total += following.Total; + } +} + +var payroll = await FixedWidth.AggregateFileAsync("people.txt", layout); +// GeneratedPayroll: the same accumulator over GeneratedPerson. +var generated = await FixedWidth.AggregateFileAsync("people.txt"); +``` + +The file is cut into byte ranges, each folded into its own accumulator on a worker thread, and the +accumulators are merged left to right in file order. `Add` must touch only its own instance; `Merge` +runs on one thread. Every line is parsed exactly once, and a failing line reports the same line number +as a sequential read. `FixedWidthReaderOptions.DegreeOfParallelism` caps the threads (`0`, the +default, means one per processor; `1` reads the file in one pass); other reads ignore it. `OnParseError`, when set, runs on +the worker threads. + +`AggregateAsync` does the same over a `Stream`, which is read once: one thread reads it into chunks +and the others parse. + +## Options + +Every read takes an optional `FixedWidthReaderOptions`: + +```csharp +var options = new FixedWidthReaderOptions +{ + FormatProvider = CultureInfo.GetCultureInfo("pt-BR"), + Encoding = Encoding.Latin1, + StringPool = new StringPool(), + BufferSize = 64 * 1024, + MaxLineLength = 4096, + OnParseError = ex => { log.Warning(ex.Message); return true; }, // skip the line +}; +var people = FixedWidth.ReadFile("people.txt", layout, options); +``` + +| Option | Default | Meaning | +|---|---|---| +| `FormatProvider` | invariant culture | Numbers and dates. | +| `Encoding` | UTF-8 | Text columns. Must be ASCII-compatible. | +| `StringPool` | none | Interns repeated string values. | +| `BufferSize` | 65,536 | Bytes read from the source at a time. | +| `MaxLineLength` | 1,048,576 | Longest line accepted; a longer one throws `InvalidDataException`. | +| `OnParseError` | throw | Return `true` to skip a line that fails to parse. | +| `DegreeOfParallelism` | processor count | Threads for `AggregateFileAsync`/`AggregateAsync`; `1` reads in one pass. | + +## Errors + +A line that fails to parse throws `FixedWidthParseException`, a `FormatException` that says where: + +```text +Line 2 could not be parsed into Person: column 'Age' (start 10, length 5) holds "XX ". +``` + +`LineNumber`, `ColumnName`, `ColumnStart`, `ColumnLength` and `RawValue` carry the same information. A line shorter than the layout reports how long it is and how long the layout needs it to be. The parser pays nothing for this on lines that parse: a failed line is parsed a second time to find the column. + +## Untrusted input + +A reader buffers a line until it finds its line break. `MaxLineLength` bounds that buffer, so a request body or file without line breaks throws `InvalidDataException` instead of growing the buffer until memory runs out. Set it close to your layout's line length when the source is untrusted. + +## Single Lines + +```csharp +FixedWidth.TryParse("John Doe 30 60000.00 "u8, layout, default, out Person person); + +FixedWidth.TryParse(lineBytes, new FixedWidthContext { FormatProvider = culture }, out var generated); +``` + +`FixedWidthContext` carries the format provider, the encoding and the string pool for one line; `default` means invariant culture, UTF-8 and no pool. diff --git a/docs/guide/versioning.md b/docs/guide/versioning.md new file mode 100644 index 0000000..f93d1f1 --- /dev/null +++ b/docs/guide/versioning.md @@ -0,0 +1,7 @@ +# Versioning + +The package follows [Semantic Versioning](https://semver.org/). A minor version may add types, members, options and enum values, but does not remove or change anything; every pull request checks the package, binary-compatibly, against the last release. + +- **Settings** arrive as new properties on the options objects (`FixedWidthReaderOptions`, `FixedWidthWriterOptions`, `FixedWidthColumnOptions`) or on `FixedWidthContext`, never as a new parameter on an existing method. +- **Interfaces you implement** — `IFixedWidthModel` (normally through the generator), `IFixedWidthConverter` and `IRecordAccumulator` — never gain a required member. Anything added comes with a default implementation (`static virtual` or a default interface method). +- **`FixedWidthParser.Infrastructure`** holds what is public only because generated code calls it (`FixedWidthRuntime`, `ColumnParseOptions`, `ColumnFormatOptions`) or because a method returns it so `foreach` allocates nothing (the `*RecordEnumerable`/`*RecordEnumerator` types and the `*LineParser` strategies). These types are hidden from IntelliSense and not meant for direct use, but they follow the same rules as everything else: code compiled against one 2.x version, generated code included, keeps working with every later 2.x version. A new setting for generated code is a new constructor overload on the options structs. diff --git a/docs/guide/writing.md b/docs/guide/writing.md new file mode 100644 index 0000000..6db549b --- /dev/null +++ b/docs/guide/writing.md @@ -0,0 +1,17 @@ +# Writing + +```csharp +var people = new[] +{ + new Person { Name = "John Doe", Age = 30, Salary = 60000 }, + new Person { Name = "Jane", Age = 28, Salary = 55000 }, +}; + +using var stream = File.Create("out.txt"); +FixedWidth.WriteMany(stream, people.AsSpan(), layout); +FixedWidth.WriteMany(stream, generatedPeople); // generated models need no layout +``` + +`Write`, `WriteMany` (span or enumerable) and their async versions write each record followed by `NewLine`, batched into one pooled buffer. `FixedWidthWriterOptions` sets the `FormatProvider`, the `Encoding` and the `NewLine` (default `Environment.NewLine`). `FixedWidth.TryFormat(in model, layout, Span, in FixedWidthContext, out int)`, or its generated overload without the layout, formats one line into your own buffer. + +Column alignment, padding, format strings and overflow are set per column: see [Formatting Options](models.md#formatting-options). diff --git a/docs/performance/benchmarks.md b/docs/performance/benchmarks.md new file mode 100644 index 0000000..bb1a933 --- /dev/null +++ b/docs/performance/benchmarks.md @@ -0,0 +1,81 @@ +# Benchmarks against other libraries + +## Machine and method + +AMD Ryzen 7 5700X (8C/16T), Windows 10 Pro 19045, .NET SDK 11.0.100-rc.2, `net10.0` (.NET 10.0.13), +Release. BenchmarkDotNet 0.15.8, default job, `MemoryDiagnoser`. Measured on 2.0 (`feat/v2-api`, October 2026). + +The model has three columns: a 10-byte string, a 5-byte `int` and a 10-byte `double`, 25 bytes a line. +The other libraries are [FileHelpers](https://www.filehelpers.net/) 3.5.2, +[FlatFiles](https://github.com/jehugaleahsa/FlatFiles) 5.0.4 and +[RecordParser](https://github.com/leandromoh/RecordParser) 2.3.0, with parallelism off. + +The comparison is not like-for-like, by design: + +- **Reading.** The others read text, so they are handed the already-decoded `string`; FixedWidthParser + reads the file's bytes from a `MemoryStream`, which is what a caller holding the file would give it. + The others are spared the UTF-8 decode. +- **Writing.** Everything targets `Stream.Null`, to measure formatting rather than I/O. The others write + text through a `StreamWriter`; FixedWidthParser writes bytes straight to the stream. + +Run them with: + +```bash +dotnet run -c Release --project tests/Benchmarks/Benchmarks.csproj -- --filter "*Comparison*" +``` + +Reports are written to `tests/Benchmarks/BenchmarkDotNet.Artifacts/results`; the +[live results](https://gabrielmarquezmatte.github.io/FixedWidthParser/dev/bench/) track the library's own suite +across commits. + +## Reading + +N records, summing one column. + +| Method | Count | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Gen1 | Allocated | Alloc Ratio | +|--------------------------- |------ |-----------:|----------:|----------:|------:|--------:|--------:|-------:|----------:|------------:| +| FixedWidthParser_Generated | 100 | 3.851 μs | 0.0131 μs | 0.0110 μs | 0.18 | 0.00 | 0.2365 | - | 3.89 KB | 0.09 | +| FixedWidthParser_Read | 100 | 5.268 μs | 0.1010 μs | 0.1994 μs | 0.24 | 0.01 | 0.2365 | - | 3.89 KB | 0.09 | +| RecordParser_Read | 100 | 7.956 μs | 0.0140 μs | 0.0117 μs | 0.37 | 0.00 | 0.2441 | - | 4.23 KB | 0.10 | +| FileHelpers_Read | 100 | 21.708 μs | 0.3205 μs | 0.2503 μs | 1.00 | 0.02 | 2.5330 | 0.0610 | 41.49 KB | 1.00 | +| FlatFiles_Read | 100 | 41.812 μs | 0.8312 μs | 0.6489 μs | 1.93 | 0.04 | 6.2866 | 0.1831 | 103.13 KB | 2.49 | +| | | | | | | | | | | | +| FixedWidthParser_Generated | 1000 | 40.220 μs | 0.7687 μs | 1.4248 μs | 0.18 | 0.01 | 2.3193 | - | 38.34 KB | 0.09 | +| FixedWidthParser_Read | 1000 | 50.214 μs | 0.2093 μs | 0.1748 μs | 0.23 | 0.00 | 2.3193 | - | 38.34 KB | 0.09 | +| RecordParser_Read | 1000 | 73.700 μs | 0.1759 μs | 0.1560 μs | 0.33 | 0.00 | 2.3193 | - | 38.69 KB | 0.10 | +| FileHelpers_Read | 1000 | 220.773 μs | 1.1137 μs | 0.9872 μs | 1.00 | 0.01 | 24.6582 | 5.8594 | 406.42 KB | 1.00 | +| FlatFiles_Read | 1000 | 414.094 μs | 5.4961 μs | 4.8721 μs | 1.88 | 0.02 | 58.1055 | 1.9531 | 953.2 KB | 2.35 | + +The generated model reads about 5.5 times faster than FileHelpers and the attribute layout about 4.2 times, +both with a tenth of its allocation, which is the strings the records hold. RecordParser, the fastest of +the others, takes 1.8 to 2.1 times as long as the generated model. + +## Writing + +N records written to `Stream.Null` through the attribute layout. + +| Method | Count | Mean | Error | StdDev | Median | Ratio | RatioSD | Gen0 | Gen1 | Allocated | Alloc Ratio | +|----------------------- |------ |--------------:|-------------:|-------------:|--------------:|-------:|--------:|--------:|--------:|----------:|------------:| +| FixedWidthParser_Write | 1 | 66.84 ns | 0.273 ns | 0.256 ns | 66.80 ns | 0.52 | 0.02 | - | - | - | 0.00 | +| FileHelpers_Write | 1 | 128.67 ns | 2.606 ns | 6.294 ns | 126.76 ns | 1.00 | 0.07 | 0.0215 | - | 360 B | 1.00 | +| FlatFiles_Write | 1 | 273.24 ns | 3.325 ns | 3.110 ns | 272.65 ns | 2.13 | 0.10 | 0.0424 | - | 712 B | 1.98 | +| RecordParser_Write | 1 | 15,581.73 ns | 550.422 ns | 1,570.385 ns | 14,975.60 ns | 121.37 | 13.44 | 4.0283 | 0.6714 | 65384 B | 181.62 | +| | | | | | | | | | | | | +| FixedWidthParser_Write | 100 | 5,159.48 ns | 8.964 ns | 8.385 ns | 5,156.54 ns | 0.43 | 0.00 | - | - | - | 0.00 | +| FileHelpers_Write | 100 | 11,897.36 ns | 77.800 ns | 64.966 ns | 11,890.04 ns | 1.00 | 0.01 | 1.9531 | - | 32832 B | 1.00 | +| FlatFiles_Write | 100 | 30,719.14 ns | 510.520 ns | 452.562 ns | 30,769.02 ns | 2.58 | 0.04 | 4.2114 | - | 71200 B | 2.17 | +| RecordParser_Write | 100 | 54,366.37 ns | 1,017.927 ns | 952.169 ns | 54,458.00 ns | 4.57 | 0.08 | 4.0283 | 0.6104 | 67696 B | 2.06 | +| | | | | | | | | | | | | +| FixedWidthParser_Write | 1000 | 57,478.75 ns | 65.891 ns | 61.634 ns | 57,493.25 ns | 0.47 | 0.00 | - | - | - | 0.00 | +| FileHelpers_Write | 1000 | 121,174.48 ns | 500.343 ns | 443.541 ns | 121,058.04 ns | 1.00 | 0.00 | 19.5313 | - | 328032 B | 1.00 | +| FlatFiles_Write | 1000 | 262,811.90 ns | 3,274.975 ns | 3,063.413 ns | 262,086.38 ns | 2.17 | 0.03 | 42.4805 | - | 712000 B | 2.17 | +| RecordParser_Write | 1000 | 346,479.69 ns | 5,197.862 ns | 4,862.083 ns | 344,529.69 ns | 2.86 | 0.04 | 53.7109 | 52.7344 | 880427 B | 2.68 | + +The writer takes about half of FileHelpers' time and allocates nothing, whatever the record count. + +## Not measured + +- Files on disk, cold or cached: every figure here is in memory. +- RecordParser and the parallel modes of FixedWidthParser with parallelism on; + [parallel-aggregate.md](../parallel-aggregate.md) covers FixedWidthParser's own. +- Other models: wider lines, `decimal` and date columns, non-ASCII text. From 62684827fc75de7e5c17185b7abc0e5be4889127 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 04:15:17 -0300 Subject: [PATCH 60/85] docs: bring the docs up to date with the code The changelog said the double fast path ran only for '.', and missed the comma-culture and StringPool speedups, C++ struct mapping and the generator's inherited-columns fix. ARCHITECTURE listed retired diagnostics, STYLEGUIDE a runtime constructor 2.0 no longer has and three of four twin paths, and CONTRIBUTING two of three warnings-as-errors projects and only part of CI, with no steps for the native, Python, C++ or AOT checks. The native README now names fixedwidth_arrow.h, SECURITY.md covers the native library and bindings, and the PR template asks for the changelog and PublicAPI entries. Inherited columns, Python's LayoutOptions, WriteOptions and Table members, and C++'s layout_for, at, table and TypedReader are documented; parallel-aggregate.md says its stream rows predate incremental merging. --- .github/pull_request_template.md | 4 +++- ARCHITECTURE.md | 3 ++- CHANGELOG.md | 14 ++++++++++--- CONTRIBUTING.md | 30 ++++++++++++++++++++------- SECURITY.md | 6 ++++-- STYLEGUIDE.md | 10 ++++++--- cpp/README.md | 7 ++++++- docs/guide/models.md | 4 ++++ docs/parallel-aggregate.md | 5 +++++ python/README.md | 13 +++++++++++- src/FixedWidthParser.Native/README.md | 5 +++-- 11 files changed, 80 insertions(+), 21 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 5d8f219..bfecb03 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -15,7 +15,9 @@ - [ ] Tests added/updated for the change - [ ] `dotnet build --configuration Release` is warning-clean - [ ] `dotnet test --configuration Release` passes -- [ ] Public API changes are documented (XML docs + README if relevant) +- [ ] Public API changes are documented (XML docs, plus `docs/guide/` if relevant) and recorded in `PublicAPI.Unshipped.txt` +- [ ] User-visible changes are in `CHANGELOG.md` +- [ ] Native, Python or C++ changes: the Python, C and C++ tests pass (see CONTRIBUTING.md) - [ ] For performance-sensitive changes: benchmark numbers included below ## Benchmark impact (if applicable) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 8b1de80..5b620f5 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -191,7 +191,8 @@ Studio can load it. It is an incremental generator: `Extract` turns each candida change a model's columns does not regenerate it. `Emit` validates the model — partial, top-level, a supported type for each column, layout, converter -type, overflow, ASCII padding — and reports diagnostics `FWP001`–`FWP012` instead of generating code +type, overflow, ASCII padding — and reports diagnostics `FWP001`–`FWP012` (`FWP007` and `FWP009` are +retired) instead of generating code for an invalid model. Otherwise it writes `TryParse` and `TryFormat` as straight-line code: one `FixedWidthRuntime` call per column. Each column's settings become two `static readonly` fields, a `ColumnParseOptions` and a `ColumnFormatOptions`, passed to the helpers by `in`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 8668a8b..845246f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,7 +69,7 @@ Windows-1252 after registering `CodePagesEncodingProvider`). thread count. Not on NuGet. - The `fixedwidth` Python package over it (NumPy views, pyarrow, pandas, polars), 52 times faster than `pandas.read_fwf` on a 1,000,000-line COTAHIST-shaped file, and a header-only C++23 wrapper - (`fixedwidth.hpp`). + (`fixedwidth.hpp`) that also maps structs to columns through `FixedWidthMapper`. - `ref struct` models on `Read`, `ReadFile`, `ReadAsync`, `ReadFileAsync`, `AggregateFileAsync` and `AggregateAsync`, generated and through a layout. The enumerators parse a line when `Current` is read, so reading it twice parses twice; with `OnParseError` set, each line is parsed once more to @@ -124,7 +124,12 @@ Windows-1252 after registering `CodePagesEncodingProvider`). scaled integer, about 5x faster, with the same bytes. A line of a string, an `int` and a `double` writes about 2x faster. - The generator targets `netstandard2.0`, so Visual Studio's .NET Framework compiler host can load it. -- The library and generator build with warnings as errors. +- `double`/`float` columns parse through csFastFloat under any culture whose decimal separator is one + ASCII character and whose negative sign is `-`, not only `'.'`: a pt-BR read is about 40% faster + ([docs/reader-performance.md](docs/reader-performance.md)). +- Reading with a `StringPool` checks a small per-thread cache before the pool, about halving the cost of + a pooled read. +- The library, generator and native library build with warnings as errors. ### Removed - **Breaking:** `FixedWidthParser`, `FixedWidthReader` and `FixedWidthWriter`. Pass @@ -151,7 +156,10 @@ Windows-1252 after registering `CodePagesEncodingProvider`). - `double`/`float` columns with a thousands separator under a `'.'` culture (`"1,234.56"`) failed instead of parsing. - `double`/`float` columns under a `','` culture could silently return a truncated value: `"1.234,50"` - became `1.0`. The fast path is now trusted only for `'.'`, and only when it consumes the whole field. + became `1.0`. The fast path is now trusted only when it consumes the whole field. +- A generated model skipped `[FixedColumn]` members declared on a base class: it left them at their + default when reading and wrote filler over their columns. Inherited columns are now generated as + reflection maps them, with an override or a `new` member replacing the one it hides. - The decimal-separator cache thrashed when two format providers alternated. ## [1.1.1] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b143725..144eeff 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ rewrites — see [ARCHITECTURE.md](ARCHITECTURE.md) for the shape of the codebas ## Build expectations -- **Warnings are errors in `src/`.** Both library projects set `TreatWarningsAsErrors`, under a +- **Warnings are errors in `src/`.** The library, generator and native projects set `TreatWarningsAsErrors`, under a curated `AnalysisMode=All` analyzer set (NetAnalyzers, Meziantou, SonarAnalyzer, Roslynator and others, in `Directory.Build.props`). A PR that doesn't build clean locally won't build clean in CI either — run a full build before pushing: @@ -20,8 +20,8 @@ rewrites — see [ARCHITECTURE.md](ARCHITECTURE.md) for the shape of the codebas public, update `src/FixedWidthParser/PublicAPI.Unshipped.txt`; the IDE's code fix writes the line for you. At a release, the unshipped entries move to `PublicAPI.Shipped.txt`. -- **Minor versions don't break callers.** The policy is in the README's - [Versioning](docs/guide/versioning.md) section; in practice: +- **Minor versions don't break callers.** The policy is in + [docs/guide/versioning.md](docs/guide/versioning.md); in practice: - `IFixedWidthModel` and `IFixedWidthConverter` are implemented by users, so a minor version never adds a member to them. New information reaches them as a new property on `FixedWidthContext`. @@ -43,8 +43,22 @@ rewrites — see [ARCHITECTURE.md](ARCHITECTURE.md) for the shape of the codebas Codecov from the Linux leg. A change to one path needs the same change, and a parity test, on its twin: the runtime and - generated paths, the sync and async readers, and the file and stream aggregators - ([STYLEGUIDE.md § Twins](STYLEGUIDE.md#twins)). Line-splitting code reads untrusted input; read + generated paths, the sync and async readers and writers, the sequential and parallel reads, and the + aggregator's model and sink loops ([STYLEGUIDE.md § Twins](STYLEGUIDE.md#twins)). + +- **The native library, its bindings and NativeAOT have their own checks.** A change under + `src/FixedWidthParser.Native`, `python/` or `cpp/` needs the Python, C and C++ tests; a change to the + fluent layout or anything it reaches needs the AOT smoke app to publish without an `IL2xxx`/`IL3xxx` + warning and to run: + + ```bash + python python/scripts/build_native.py # publishes the library into python/src/fixedwidth/_lib + pip install -e "python[dev,pandas,polars]" && pytest python/tests + cmake -S cpp -B build/cpp && cmake --build build/cpp && ctest --test-dir build/cpp --output-on-failure + dotnet publish tests/AotSmoke/AotSmoke.csproj -c Release -r win-x64 -p:IlcTreatWarningsAsErrors=true + ``` + + On Windows, run the CMake steps from a Visual Studio developer prompt, which puts `lib.exe` on the path. Line-splitting code reads untrusted input; read [STYLEGUIDE.md § Untrusted Input](STYLEGUIDE.md#untrusted-input) before touching it. - **Performance claims come with numbers.** This is a performance-sensitive library: avoid @@ -73,8 +87,10 @@ mood, lowercase summary, no trailing period. Common types in this repo: `feat`, - One focused change per PR — don't batch unrelated fixes into one commit or one PR. - If a change is user-visible (new API, behavior change, performance claim), add it to [CHANGELOG.md](CHANGELOG.md) and mention it in the PR description. -- CI builds and tests on Linux, Windows, and macOS, and runs CodeQL and the benchmark comparison — a - change that only builds on one OS isn't ready to merge. +- CI builds and tests on Linux, Windows, and macOS: the .NET suite, and the native library with its + Python, C and C++ bindings. It also validates the package against the last release, audits + dependencies, publishes and runs the NativeAOT smoke app, and runs CodeQL and the benchmark + comparison. A change that only builds on one OS isn't ready to merge. ## Reporting bugs / requesting features diff --git a/SECURITY.md b/SECURITY.md index 22a4fd5..64882f9 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,7 +4,8 @@ Only the latest published release of [`FixedWidthParser.NET`](https://www.nuget.org/packages/FixedWidthParser.NET) receives security fixes. Older releases are not patched retroactively; upgrade to the latest version -before reporting an issue. +before reporting an issue. The native library (`FixedWidthParser.Native`), the `fixedwidth` Python +package and the C++ header are built from the same source and reported the same way. ## Reporting a Vulnerability @@ -16,7 +17,8 @@ or by email to **gabrielandremarquez.matte@gmail.com**. Include, where possible: - A minimal reproduction: the model and the input that triggers it -- The reading or writing path involved (file or stream, sync, async or parallel, generated or runtime) +- The reading or writing path involved (file or stream, sync, async or parallel, generated or runtime, + or the native library through C, C++ or Python) - The impact you observed (crash, excessive memory/CPU, incorrect data, etc.) - The affected version(s) diff --git a/STYLEGUIDE.md b/STYLEGUIDE.md index 9bf5c16..2869508 100644 --- a/STYLEGUIDE.md +++ b/STYLEGUIDE.md @@ -258,7 +258,7 @@ Match the exception to whose fault it is: |---|---| | A line that does not fit the layout | `FixedWidthParseException`, naming the line, column and raw text | | A source that breaks a configured limit (`MaxLineLength`) | `InvalidDataException` | -| A model whose layout is invalid | `InvalidOperationException` from the runtime constructor, or a generator diagnostic | +| A model whose layout is invalid | `InvalidOperationException` from `FixedWidthLayout.Build` or `FromAttributes`, or a generator diagnostic | | A value too wide for its column when writing | `InvalidOperationException` naming the column | | A bad argument from the caller | `ArgumentException` and its subtypes | @@ -310,7 +310,7 @@ not a performance nit. ## Twins -Three sets of code paths in this library must stay behaviorally identical, and nothing in the compiler +Four sets of code paths in this library must stay behaviorally identical, and nothing in the compiler keeps them honest: - **Layout and generated.** `FixedWidthLayout` (from attributes or `Build`) and the generator's @@ -319,7 +319,11 @@ keeps them honest: - **Sequential and parallel reads.** `AggregateFileAsync` and `AggregateAsync` must return what a sequential read folded into one accumulator would, with the same line numbers in errors. Their per-line work shares `ParallelAggregator.TryAdd`. -- **Sync and async writers.** `WriteMany` and `WriteManyAsync` batch the same way. +- **Sync and async.** The readers (`RecordEnumerator` and `AsyncRecordEnumerator`) split and parse lines + the same way, and the writers (`WriteMany` and `WriteManyAsync`) batch the same way. +- **Model and sink loops.** `ParallelAggregator.AggregateChunk` folds models into an accumulator and + `SinkChunk` feeds the native layer's sinks; they are copies on purpose (the comment above them says + why). So: diff --git a/cpp/README.md b/cpp/README.md index 40d2795..e6ec580 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -56,7 +56,12 @@ null read leaves a plain field at its default. `make_field`'s last argument, `fw rest of `ColumnSpec`: `scale`, `format`, `trim_char`, `trim_mode`, `alignment`, `padding`, `overflow`. `TableView` keeps the library's columns and converts a row on access; a `std::string_view` field points -into it. An integer field narrower than `int64_t` that a value does not fit fails the read with +into it. `view[i]` is unchecked, `view.at(i)` returns an empty optional out of range, and `view.table()` is +the `fw::Table` underneath, with `lines_read()` and `lines_skipped()`. `fw::TypedReader::open_file` or +`open_memory` reads in batches: `next()` returns the next `TableView`, or an empty optional at the end. +`fw::layout_for()` returns the `fw::Layout` the bindings describe, for the untyped API. + +An integer field narrower than `int64_t` that a value does not fit fails the read with `FW_INVALID_ARGUMENT` instead of wrapping. ## CMake diff --git a/docs/guide/models.md b/docs/guide/models.md index 40904d9..bd46ff8 100644 --- a/docs/guide/models.md +++ b/docs/guide/models.md @@ -37,6 +37,10 @@ public readonly partial record struct GeneratedPerson : IFixedWidthModel Date: Sat, 24 Oct 2026 17:29:34 -0300 Subject: [PATCH 61/85] build: turn off package validation while 2.0 is in development 2.0 breaks the 1.x API on purpose, so validating against the 1.1.1 baseline only produced a 372-line suppression list regenerated on every break. Validation goes back on with a 2.0.0 baseline once 2.0.0 ships; CONTRIBUTING and the CI step name say so. --- .github/workflows/ci.yml | 2 +- CONTRIBUTING.md | 8 +- .../CompatibilitySuppressions.xml | 372 ------------------ src/FixedWidthParser/FixedWidthParser.csproj | 8 +- 4 files changed, 8 insertions(+), 382 deletions(-) delete mode 100644 src/FixedWidthParser/CompatibilitySuppressions.xml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3b075d..8a4f7b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: - name: Build (Release) run: dotnet build FixedWidthParser.slnx --configuration Release --no-restore -p:DeterministicSourcePaths=false - - name: Validate package against the last release + - name: Pack (validates against the baseline when package validation is on) if: matrix.os == 'ubuntu-latest' run: dotnet pack src/FixedWidthParser/FixedWidthParser.csproj --configuration Release --no-restore --output ./package-validation diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 144eeff..7d6fa88 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,10 +28,10 @@ rewrites — see [ARCHITECTURE.md](ARCHITECTURE.md) for the shape of the codebas - New inputs go on the options objects or a new overload, never as a new optional parameter on an existing method. - `dotnet pack` runs package validation against `PackageValidationBaselineVersion` in - `src/FixedWidthParser/FixedWidthParser.csproj`, and CI runs it on every pull request. A break that - belongs in a major version is listed in `CompatibilitySuppressions.xml` (generate it with - `dotnet pack -p:ApiCompatGenerateSuppressionFile=true`) and in the changelog. After each release, - move the baseline to the version just published and delete the suppressions file. + `src/FixedWidthParser/FixedWidthParser.csproj`, and CI runs it on every pull request. It is off + while a major version is in development, since that version breaks the last one on purpose; list + those breaks in the changelog. After each release, turn validation back on with the baseline set + to the version just published. - **Tests are required for behavior changes.** Run the suite before opening a PR: diff --git a/src/FixedWidthParser/CompatibilitySuppressions.xml b/src/FixedWidthParser/CompatibilitySuppressions.xml deleted file mode 100644 index 96f013b..0000000 --- a/src/FixedWidthParser/CompatibilitySuppressions.xml +++ /dev/null @@ -1,372 +0,0 @@ - - - - - CP0001 - T:FixedWidthParser.Attributes.Alignment - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Attributes.ColumnLayoutValidator - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Attributes.FixedColumnAttribute - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Attributes.OverflowBehavior - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.FixedWidthRuntime - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.FixedWidthUtf8 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Formatters.ColumnFormatOptions - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Formatters.IColumnFormatter`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.IUtf8FixedWidthModel`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Parsers.FixedWidthParser`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Parsers.Utf8FixedWidthParser`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Processors.ColumnParser`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Processors.ColumnParserRegistry - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Processors.ColumnValueParser`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Processors.Utf8ColumnParser`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Processors.Utf8ColumnParserRegistry - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Processors.Utf8ColumnValueParser`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.AsyncRecordEnumeratorCore`2 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.FixedWidthAsyncRecordEnumerable`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.FixedWidthByteReader`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.FixedWidthReader`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.FixedWidthRecordEnumerable`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.GeneratedFixedWidthAsyncRecordEnumerable`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.GeneratedFixedWidthRecordEnumerable`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.GeneratedLineParser`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.GeneratedUtf8FixedWidthAsyncRecordEnumerable`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.GeneratedUtf8FixedWidthPipeRecordEnumerable`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.GeneratedUtf8FixedWidthRecordEnumerable`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.GeneratedUtf8LineParser`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.ILineParser`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.IUtf8LineParser`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.ReflectionLineParser`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.ReflectionUtf8LineParser`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.Utf8AsyncRecordEnumeratorCore`2 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.Utf8FixedWidthAsyncRecordEnumerable`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.Utf8FixedWidthPipeRecordEnumerable`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.Utf8FixedWidthRecordEnumerable`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Readers.Utf8PipeRecordEnumeratorCore`2 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.RefAction`2 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.RefGetter`2 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Utf8FixedWidthRuntime - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0001 - T:FixedWidthParser.Writers.FixedWidthWriter`1 - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.FixedWidth.Read``1(System.IO.Stream,System.Text.Encoding,System.Boolean,System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,System.Int32) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.FixedWidth.Read``1(System.IO.TextReader,System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,System.Int32) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.FixedWidth.ReadAsync``1(System.IO.Stream,System.Text.Encoding,System.Boolean,System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,System.Int32) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.FixedWidth.ReadAsync``1(System.IO.TextReader,System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,System.Int32) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.FixedWidth.ReadFile``1(System.String,System.Text.Encoding,System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,System.Int32) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.FixedWidth.ReadFileAsync``1(System.String,System.Text.Encoding,System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,System.Int32) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.FixedWidth.TryParse``1(System.ReadOnlySpan{System.Char},System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,``0@) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0002 - M:FixedWidthParser.IFixedWidthModel`1.TryParse(System.ReadOnlySpan{System.Char},System.IFormatProvider,CommunityToolkit.HighPerformance.Buffers.StringPool,`0@) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0006 - M:FixedWidthParser.IFixedWidthModel`1.TryFormat(`0@,System.Span{System.Byte},FixedWidthParser.FixedWidthContext@,System.Int32@) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - CP0006 - M:FixedWidthParser.IFixedWidthModel`1.TryParse(System.ReadOnlySpan{System.Byte},FixedWidthParser.FixedWidthContext@,`0@) - lib/net10.0/FixedWidthParser.dll - lib/net10.0/FixedWidthParser.dll - true - - - PKV006 - net8.0 - - \ No newline at end of file diff --git a/src/FixedWidthParser/FixedWidthParser.csproj b/src/FixedWidthParser/FixedWidthParser.csproj index d4fae77..a5f09b4 100644 --- a/src/FixedWidthParser/FixedWidthParser.csproj +++ b/src/FixedWidthParser/FixedWidthParser.csproj @@ -21,12 +21,10 @@ $([System.String]::Copy('$(NoWarn)').Replace('RS0016', '').Replace('RS0037', ''));CS1591;S2743;CA1720;S3011 - + - true - 1.1.1 + false From 5241a199934f1a9be400da13e1ea2f6e1873ae70 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 17:30:10 -0300 Subject: [PATCH 62/85] perf: replace csFastFloat with ports of ExcelReader's FastDouble and FastDecimal double, float and decimal columns now parse through built-in fast paths that take the culture's decimal separator as a parameter, so comma cultures stay fast. Anything a fast path refuses (thousands separators, trailing text, too many digits) goes to the BCL over the decoded text, which decides the result; the BCL's UTF-8 parser would reject a space standing in for fr-FR's no-break-space group separator. float gets fast_float's binary32 fast path (a mantissa up to 2^24 and a scale up to 10, one rounding) rather than a rounded double, which can be one ulp off. decimal keeps the scale as written. CultureHelpers folds into FixedWidthRuntime, where one decode helper now serves the number fallback, the exact date/time parsers and TryParseDecoded. New tests check float and decimal against the BCL in invariant, de-DE and fr-FR. A 2M-line read of string, int, double, decimal and float columns drops from ~163 to ~122 ns/line. --- CHANGELOG.md | 9 +- README.md | 4 +- STYLEGUIDE.md | 2 +- docs/reader-performance.md | 4 +- src/FixedWidthParser/FixedWidthContext.cs | 2 +- src/FixedWidthParser/FixedWidthParser.csproj | 1 - .../Infrastructure/FastDecimal.cs | 64 +++ .../Infrastructure/FastDouble.cs | 376 ++++++++++++++++++ .../Infrastructure/FixedWidthRuntime.cs | 119 +++--- .../Processors/CultureHelpers.cs | 77 ---- tests/FixedWidthParser.Tests/CultureTests.cs | 60 ++- 11 files changed, 563 insertions(+), 155 deletions(-) create mode 100644 src/FixedWidthParser/Infrastructure/FastDecimal.cs create mode 100644 src/FixedWidthParser/Infrastructure/FastDouble.cs delete mode 100644 src/FixedWidthParser/Processors/CultureHelpers.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 845246f..12fd99d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -124,9 +124,12 @@ Windows-1252 after registering `CodePagesEncodingProvider`). scaled integer, about 5x faster, with the same bytes. A line of a string, an `int` and a `double` writes about 2x faster. - The generator targets `netstandard2.0`, so Visual Studio's .NET Framework compiler host can load it. -- `double`/`float` columns parse through csFastFloat under any culture whose decimal separator is one - ASCII character and whose negative sign is `-`, not only `'.'`: a pt-BR read is about 40% faster - ([docs/reader-performance.md](docs/reader-performance.md)). +- `double`, `float` and `decimal` columns parse through a built-in fast path (ExcelReader's `FastDouble` + and `FastDecimal`) under any culture whose decimal separator is one ASCII character and whose + negative sign is `-`, not only `'.'`: a pt-BR read is about 40% faster + ([docs/reader-performance.md](docs/reader-performance.md)). Anything the fast path refuses goes to + the BCL parser, which decides the result. +- **Breaking:** the csFastFloat dependency is gone. - Reading with a `StringPool` checks a small per-thread cache before the pool, about halving the cost of a pooled read. - The library, generator and native library build with warnings as errors. diff --git a/README.md b/README.md index dff2ad2..b9d6748 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Upgrading from 1.x? See [the 2.0 entry in the changelog](CHANGELOG.md#200). - **Bounded memory on untrusted input**: `MaxLineLength` stops a source without line breaks from growing the buffer. - **Writing** of single records and batches, synchronous and asynchronous, batched into one pooled buffer. - **Configurable formatting** per column: alignment, padding character, format string and explicit overflow policy. -- **Culture-aware** numbers and dates, including `double`/`float` via csFastFloat and any `IUtf8SpanParsable`/`ISpanParsable` type. +- **Culture-aware** numbers and dates, with a fast path for `double`, `float` and `decimal`, and any `IUtf8SpanParsable`/`ISpanParsable` type. - **Layout validation** when a layout is built or generated: negative `Start`, non-positive `Length`, overlapping columns and non-ASCII padding fail clearly. - **`ref struct` model support** on single-line parsing and formatting, on every reader and on parallel aggregation. - **Python and C/C++** through a NativeAOT library with a C ABI: columnar and Arrow reads, parallel reads and writes. See [Python and C/C++](#python-and-cc). @@ -36,7 +36,7 @@ Upgrading from 1.x? See [the 2.0 entry in the changelog](CHANGELOG.md#200). - `net10.0`. -Dependencies: [CommunityToolkit.HighPerformance](https://www.nuget.org/packages/CommunityToolkit.HighPerformance) (`StringPool`) and [csFastFloat](https://www.nuget.org/packages/csFastFloat) (fast `double`/`float` parsing). +Dependency: [CommunityToolkit.HighPerformance](https://www.nuget.org/packages/CommunityToolkit.HighPerformance) (`StringPool`). ## Installation diff --git a/STYLEGUIDE.md b/STYLEGUIDE.md index 2869508..3896934 100644 --- a/STYLEGUIDE.md +++ b/STYLEGUIDE.md @@ -200,7 +200,7 @@ because the reason lives outside the code and no amount of naming can bring it i ```csharp // OK: a dependency's behavior the code cannot express -// csFastFloat does not fail on trailing content; it stops and returns the prefix it parsed. +// The BCL's UTF-8 parser does not take a space for fr-FR's no-break-space group separator. // NOT OK: describes what the code does // Loop through each column and parse its value. diff --git a/docs/reader-performance.md b/docs/reader-performance.md index 5877878..fd1ded6 100644 --- a/docs/reader-performance.md +++ b/docs/reader-performance.md @@ -22,7 +22,9 @@ fast path against `double.TryParse` in de-DE, pt-BR and fr-FR. | `.` separator, invariant | 57.1 ns/line | 58.5 | | `,` separator, pt-BR | 98.4 ns/line | 58.5 | -One value: 42.3 ns decoded and BCL-parsed, 7.7 ns through csFastFloat. +One value: 42.3 ns decoded and BCL-parsed, 7.7 ns through csFastFloat. csFastFloat has since been +replaced by a port of ExcelReader's `FastDouble`, which keeps the same rule: a fast parse counts only +when it takes the whole field. ## `StringPool` diff --git a/src/FixedWidthParser/FixedWidthContext.cs b/src/FixedWidthParser/FixedWidthContext.cs index 4bf6d9f..0d8b1fd 100644 --- a/src/FixedWidthParser/FixedWidthContext.cs +++ b/src/FixedWidthParser/FixedWidthContext.cs @@ -32,7 +32,7 @@ public IFormatProvider FormatProvider /// /// The provider's decimal separator as one ASCII byte when its negative sign is "-", so a plain - /// number can be written or parsed without going through the provider; 0 otherwise. + /// number can be written without going through the provider; 0 otherwise. /// // ponytail: read once, so a NumberFormatInfo mutated after this context was built is not seen. internal byte DecimalSeparator { get => _formatProvider is null ? (byte)'.' : field; private init; } diff --git a/src/FixedWidthParser/FixedWidthParser.csproj b/src/FixedWidthParser/FixedWidthParser.csproj index a5f09b4..e5816e2 100644 --- a/src/FixedWidthParser/FixedWidthParser.csproj +++ b/src/FixedWidthParser/FixedWidthParser.csproj @@ -66,7 +66,6 @@ - runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/src/FixedWidthParser/Infrastructure/FastDecimal.cs b/src/FixedWidthParser/Infrastructure/FastDecimal.cs new file mode 100644 index 0000000..30c40de --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/FastDecimal.cs @@ -0,0 +1,64 @@ +using System.Runtime.CompilerServices; + +namespace FixedWidthParser.Infrastructure +{ + /// + /// ExcelReader's FastDecimal with the decimal separator as a parameter: [sign]digits[separator digits], + /// at most 19 digits, scale kept as written. for anything else, and the caller falls + /// back to the BCL. + /// + internal static class FastDecimal + { + private const int MaxMantissaDigits = 19; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool TryParse(ReadOnlySpan s, byte decimalSeparator, out decimal value) + { + value = 0; + if (s.IsEmpty) + { + return false; + } + int i = 0; + bool neg = s[0] == (byte)'-'; + if (neg || s[0] == (byte)'+') + { + i = 1; + } + + ulong mantissa = 0; + int integerStart = i; + for (; i < s.Length; i++) + { + uint d = (uint)(s[i] - (byte)'0'); + if (d > 9) + { + break; + } + mantissa = (mantissa * 10) + d; + } + int integerDigits = i - integerStart; + int scale = 0; + if (i < s.Length && s[i] == decimalSeparator) + { + int fractionStart = ++i; + for (; i < s.Length; i++) + { + uint d = (uint)(s[i] - (byte)'0'); + if (d > 9) + { + break; + } + mantissa = (mantissa * 10) + d; + } + scale = i - fractionStart; + } + if (i != s.Length || integerDigits + scale is 0 or > MaxMantissaDigits) + { + return false; + } + value = new decimal((int)mantissa, (int)(mantissa >> 32), 0, neg, (byte)scale); + return true; + } + } +} diff --git a/src/FixedWidthParser/Infrastructure/FastDouble.cs b/src/FixedWidthParser/Infrastructure/FastDouble.cs new file mode 100644 index 0000000..1f45008 --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/FastDouble.cs @@ -0,0 +1,376 @@ +using System.Buffers.Binary; +using System.Numerics; +using System.Runtime.CompilerServices; + +namespace FixedWidthParser.Infrastructure +{ + /// + /// ExcelReader's FastDouble (fast_float's algorithm) with the decimal separator as a parameter. + /// All or nothing: for anything it cannot parse exactly (thousands separators, + /// trailing text, too many digits, an exponent out of its range), and the caller falls back to the BCL. + /// + internal static class FastDouble + { + private const int MaxExponentDigits = 3; + private const int MaxMantissaDigits = 19; + private const int MinPowerOfFive = -64; + private const int MaxPowerOfFive = 64; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool TryParse(ReadOnlySpan s, byte decimalSeparator, out double value) + { + value = 0; + if (!TryScan(s, decimalSeparator, out ulong mantissa, out int scale, out bool neg)) + { + return false; + } + if (scale is < -22 or > 22 || mantissa > 1UL << 53) + { + return EiselLemire(mantissa, -scale, neg, out value); + } + + double result = mantissa; + if (scale > 0) + { + result /= Pow10(scale); + } + else if (scale < 0) + { + result *= Pow10(-scale); + } + value = neg ? -result : result; + return true; + } + + /// + /// fast_float's binary32 fast path only: a mantissa and a power of ten that are both exact floats, so + /// the one division or multiplication rounds once. Rounding the double result instead can be one ulp off. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool TryParse(ReadOnlySpan s, byte decimalSeparator, out float value) + { + value = 0; + if (!TryScan(s, decimalSeparator, out ulong mantissa, out int scale, out bool neg) || scale is < -10 or > 10 || mantissa > 1UL << 24) + { + return false; + } + float result = mantissa; + if (scale > 0) + { + result /= (float)Pow10(scale); + } + else if (scale < 0) + { + result *= (float)Pow10(-scale); + } + value = neg ? -result : result; + return true; + } + + // Reads [sign]digits[separator digits][e[sign]digits] as mantissa * 10^-scale; false for any other + // text, or more than 19 significant digits. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryScan(ReadOnlySpan s, byte decimalSeparator, out ulong mantissa, out int scale, out bool neg) + { + mantissa = 0; + scale = 0; + neg = false; + if (s.IsEmpty) + { + return false; + } + int i = 0; + neg = s[0] == (byte)'-'; + if (neg || s[0] == (byte)'+') + { + i = 1; + } + + int integerStart = i; + for (; i < s.Length; i++) + { + uint d = (uint)(s[i] - (byte)'0'); + if (d > 9) + { + break; + } + mantissa = (mantissa * 10) + d; + } + int integerDigits = i - integerStart; + if (i < s.Length && s[i] == decimalSeparator) + { + int fractionStart = ++i; + i = ReadFractionDigits(s, i, ref mantissa); + scale = i - fractionStart; + } + int allDigits = integerDigits + scale; + if (allDigits == 0 || HasTooManyDigits(s[integerStart..i], allDigits, decimalSeparator)) + { + return false; + } + int exponent = 0; + if (i < s.Length && (s[i] is not ((byte)'e' or (byte)'E') || !TryExponent(s[(i + 1)..], out exponent))) + { + return false; + } + + scale -= exponent; + return true; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool EiselLemire(ulong w, int q, bool neg, out double value) + { + value = 0; + if (q is < MinPowerOfFive or > MaxPowerOfFive) + { + return false; + } + if (w == 0) + { + value = neg ? -0.0 : 0.0; + return true; + } + int lz = BitOperations.LeadingZeroCount(w); + w <<= lz; + int index = 2 * (q - MinPowerOfFive); + ReadOnlySpan powers = PowersOfFive; + ulong hi = Math.BigMul(w, powers[index], out ulong lo); + const ulong PrecisionMask = ulong.MaxValue >> 55; + if ((hi & PrecisionMask) == PrecisionMask) + { + ulong hi2 = Math.BigMul(w, powers[index + 1], out _); + lo += hi2; + if (hi2 > lo) + { + hi++; + } + if (lo == ulong.MaxValue && q is < -27 or > 55) + { + return false; + } + } + int upperBit = (int)(hi >> 63); + int shift = upperBit + 9; + ulong m = hi >> shift; + int biasedExponent = ((217706 * q) >> 16) + 63 + upperBit - lz + 1023; + if (biasedExponent <= 0) + { + return false; + } + if (lo <= 1 && q is >= -4 and <= 23 && (m & 3) == 1 && (m << shift) == hi) + { + m &= ~1UL; + } + m += m & 1; + m >>= 1; + if (m >= 2UL << 52) + { + m = 1UL << 52; + biasedExponent++; + } + m &= ~(1UL << 52); + if (biasedExponent >= 0x7FF) + { + return false; + } + value = BitConverter.UInt64BitsToDouble(m | ((ulong)biasedExponent << 52) | (neg ? 1UL << 63 : 0)); + return true; + } + + // fast_float's digit loop: eight digits per step while the next eight bytes are all digits, then one + // four-digit step. Integer parts stay byte by byte, since most are too short for a step to pay off. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ReadFractionDigits(ReadOnlySpan s, int i, ref ulong mantissa) + { + while (s.Length - i >= 8) + { + ulong eight = BinaryPrimitives.ReadUInt64LittleEndian(s[i..]); + if ((((eight + 0x4646464646464646) | (eight - 0x3030303030303030)) & 0x8080808080808080) != 0) + { + break; + } + mantissa = (mantissa * 100_000_000) + ParseEightDigits(eight); + i += 8; + } + if (s.Length - i >= 4) + { + uint four = BinaryPrimitives.ReadUInt32LittleEndian(s[i..]); + if ((((four + 0x46464646) | (four - 0x30303030)) & 0x80808080) == 0) + { + mantissa = (mantissa * 10_000) + ParseFourDigits(four); + i += 4; + } + } + for (; i < s.Length; i++) + { + uint d = (uint)(s[i] - (byte)'0'); + if (d > 9) + { + break; + } + mantissa = (mantissa * 10) + d; + } + return i; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong ParseEightDigits(ulong eight) + { + eight -= 0x3030303030303030; + eight = (eight * 10) + (eight >> 8); + return (((eight & 0x000000FF000000FF) * 0x000F424000000064) + (((eight >> 16) & 0x000000FF000000FF) * 0x0000271000000001)) >> 32; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint ParseFourDigits(uint four) + { + four -= 0x30303030; + four = (four * 10) + (four >> 8); + return (((four & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; + } + + // A leading non-zero digit makes every digit significant, so only leading zeros need the count. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool HasTooManyDigits(ReadOnlySpan number, int allDigits, byte decimalSeparator) + { + if (allDigits <= MaxMantissaDigits) + { + return false; + } + if (number[0] is >= (byte)'1' and <= (byte)'9') + { + return true; + } + return SignificantDigits(number, decimalSeparator) > MaxMantissaDigits; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int SignificantDigits(ReadOnlySpan number, byte decimalSeparator) + { + int digits = 0; + foreach (ref readonly byte c in number) + { + if (c != decimalSeparator && (digits > 0 || c != (byte)'0')) + { + digits++; + } + } + return digits; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool TryExponent(ReadOnlySpan s, out int exponent) + { + exponent = 0; + if (s.IsEmpty) + { + return false; + } + int i = 0; + bool neg = s[0] == (byte)'-'; + if (neg || s[0] == (byte)'+') + { + i = 1; + } + if (i >= s.Length || s.Length - i > MaxExponentDigits) + { + return false; + } + int value = 0; + for (; i < s.Length; i++) + { + uint d = (uint)(s[i] - (byte)'0'); + if (d > 9) + { + return false; + } + value = (value * 10) + (int)d; + } + exponent = neg ? -value : value; + return true; + } + + private static double Pow10(int scale) + { + return PowersOfTen[scale]; + } + + private static ReadOnlySpan PowersOfTen => + [ + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22, + ]; + + // ponytail: fast_float's 128-bit truncated 5^q table cut to q in [-64, 64]; widen to its full + // [-342, 308] range if values outside 1e±64 need the fast path (they fall back to the BCL now). + private static ReadOnlySpan PowersOfFive => + [ + 0xA87FEA27A539E9A5, 0x3F2398D747B36224, 0xD29FE4B18E88640E, 0x8EEC7F0D19A03AAD, + 0x83A3EEEEF9153E89, 0x1953CF68300424AC, 0xA48CEAAAB75A8E2B, 0x5FA8C3423C052DD7, + 0xCDB02555653131B6, 0x3792F412CB06794D, 0x808E17555F3EBF11, 0xE2BBD88BBEE40BD0, + 0xA0B19D2AB70E6ED6, 0x5B6ACEAEAE9D0EC4, 0xC8DE047564D20A8B, 0xF245825A5A445275, + 0xFB158592BE068D2E, 0xEED6E2F0F0D56712, 0x9CED737BB6C4183D, 0x55464DD69685606B, + 0xC428D05AA4751E4C, 0xAA97E14C3C26B886, 0xF53304714D9265DF, 0xD53DD99F4B3066A8, + 0x993FE2C6D07B7FAB, 0xE546A8038EFE4029, 0xBF8FDB78849A5F96, 0xDE98520472BDD033, + 0xEF73D256A5C0F77C, 0x963E66858F6D4440, 0x95A8637627989AAD, 0xDDE7001379A44AA8, + 0xBB127C53B17EC159, 0x5560C018580D5D52, 0xE9D71B689DDE71AF, 0xAAB8F01E6E10B4A6, + 0x9226712162AB070D, 0xCAB3961304CA70E8, 0xB6B00D69BB55C8D1, 0x3D607B97C5FD0D22, + 0xE45C10C42A2B3B05, 0x8CB89A7DB77C506A, 0x8EB98A7A9A5B04E3, 0x77F3608E92ADB242, + 0xB267ED1940F1C61C, 0x55F038B237591ED3, 0xDF01E85F912E37A3, 0x6B6C46DEC52F6688, + 0x8B61313BBABCE2C6, 0x2323AC4B3B3DA015, 0xAE397D8AA96C1B77, 0xABEC975E0A0D081A, + 0xD9C7DCED53C72255, 0x96E7BD358C904A21, 0x881CEA14545C7575, 0x7E50D64177DA2E54, + 0xAA242499697392D2, 0xDDE50BD1D5D0B9E9, 0xD4AD2DBFC3D07787, 0x955E4EC64B44E864, + 0x84EC3C97DA624AB4, 0xBD5AF13BEF0B113E, 0xA6274BBDD0FADD61, 0xECB1AD8AEACDD58E, + 0xCFB11EAD453994BA, 0x67DE18EDA5814AF2, 0x81CEB32C4B43FCF4, 0x80EACF948770CED7, + 0xA2425FF75E14FC31, 0xA1258379A94D028D, 0xCAD2F7F5359A3B3E, 0x096EE45813A04330, + 0xFD87B5F28300CA0D, 0x8BCA9D6E188853FC, 0x9E74D1B791E07E48, 0x775EA264CF55347E, + 0xC612062576589DDA, 0x95364AFE032A819E, 0xF79687AED3EEC551, 0x3A83DDBD83F52205, + 0x9ABE14CD44753B52, 0xC4926A9672793543, 0xC16D9A0095928A27, 0x75B7053C0F178294, + 0xF1C90080BAF72CB1, 0x5324C68B12DD6339, 0x971DA05074DA7BEE, 0xD3F6FC16EBCA5E04, + 0xBCE5086492111AEA, 0x88F4BB1CA6BCF585, 0xEC1E4A7DB69561A5, 0x2B31E9E3D06C32E6, + 0x9392EE8E921D5D07, 0x3AFF322E62439FD0, 0xB877AA3236A4B449, 0x09BEFEB9FAD487C3, + 0xE69594BEC44DE15B, 0x4C2EBE687989A9B4, 0x901D7CF73AB0ACD9, 0x0F9D37014BF60A11, + 0xB424DC35095CD80F, 0x538484C19EF38C95, 0xE12E13424BB40E13, 0x2865A5F206B06FBA, + 0x8CBCCC096F5088CB, 0xF93F87B7442E45D4, 0xAFEBFF0BCB24AAFE, 0xF78F69A51539D749, + 0xDBE6FECEBDEDD5BE, 0xB573440E5A884D1C, 0x89705F4136B4A597, 0x31680A88F8953031, + 0xABCC77118461CEFC, 0xFDC20D2B36BA7C3E, 0xD6BF94D5E57A42BC, 0x3D32907604691B4D, + 0x8637BD05AF6C69B5, 0xA63F9A49C2C1B110, 0xA7C5AC471B478423, 0x0FCF80DC33721D54, + 0xD1B71758E219652B, 0xD3C36113404EA4A9, 0x83126E978D4FDF3B, 0x645A1CAC083126EA, + 0xA3D70A3D70A3D70A, 0x3D70A3D70A3D70A4, 0xCCCCCCCCCCCCCCCC, 0xCCCCCCCCCCCCCCCD, + 0x8000000000000000, 0x0000000000000000, 0xA000000000000000, 0x0000000000000000, + 0xC800000000000000, 0x0000000000000000, 0xFA00000000000000, 0x0000000000000000, + 0x9C40000000000000, 0x0000000000000000, 0xC350000000000000, 0x0000000000000000, + 0xF424000000000000, 0x0000000000000000, 0x9896800000000000, 0x0000000000000000, + 0xBEBC200000000000, 0x0000000000000000, 0xEE6B280000000000, 0x0000000000000000, + 0x9502F90000000000, 0x0000000000000000, 0xBA43B74000000000, 0x0000000000000000, + 0xE8D4A51000000000, 0x0000000000000000, 0x9184E72A00000000, 0x0000000000000000, + 0xB5E620F480000000, 0x0000000000000000, 0xE35FA931A0000000, 0x0000000000000000, + 0x8E1BC9BF04000000, 0x0000000000000000, 0xB1A2BC2EC5000000, 0x0000000000000000, + 0xDE0B6B3A76400000, 0x0000000000000000, 0x8AC7230489E80000, 0x0000000000000000, + 0xAD78EBC5AC620000, 0x0000000000000000, 0xD8D726B7177A8000, 0x0000000000000000, + 0x878678326EAC9000, 0x0000000000000000, 0xA968163F0A57B400, 0x0000000000000000, + 0xD3C21BCECCEDA100, 0x0000000000000000, 0x84595161401484A0, 0x0000000000000000, + 0xA56FA5B99019A5C8, 0x0000000000000000, 0xCECB8F27F4200F3A, 0x0000000000000000, + 0x813F3978F8940984, 0x4000000000000000, 0xA18F07D736B90BE5, 0x5000000000000000, + 0xC9F2C9CD04674EDE, 0xA400000000000000, 0xFC6F7C4045812296, 0x4D00000000000000, + 0x9DC5ADA82B70B59D, 0xF020000000000000, 0xC5371912364CE305, 0x6C28000000000000, + 0xF684DF56C3E01BC6, 0xC732000000000000, 0x9A130B963A6C115C, 0x3C7F400000000000, + 0xC097CE7BC90715B3, 0x4B9F100000000000, 0xF0BDC21ABB48DB20, 0x1E86D40000000000, + 0x96769950B50D88F4, 0x1314448000000000, 0xBC143FA4E250EB31, 0x17D955A000000000, + 0xEB194F8E1AE525FD, 0x5DCFAB0800000000, 0x92EFD1B8D0CF37BE, 0x5AA1CAE500000000, + 0xB7ABC627050305AD, 0xF14A3D9E40000000, 0xE596B7B0C643C719, 0x6D9CCD05D0000000, + 0x8F7E32CE7BEA5C6F, 0xE4820023A2000000, 0xB35DBF821AE4F38B, 0xDDA2802C8A800000, + 0xE0352F62A19E306E, 0xD50B2037AD200000, 0x8C213D9DA502DE45, 0x4526F422CC340000, + 0xAF298D050E4395D6, 0x9670B12B7F410000, 0xDAF3F04651D47B4C, 0x3C0CDD765F114000, + 0x88D8762BF324CD0F, 0xA5880A69FB6AC800, 0xAB0E93B6EFEE0053, 0x8EEA0D047A457A00, + 0xD5D238A4ABE98068, 0x72A4904598D6D880, 0x85A36366EB71F041, 0x47A6DA2B7F864750, + 0xA70C3C40A64E6C51, 0x999090B65F67D924, 0xD0CF4B50CFE20765, 0xFFF4B4E3F741CF6D, + 0x82818F1281ED449F, 0xBFF8F10E7A8921A4, 0xA321F2D7226895C7, 0xAFF72D52192B6A0D, + 0xCBEA6F8CEB02BB39, 0x9BF4F8A69F764490, 0xFEE50B7025C36A08, 0x02F236D04753D5B4, + 0x9F4F2726179A2245, 0x01D762422C946590, 0xC722F0EF9D80AAD6, 0x424D3AD2B7B97EF5, + 0xF8EBAD2B84E0D58B, 0xD2E0898765A7DEB2, 0x9B934C3B330C8577, 0x63CC55F49F88EB2F, + 0xC2781F49FFCFA6D5, 0x3CBF6B71C76B25FB + ]; + } +} diff --git a/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs index 6806f58..1e05ea8 100644 --- a/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs +++ b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs @@ -5,8 +5,8 @@ using System.Globalization; using System.Runtime.CompilerServices; using System.Text.Unicode; +using System.Numerics; using System.Text; -using FixedWidthParser.Processors; namespace FixedWidthParser.Infrastructure { @@ -47,18 +47,34 @@ public static string String(ReadOnlySpan column, in FixedWidthContext cont return context.StringPool is { } pool ? Utf8StringCache.GetOrAdd(pool, trimmed, context.Encoding) : context.Encoding.GetString(trimmed); } - /// Parses a column, honoring the provider's decimal separator. - [MethodImpl(MethodImplOptions.AggressiveInlining)] + // The fast parsers need the provider's decimal separator as one ASCII byte and "-" as its negative sign + // (FixedWidthContext.DecimalSeparator). What they refuse, thousands separators included, goes to the + // BCL over the decoded text: its UTF-8 parser does not take a space for a no-break-space group + // separator, as fr-FR writes one. Leading spaces are trimmed too, so a right-aligned column stays fast. + private const NumberStyles FloatStyles = NumberStyles.Float | NumberStyles.AllowThousands; + + /// Parses a column, honoring the provider's separators. public static bool TryDouble(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out double value) { - return CultureHelpers.TryParseDouble(column, in context, in options, out value); + var trimmed = TrimColumn(column, in options).TrimStart((byte)' '); + byte separator = context.DecimalSeparator; + if (separator != 0 && FastDouble.TryParse(trimmed, separator, out value)) + { + return true; + } + return ParseDecoded(trimmed, in context, null, static (text, _, provider, out result) => double.TryParse(text, FloatStyles, provider, out result), out value); } - /// Parses a column, honoring the provider's decimal separator. - [MethodImpl(MethodImplOptions.AggressiveInlining)] + /// Parses a column, honoring the provider's separators. public static bool TryFloat(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out float value) { - return CultureHelpers.TryParseFloat(column, in context, in options, out value); + var trimmed = TrimColumn(column, in options).TrimStart((byte)' '); + byte separator = context.DecimalSeparator; + if (separator != 0 && FastDouble.TryParse(trimmed, separator, out value)) + { + return true; + } + return ParseDecoded(trimmed, in context, null, static (text, _, provider, out result) => float.TryParse(text, FloatStyles, provider, out result), out value); } /// Parses an column straight from its bytes. @@ -66,24 +82,21 @@ public static bool TryFloat(ReadOnlySpan column, in FixedWidthContext cont public static bool TryParse(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, [MaybeNullWhen(false)] out TValue value) where TValue : IUtf8SpanParsable { - return TValue.TryParse(TrimColumn(column, in options), context.FormatProvider, out value); + var trimmed = TrimColumn(column, in options); + if (typeof(TValue) == typeof(decimal) && context.DecimalSeparator is var separator and not 0 + && FastDecimal.TryParse(trimmed.TrimStart((byte)' '), separator, out decimal fast)) + { + value = Unsafe.As(ref fast); + return true; + } + return TValue.TryParse(trimmed, context.FormatProvider, out value); } /// Parses an column that has no UTF-8 parser, decoding it with the context's encoding first. - [SkipLocalsInit] public static bool TryParseDecoded(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, [MaybeNullWhen(false)] out TValue value) where TValue : ISpanParsable { - Span stack = stackalloc char[StackBufferSize]; - var text = Decode(TrimColumn(column, in options), context.Encoding, stack, out char[]? rented); - try - { - return TValue.TryParse(text, context.FormatProvider, out value); - } - finally - { - Return(rented); - } + return ParseDecoded(TrimColumn(column, in options), in context, null, static (text, _, provider, [MaybeNullWhen(false)] out result) => TValue.TryParse(text, provider, out result), out value); } /// Parses a column through an instance. @@ -96,62 +109,40 @@ public static bool TryConvert( } /// Parses a column with exactly , decoding it first. - [SkipLocalsInit] public static bool TryDateTimeExact(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out DateTime value) { - Span stack = stackalloc char[StackBufferSize]; - var text = Decode(TrimColumn(column, in options), context.Encoding, stack, out char[]? rented); - try - { - return DateTime.TryParseExact(text, options.Format, context.FormatProvider, DateTimeStyles.None, out value); - } - finally - { - Return(rented); - } + return ParseDecoded(TrimColumn(column, in options), in context, options.Format, static (text, format, provider, out result) => DateTime.TryParseExact(text, format, provider, DateTimeStyles.None, out result), out value); } /// Parses a column with exactly , decoding it first. - [SkipLocalsInit] public static bool TryDateOnlyExact(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out DateOnly value) { - Span stack = stackalloc char[StackBufferSize]; - var text = Decode(TrimColumn(column, in options), context.Encoding, stack, out char[]? rented); - try - { - return DateOnly.TryParseExact(text, options.Format, context.FormatProvider, DateTimeStyles.None, out value); - } - finally - { - Return(rented); - } + return ParseDecoded(TrimColumn(column, in options), in context, options.Format, static (text, format, provider, out result) => DateOnly.TryParseExact(text, format, provider, DateTimeStyles.None, out result), out value); } /// Parses a column with exactly , decoding it first. - [SkipLocalsInit] public static bool TryTimeOnlyExact(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out TimeOnly value) { - Span stack = stackalloc char[StackBufferSize]; - var text = Decode(TrimColumn(column, in options), context.Encoding, stack, out char[]? rented); - try - { - return TimeOnly.TryParseExact(text, options.Format, context.FormatProvider, DateTimeStyles.None, out value); - } - finally - { - Return(rented); - } + return ParseDecoded(TrimColumn(column, in options), in context, options.Format, static (text, format, provider, out result) => TimeOnly.TryParseExact(text, format, provider, DateTimeStyles.None, out result), out value); } /// Parses a column with exactly , decoding it first. - [SkipLocalsInit] public static bool TryDateTimeOffsetExact(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out DateTimeOffset value) + { + return ParseDecoded(TrimColumn(column, in options), in context, options.Format, static (text, format, provider, out result) => DateTimeOffset.TryParseExact(text, format, provider, DateTimeStyles.None, out result), out value); + } + + private delegate bool CharParser(ReadOnlySpan text, string? format, IFormatProvider provider, [MaybeNullWhen(false)] out TValue value); + + // Decodes already-trimmed bytes with the context's encoding, on the stack when they fit. + [SkipLocalsInit] + private static bool ParseDecoded(ReadOnlySpan bytes, in FixedWidthContext context, string? format, CharParser parse, [MaybeNullWhen(false)] out TValue value) { Span stack = stackalloc char[StackBufferSize]; - var text = Decode(TrimColumn(column, in options), context.Encoding, stack, out char[]? rented); + var text = Decode(bytes, context.Encoding, stack, out char[]? rented); try { - return DateTimeOffset.TryParseExact(text, options.Format, context.FormatProvider, DateTimeStyles.None, out value); + return parse(text, format, context.FormatProvider, out value); } finally { @@ -370,9 +361,15 @@ private static void WriteText(ReadOnlySpan text, Span slice, Encodin Span bytes = rented is null ? stackalloc byte[StackBufferSize] : rented; try { - int written = ascii - ? NarrowAscii(text, bytes) - : encoding.GetBytes(text, bytes); + int written; + if (ascii) + { + Ascii.FromUtf16(text, bytes, out written); + } + else + { + written = encoding.GetBytes(text, bytes); + } options.WriteInto(bytes[..written], slice, columnName); } finally @@ -384,12 +381,6 @@ private static void WriteText(ReadOnlySpan text, Span slice, Encodin } } - private static int NarrowAscii(ReadOnlySpan text, Span bytes) - { - Ascii.FromUtf16(text, bytes, out int written); - return written; - } - // The longest run of whole characters, from the alignment side, whose encoding fits in width bytes. private static ReadOnlySpan FitText(ReadOnlySpan text, int width, Encoding encoding, Alignment alignment) { diff --git a/src/FixedWidthParser/Processors/CultureHelpers.cs b/src/FixedWidthParser/Processors/CultureHelpers.cs deleted file mode 100644 index 0db245b..0000000 --- a/src/FixedWidthParser/Processors/CultureHelpers.cs +++ /dev/null @@ -1,77 +0,0 @@ -using System.Buffers; -using System.Diagnostics.CodeAnalysis; -using System.Globalization; -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Text; -using FixedWidthParser.Infrastructure; -using csFastFloat; - -namespace FixedWidthParser.Processors -{ - /// - /// Culture-aware double/float parsing. csFastFloat is trusted only when the provider's - /// decimal separator is one ASCII character and its negative sign is "-" - /// (), and even then only after checking every byte of the - /// (trimmed) field was consumed: csFastFloat does not fail on trailing content it doesn't recognize, it - /// silently stops and returns the leading prefix it did parse. Any other provider, and a value using - /// thousands separators, falls back to the -aware BCL parser over the decoded text. - /// - internal static class CultureHelpers - { - /// Parses a (not yet trimmed) column as , honoring the context's provider. - public static bool TryParseDouble(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out double value) - { - // Also trim leading spaces: csFastFloat's characters_consumed does not reliably include - // skipped leading whitespace, so a right-aligned column would otherwise fail the - // full-consumption check below. - var trimmed = FixedWidthRuntime.TrimColumn(column, in options).TrimStart((byte)' '); - byte separator = context.DecimalSeparator; - if (separator != 0 - && FastDoubleParser.TryParseDouble(trimmed, out int consumed, out value, decimal_separator: separator) - && consumed == trimmed.Length) - { - return true; - } - return TryParseDecoded(trimmed, context.FormatProvider, context.Encoding, out value); - } - - /// Parses a (not yet trimmed) column as , honoring the context's provider. - public static bool TryParseFloat(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out float value) - { - var trimmed = FixedWidthRuntime.TrimColumn(column, in options).TrimStart((byte)' '); - byte separator = context.DecimalSeparator; - if (separator != 0 - && FastFloatParser.TryParseFloat(trimmed, out int consumed, out value, decimal_separator: separator) - && consumed == trimmed.Length) - { - return true; - } - return TryParseDecoded(trimmed, context.FormatProvider, context.Encoding, out value); - } - - [SkipLocalsInit] - private static bool TryParseDecoded(ReadOnlySpan field, IFormatProvider formatProvider, Encoding encoding, [MaybeNullWhen(false)] out TValue value) - where TValue : INumberBase - { - const NumberStyles styles = NumberStyles.Float | NumberStyles.AllowThousands; - int max = encoding.GetMaxCharCount(field.Length); - if (max <= 128) - { - Span chars = stackalloc char[128]; - int written = encoding.GetChars(field, chars); - return TValue.TryParse(chars[..written], styles, formatProvider, out value); - } - char[] rented = ArrayPool.Shared.Rent(max); - try - { - int written = encoding.GetChars(field, rented); - return TValue.TryParse(rented.AsSpan(0, written), styles, formatProvider, out value); - } - finally - { - ArrayPool.Shared.Return(rented); - } - } - } -} diff --git a/tests/FixedWidthParser.Tests/CultureTests.cs b/tests/FixedWidthParser.Tests/CultureTests.cs index 4f0acf0..cf2634f 100644 --- a/tests/FixedWidthParser.Tests/CultureTests.cs +++ b/tests/FixedWidthParser.Tests/CultureTests.cs @@ -47,8 +47,8 @@ public void Parse_FloatColumn_HonorsCommaCulture() Assert.Equal(3.14f, model.Value, 2); } - // csFastFloat's single decimal_separator override can't express a thousands separator; the - // BCL fallback accepts the value and still rejects real garbage. + // The fast path refuses a thousands separator; the BCL fallback accepts the value and still + // rejects real garbage. [Fact] public void Parse_DoubleColumn_DotCultureThousandsSeparator_ParsesFullValue() { @@ -56,7 +56,7 @@ public void Parse_DoubleColumn_DotCultureThousandsSeparator_ParsesFullValue() Assert.Equal(1234.50, model.Salary, 2); } - // Regression: "1.234,50" under de-DE used to come back as 1.0 from csFastFloat. + // Regression: "1.234,50" under de-DE used to come back as 1.0 from a prefix-only fast parse. [Fact] public void Parse_DoubleColumn_DeDeThousandsSeparator_ParsesFullValue() { @@ -88,6 +88,57 @@ public void Parse_Double_CommaCulture_MatchesTheBcl(string cultureName) } } + // The float fast path rounds once in binary32; going through a double could land one ulp off. + [Theory] + [InlineData("")] + [InlineData("de-DE")] + [InlineData("fr-FR")] + public void Parse_Float_MatchesTheBcl(string cultureName) + { + var culture = CultureInfo.GetCultureInfo(cultureName); + var context = CtxWith(culture); + var options = new Infrastructure.ColumnParseOptions(' ', TrimMode.Both, format: null); + string separator = culture.NumberFormat.NumberDecimalSeparator; + string[] edges = ["-0", "16777216", "16777217", $"1{separator}5e3", $"0{separator}1", $"3{separator}4028235E+38", "1e", "--1"]; + var random = new Random(42); + var shortDecimals = Enumerable.Range(0, 2000).Select(_ => (random.Next(-20_000_000, 20_000_000) / Math.Pow(10, random.Next(0, 11))).ToString("0.##########", culture)); + var roundTrips = Enumerable.Range(0, 2000).Select(_ => ((float)(random.NextDouble() * Math.Pow(10, random.Next(-8, 12)))).ToString("R", culture)); + + foreach (string text in edges.Concat(shortDecimals).Concat(roundTrips)) + { + bool expected = float.TryParse(text, NumberStyles.Float | NumberStyles.AllowThousands, culture, out float bcl); + bool actual = Infrastructure.FixedWidthRuntime.TryFloat(Bytes(text), in context, in options, out float parsed); + + Assert.True(expected == actual, $"'{text}': BCL {expected}, parser {actual}"); + Assert.True(!expected || bcl.Equals(parsed), $"'{text}': BCL {bcl:R}, parser {parsed:R}"); + } + } + + // The decimal fast path keeps the scale as written, as the BCL does: 1,50 stays 1,50. + [Theory] + [InlineData("")] + [InlineData("de-DE")] + [InlineData("fr-FR")] + public void Parse_Decimal_MatchesTheBcl(string cultureName) + { + var culture = CultureInfo.GetCultureInfo(cultureName); + var context = CtxWith(culture); + var options = new Infrastructure.ColumnParseOptions(' ', TrimMode.Both, format: null); + string separator = culture.NumberFormat.NumberDecimalSeparator; + string[] edges = ["-0", $"1{separator}50", $"{separator}5", $"5{separator}", "9999999999999999999", "99999999999999999999", $"0{separator}0000000000000000001", "1e3", "--1", "+7"]; + var random = new Random(42); + var values = Enumerable.Range(0, 2000).Select(_ => (random.NextInt64(-1_000_000_000_000, 1_000_000_000_000) / (decimal)Math.Pow(10, random.Next(0, 13))).ToString(culture)); + + foreach (string text in edges.Concat(values)) + { + bool expected = decimal.TryParse(text, NumberStyles.Number, culture, out decimal bcl); + bool actual = Infrastructure.FixedWidthRuntime.TryParse(Bytes(text), in context, in options, out decimal parsed); + + Assert.True(expected == actual, $"'{text}': BCL {expected}, parser {actual}"); + Assert.True(!expected || bcl.ToString(CultureInfo.InvariantCulture) == parsed.ToString(CultureInfo.InvariantCulture), $"'{text}': BCL {bcl}, parser {parsed}"); + } + } + [Fact] public void Parse_DoubleColumn_TrailingGarbageAfterNumber_Fails() { @@ -100,8 +151,7 @@ public void Parse_FloatColumn_TrailingGarbageAfterNumber_Fails() Assert.False(FixedWidth.TryParse("12x "u8, FixedWidthLayout.FromAttributes(), Ctx, out _)); } - // csFastFloat's characters_consumed does not consistently include skipped leading whitespace, - // so right-aligned columns must be trimmed before the fast path. + // The fast path takes no whitespace, so right-aligned columns must be trimmed before it. [Fact] public void Parse_DoubleColumn_RightAligned_DotCulture_Parses() { From 8da9f7a08172c0a0075757d9e2876251178b8310 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 17:30:27 -0300 Subject: [PATCH 63/85] refactor: collapse duplicated parse, format and read helpers - ColumnParserFactory: nine near-identical parse-then-assign builders become one Bind over a ColumnValueParser, fed the FixedWidthRuntime helpers as method groups; ForString and ForConverter stop forwarding. - ColumnFormatOptions.WriteInto reuses AlignInPlace instead of repeating the alignment and sign-leftmost logic. - LineBufferState.Fill(Stream) replaces the refill loop copied in the sync enumerator and the native table reader. - ResolveOverflow becomes one switch expression; the unused ParserBuilder.ComputeRequiredLineLength is gone. - FixedColumnAttribute docs no longer mention the removed IUtf8FixedWidthConverter. --- .../Reading/NativeTableReader.cs | 4 +- src/FixedWidthParser/FixedColumnAttribute.cs | 13 +- .../Formatters/FormatterBuilder.cs | 21 +-- .../Infrastructure/ColumnFormatOptions.cs | 28 +-- .../Infrastructure/RecordEnumerator.cs | 4 +- src/FixedWidthParser/Parsers/ParserBuilder.cs | 12 +- .../Processors/ColumnParser.cs | 6 + .../Processors/ColumnParserFactory.cs | 167 +++--------------- .../Readers/LineBufferState.cs | 8 + 9 files changed, 57 insertions(+), 206 deletions(-) diff --git a/src/FixedWidthParser.Native/Reading/NativeTableReader.cs b/src/FixedWidthParser.Native/Reading/NativeTableReader.cs index 78a5d26..044c375 100644 --- a/src/FixedWidthParser.Native/Reading/NativeTableReader.cs +++ b/src/FixedWidthParser.Native/Reading/NativeTableReader.cs @@ -87,9 +87,7 @@ private void Fill(ref TableSink sink) } if (status == LineStatus.NeedData) { - _lines.Compact(); - _lines.GrowIfFull(); - _lines.Advance(_stream.Read(_lines.Buffer.AsSpan(_lines.End))); + _lines.Fill(_stream); continue; } if (!sink.TryAdd(line, in _context)) diff --git a/src/FixedWidthParser/FixedColumnAttribute.cs b/src/FixedWidthParser/FixedColumnAttribute.cs index a541366..91a07af 100644 --- a/src/FixedWidthParser/FixedColumnAttribute.cs +++ b/src/FixedWidthParser/FixedColumnAttribute.cs @@ -39,10 +39,9 @@ public sealed class FixedColumnAttribute(int start, int length) : Attribute public char Padding { get; set; } = ' '; /// - /// Character trimmed from the end of the column when parsing (e.g. '0' for a zero-padded - /// numeric column written by another producer). Ignored for string columns' interning - /// identity in the same way as the default — it only changes which character is stripped - /// before the value is handed to the parser/converter. Default: space. + /// ASCII character trimmed from the column before it is parsed (e.g. '0' for a zero-padded numeric + /// column), from the ends picks, and written for a + /// value. Default: space. /// public char TrimChar { get; set; } = ' '; @@ -56,10 +55,8 @@ public sealed class FixedColumnAttribute(int start, int length) : Attribute public OverflowBehavior Overflow { get; set; } = OverflowBehavior.Default; /// - /// A type implementing IFixedWidthConverter<T> (char parser/writer) and/or - /// IUtf8FixedWidthConverter<T> (UTF-8 parser/writer), where T is this - /// column's member type. When set, it is used instead of the built-in ISpanParsable - /// fallback, for both parsing and writing. A single instance is created once per model type. + /// A type implementing , where T is this column's member + /// type, used instead of the built-in parsing and formatting. One instance serves every line. /// public Type? Converter { get; set; } } diff --git a/src/FixedWidthParser/Formatters/FormatterBuilder.cs b/src/FixedWidthParser/Formatters/FormatterBuilder.cs index 27180e1..8505b49 100644 --- a/src/FixedWidthParser/Formatters/FormatterBuilder.cs +++ b/src/FixedWidthParser/Formatters/FormatterBuilder.cs @@ -45,21 +45,14 @@ internal static ColumnFormatOptions ResolveOptions(string columnName, Type value throw new InvalidOperationException( $"OverflowBehavior.Truncate is not supported for non-string column '{columnName}' of type '{valueType.Name}'."); } - return new ColumnFormatOptions(alignment, padding, format, ResolveOverflow(overflow, valueType)); - } - - // Default overflow resolves per type: string truncates, everything else throws. - private static OverflowBehavior ResolveOverflow(OverflowBehavior overflow, Type valueType) - { - if (overflow != OverflowBehavior.Default) + // Default overflow resolves per type: string truncates, everything else throws. + var resolved = overflow switch { - return overflow; - } - if (valueType == typeof(string)) - { - return OverflowBehavior.Truncate; - } - return OverflowBehavior.Throw; + OverflowBehavior.Default when valueType == typeof(string) => OverflowBehavior.Truncate, + OverflowBehavior.Default => OverflowBehavior.Throw, + _ => overflow, + }; + return new ColumnFormatOptions(alignment, padding, format, resolved); } // Adapts a ColumnGetter into a ColumnGetter; only called after HasValue is confirmed. diff --git a/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs b/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs index de302eb..e052a74 100644 --- a/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs +++ b/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs @@ -48,8 +48,6 @@ internal void WriteInto(ReadOnlySpan content, Span slice, string col { int width = slice.Length; int length = content.Length; - byte padding = (byte)Padding; - if (length > width) { if (Overflow == OverflowBehavior.Throw) @@ -57,31 +55,11 @@ internal void WriteInto(ReadOnlySpan content, Span slice, string col throw new InvalidOperationException( $"Value \"{Encoding.UTF8.GetString(content)}\" ({length} bytes) exceeds the width {width} of column \"{columnName}\"."); } - if (Alignment == Alignment.Right) - { - content[(length - width)..].CopyTo(slice); - return; - } - content[..width].CopyTo(slice); - return; - } - - if (Alignment != Alignment.Right) - { - content.CopyTo(slice); - slice[length..].Fill(padding); + (Alignment == Alignment.Right ? content[(length - width)..] : content[..width]).CopyTo(slice); return; } - int pad = width - length; - if (Padding != ' ' && content.Length > 0 && (content[0] == (byte)'-' || content[0] == (byte)'+')) - { - slice[0] = content[0]; - slice[1..(pad + 1)].Fill(padding); - content[1..].CopyTo(slice[(pad + 1)..]); - return; - } - slice[..pad].Fill(padding); - content.CopyTo(slice[pad..]); + content.CopyTo(slice); + AlignInPlace(slice, length); } /// diff --git a/src/FixedWidthParser/Infrastructure/RecordEnumerator.cs b/src/FixedWidthParser/Infrastructure/RecordEnumerator.cs index 838f0dc..c6b088a 100644 --- a/src/FixedWidthParser/Infrastructure/RecordEnumerator.cs +++ b/src/FixedWidthParser/Infrastructure/RecordEnumerator.cs @@ -68,9 +68,7 @@ public bool MoveNext() return false; } - _lines.Compact(); - _lines.GrowIfFull(); - _lines.Advance(_stream.Read(_lines.Buffer.AsSpan(_lines.End))); + _lines.Fill(_stream); } } diff --git a/src/FixedWidthParser/Parsers/ParserBuilder.cs b/src/FixedWidthParser/Parsers/ParserBuilder.cs index 5ef734e..8cc4705 100644 --- a/src/FixedWidthParser/Parsers/ParserBuilder.cs +++ b/src/FixedWidthParser/Parsers/ParserBuilder.cs @@ -10,7 +10,7 @@ namespace FixedWidthParser.Parsers internal readonly record struct ColumnParserInfo(int Start, int Length, string Name, ColumnParser Parse) where TModel : allows ref struct; - /// Static-build logic for : the compiled model factory, the per-column setters and the required line length. + /// Static-build logic for : the compiled model factory and the per-column setters. internal static class ParserBuilder { public static Func BuildModelFactory() where TModel : new(), allows ref struct @@ -33,15 +33,5 @@ public static ColumnParserInfo[] BuildProcessors() where TModel }); return [.. processors]; } - - public static int ComputeRequiredLineLength(ReadOnlySpan> processors) where TModel : allows ref struct - { - int required = 0; - foreach (ref readonly var processor in processors) - { - required = Math.Max(required, processor.Start + processor.Length); - } - return required; - } } } diff --git a/src/FixedWidthParser/Processors/ColumnParser.cs b/src/FixedWidthParser/Processors/ColumnParser.cs index 23f1219..c6908b9 100644 --- a/src/FixedWidthParser/Processors/ColumnParser.cs +++ b/src/FixedWidthParser/Processors/ColumnParser.cs @@ -1,3 +1,6 @@ +using System.Diagnostics.CodeAnalysis; +using FixedWidthParser.Infrastructure; + namespace FixedWidthParser.Processors { /// @@ -6,4 +9,7 @@ namespace FixedWidthParser.Processors /// internal delegate bool ColumnParser(ReadOnlySpan column, in FixedWidthContext context, ref TModel model) where TModel : allows ref struct; + + /// Parses the already-sliced bytes of a single column: the shape of the parse helpers. + internal delegate bool ColumnValueParser(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, [MaybeNullWhen(false)] out TValue value); } diff --git a/src/FixedWidthParser/Processors/ColumnParserFactory.cs b/src/FixedWidthParser/Processors/ColumnParserFactory.cs index c04d17c..411e3b9 100644 --- a/src/FixedWidthParser/Processors/ColumnParserFactory.cs +++ b/src/FixedWidthParser/Processors/ColumnParserFactory.cs @@ -1,3 +1,4 @@ +using System.Diagnostics.CodeAnalysis; using System.Reflection; using FixedWidthParser.Infrastructure; @@ -42,7 +43,7 @@ private static ColumnParser CreateValue(Type valueType, Delegate } var converter = Activator.CreateInstance(converterType) ?? throw new InvalidOperationException($"Could not instantiate converter '{converterType}' for column '{memberName}'."); - return (ColumnParser)Method(nameof(BuildConverter), typeof(TModel), valueType, converterType) + return (ColumnParser)Method(nameof(ForConverter), typeof(TModel), valueType, converterType) .Invoke(null, [setter, converter, options])!; } @@ -53,15 +54,15 @@ private static ColumnParser CreateValue(Type valueType, Delegate } if (valueType == typeof(string)) { - return BuildString((ColumnSetter)setter, options); + return ForString((ColumnSetter)setter, options); } if (Implements(valueType, typeof(IUtf8SpanParsable<>))) { - return (ColumnParser)Method(nameof(BuildUtf8Parsable), typeof(TModel), valueType).Invoke(null, [setter, options])!; + return (ColumnParser)Method(nameof(ForUtf8Parsable), typeof(TModel), valueType).Invoke(null, [setter, options])!; } if (Implements(valueType, typeof(ISpanParsable<>))) { - return (ColumnParser)Method(nameof(BuildDecodedParsable), typeof(TModel), valueType).Invoke(null, [setter, options])!; + return (ColumnParser)Method(nameof(ForDecodedParsable), typeof(TModel), valueType).Invoke(null, [setter, options])!; } throw new InvalidOperationException( @@ -71,27 +72,34 @@ private static ColumnParser CreateValue(Type valueType, Delegate internal static ColumnParser ForString(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct { - return BuildString(setter, options); + return (column, in context, ref model) => + { + setter(ref model, FixedWidthRuntime.String(column, in context, in options)); + return true; + }; } internal static ColumnParser ForUtf8Parsable(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct where TValue : IUtf8SpanParsable { - return TryBuildSpecial(typeof(TValue), setter, options) ?? BuildUtf8Parsable(setter, options); + return TryBuildSpecial(typeof(TValue), setter, options) ?? Bind(setter, options, FixedWidthRuntime.TryParse); } internal static ColumnParser ForDecodedParsable(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct where TValue : ISpanParsable { - return TryBuildSpecial(typeof(TValue), setter, options) ?? BuildDecodedParsable(setter, options); + return TryBuildSpecial(typeof(TValue), setter, options) ?? Bind(setter, options, FixedWidthRuntime.TryParseDecoded); } - internal static ColumnParser ForConverter(ColumnSetter setter, IFixedWidthConverter converter, ColumnParseOptions options) + // TConverter is the concrete type on the attribute path, so its calls devirtualize there. + internal static ColumnParser ForConverter(ColumnSetter setter, TConverter converter, ColumnParseOptions options) where TModel : allows ref struct + where TConverter : IFixedWidthConverter { - return BuildConverter(setter, converter, options); + return Bind(setter, options, (ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions o, [MaybeNullWhen(false)] out TValue value) => + FixedWidthRuntime.TryConvert(column, in context, converter, in o, out value)); } internal static ColumnParser ForNullable( @@ -118,11 +126,11 @@ internal static ColumnParser ForNullable( } if (valueType == typeof(double)) { - return BuildDouble((ColumnSetter)setter, options); + return Bind((ColumnSetter)setter, options, FixedWidthRuntime.TryDouble); } if (valueType == typeof(float)) { - return BuildFloat((ColumnSetter)setter, options); + return Bind((ColumnSetter)setter, options, FixedWidthRuntime.TryFloat); } return null; } @@ -132,19 +140,19 @@ internal static ColumnParser ForNullable( { if (valueType == typeof(DateTime)) { - return BuildDateTime((ColumnSetter)setter, options); + return Bind((ColumnSetter)setter, options, FixedWidthRuntime.TryDateTimeExact); } if (valueType == typeof(DateOnly)) { - return BuildDateOnly((ColumnSetter)setter, options); + return Bind((ColumnSetter)setter, options, FixedWidthRuntime.TryDateOnlyExact); } if (valueType == typeof(TimeOnly)) { - return BuildTimeOnly((ColumnSetter)setter, options); + return Bind((ColumnSetter)setter, options, FixedWidthRuntime.TryTimeOnlyExact); } if (valueType == typeof(DateTimeOffset)) { - return BuildDateTimeOffset((ColumnSetter)setter, options); + return Bind((ColumnSetter)setter, options, FixedWidthRuntime.TryDateTimeOffsetExact); } return null; } @@ -185,137 +193,12 @@ private static ColumnParser WrapNullable(ColumnSett }; } - private static ColumnParser BuildUtf8Parsable(ColumnSetter setter, ColumnParseOptions options) - where TModel : allows ref struct - where TValue : IUtf8SpanParsable - { - return (column, in context, ref model) => - { - if (!FixedWidthRuntime.TryParse(column, in context, in options, out var value)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - - private static ColumnParser BuildDecodedParsable(ColumnSetter setter, ColumnParseOptions options) - where TModel : allows ref struct - where TValue : ISpanParsable - { - return (column, in context, ref model) => - { - if (!FixedWidthRuntime.TryParseDecoded(column, in context, in options, out var value)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - - private static ColumnParser BuildConverter(ColumnSetter setter, TConverter converter, ColumnParseOptions options) - where TModel : allows ref struct - where TConverter : IFixedWidthConverter - { - return (column, in context, ref model) => - { - if (!FixedWidthRuntime.TryConvert(column, in context, converter, in options, out var value)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - - private static ColumnParser BuildString(ColumnSetter setter, ColumnParseOptions options) - where TModel : allows ref struct - { - return (column, in context, ref model) => - { - setter(ref model, FixedWidthRuntime.String(column, in context, in options)); - return true; - }; - } - - private static ColumnParser BuildDouble(ColumnSetter setter, ColumnParseOptions options) - where TModel : allows ref struct - { - return (column, in context, ref model) => - { - if (!FixedWidthRuntime.TryDouble(column, in context, in options, out double value)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - - private static ColumnParser BuildFloat(ColumnSetter setter, ColumnParseOptions options) - where TModel : allows ref struct - { - return (column, in context, ref model) => - { - if (!FixedWidthRuntime.TryFloat(column, in context, in options, out float value)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - - private static ColumnParser BuildDateTime(ColumnSetter setter, ColumnParseOptions options) - where TModel : allows ref struct - { - return (column, in context, ref model) => - { - if (!FixedWidthRuntime.TryDateTimeExact(column, in context, in options, out DateTime value)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - - private static ColumnParser BuildDateOnly(ColumnSetter setter, ColumnParseOptions options) - where TModel : allows ref struct - { - return (column, in context, ref model) => - { - if (!FixedWidthRuntime.TryDateOnlyExact(column, in context, in options, out DateOnly value)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - - private static ColumnParser BuildTimeOnly(ColumnSetter setter, ColumnParseOptions options) - where TModel : allows ref struct - { - return (column, in context, ref model) => - { - if (!FixedWidthRuntime.TryTimeOnlyExact(column, in context, in options, out TimeOnly value)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - - private static ColumnParser BuildDateTimeOffset(ColumnSetter setter, ColumnParseOptions options) + private static ColumnParser Bind(ColumnSetter setter, ColumnParseOptions options, ColumnValueParser parse) where TModel : allows ref struct { return (column, in context, ref model) => { - if (!FixedWidthRuntime.TryDateTimeOffsetExact(column, in context, in options, out DateTimeOffset value)) + if (!parse(column, in context, in options, out var value)) { return false; } diff --git a/src/FixedWidthParser/Readers/LineBufferState.cs b/src/FixedWidthParser/Readers/LineBufferState.cs index 8ae4837..7a396f7 100644 --- a/src/FixedWidthParser/Readers/LineBufferState.cs +++ b/src/FixedWidthParser/Readers/LineBufferState.cs @@ -155,6 +155,14 @@ private bool TrySliceLine(int rawEnd, int nextStart, out ReadOnlySpan line return !line.IsEmpty; } + /// Makes room in the buffer and reads the next block of into it. + internal void Fill(Stream stream) + { + Compact(); + GrowIfFull(); + Advance(stream.Read(Buffer.AsSpan(End))); + } + internal void Compact() { if (Start <= 0) From 4fdcdcf480ac7fd0a62e66c00910120f369b149f Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 17:30:42 -0300 Subject: [PATCH 64/85] refactor(native): share column constants and the invalid-argument helper EpochDay and Pow10 were declared in both ColumnBuilder and TableFormatter; they move to NativeType. NativeLayout and WriteSettings each had the same Fail helper; NativeApi.InvalidArgument replaces both. GrowableNativeBuffer loses Last, CopyTo and ByteLength, which nothing called. --- src/FixedWidthParser.Native/GrowableNativeBuffer.cs | 8 -------- .../Layouts/NativeColumnSpec.cs | 6 ++++++ src/FixedWidthParser.Native/Layouts/NativeLayout.cs | 12 +++--------- src/FixedWidthParser.Native/NativeApi.cs | 7 +++++++ src/FixedWidthParser.Native/Reading/ColumnBuilder.cs | 10 ++++------ .../Writing/TableFormatter.cs | 7 ++----- src/FixedWidthParser.Native/Writing/WriteSettings.cs | 10 ++-------- 7 files changed, 24 insertions(+), 36 deletions(-) diff --git a/src/FixedWidthParser.Native/GrowableNativeBuffer.cs b/src/FixedWidthParser.Native/GrowableNativeBuffer.cs index ee6deee..81f7389 100644 --- a/src/FixedWidthParser.Native/GrowableNativeBuffer.cs +++ b/src/FixedWidthParser.Native/GrowableNativeBuffer.cs @@ -18,9 +18,6 @@ internal sealed unsafe class GrowableNativeBuffer : IDisposable where T : unm private int _capacity; internal int Count { get; private set; } - internal int ByteLength => checked(Count * sizeof(T)); - - internal ref T Last => ref _data[Count - 1]; internal ref T this[int index] => ref _data[index]; @@ -64,11 +61,6 @@ internal void AddRange(ReadOnlySpan values) Count += values.Length; } - internal void CopyTo(Span destination) - { - new ReadOnlySpan(_data, ByteLength).CopyTo(destination); - } - /// Hands the block to the caller, who frees it with . internal IntPtr Detach() { diff --git a/src/FixedWidthParser.Native/Layouts/NativeColumnSpec.cs b/src/FixedWidthParser.Native/Layouts/NativeColumnSpec.cs index b84d11e..fa428c1 100644 --- a/src/FixedWidthParser.Native/Layouts/NativeColumnSpec.cs +++ b/src/FixedWidthParser.Native/Layouts/NativeColumnSpec.cs @@ -14,6 +14,12 @@ internal static class NativeType internal const int Timestamp = 6; internal const int MaxScale = 18; + + /// The day DATE values count from, as Arrow's date32 does. + internal static readonly int EpochDay = new DateOnly(1970, 1, 1).DayNumber; + + /// 10^scale for every scale a column may have. + internal static readonly double[] Pow10 = [.. Enumerable.Range(0, MaxScale + 1).Select(p => Math.Pow(10, p))]; } /// fw_column_spec. diff --git a/src/FixedWidthParser.Native/Layouts/NativeLayout.cs b/src/FixedWidthParser.Native/Layouts/NativeLayout.cs index fdf0bb6..b78a043 100644 --- a/src/FixedWidthParser.Native/Layouts/NativeLayout.cs +++ b/src/FixedWidthParser.Native/Layouts/NativeLayout.cs @@ -61,7 +61,7 @@ internal static unsafe int TryCreate(ReadOnlySpan specs, Na error ??= ValidatePositions(columns); if (error is not null) { - return Fail(error); + return NativeApi.InvalidArgument(error); } return TryFinish(columns, settings, out layout); } @@ -73,11 +73,11 @@ private static int TryFinish(NativeColumn[] columns, NativeLayoutOptionsRaw sett int lineLength = settings.LineLength == 0 ? end : settings.LineLength; if (lineLength < end) { - return Fail($"line_length {settings.LineLength} is shorter than the columns, which end at {end}."); + return NativeApi.InvalidArgument($"line_length {settings.LineLength} is shorter than the columns, which end at {end}."); } if (!NativeOptions.TryAscii(settings.Filler, ' ', out char filler)) { - return Fail($"filler {settings.Filler} is not an ASCII character."); + return NativeApi.InvalidArgument($"filler {settings.Filler} is not an ASCII character."); } layout = new NativeLayout(columns, lineLength, (byte)filler); return NativeStatus.Ok; @@ -118,11 +118,5 @@ private static int TryFinish(NativeColumn[] columns, NativeLayoutOptionsRaw sett column = NativeColumnRules.Build(spec, name, format); return null; } - - private static int Fail(string message) - { - NativeApi.SetLastError(message); - return NativeStatus.InvalidArgument; - } } } diff --git a/src/FixedWidthParser.Native/NativeApi.cs b/src/FixedWidthParser.Native/NativeApi.cs index d0cf51a..de21f6d 100644 --- a/src/FixedWidthParser.Native/NativeApi.cs +++ b/src/FixedWidthParser.Native/NativeApi.cs @@ -30,6 +30,13 @@ internal static void SetLastError(string message) _parseColumn = -1; } + /// Records and returns . + internal static int InvalidArgument(string message) + { + SetLastError(message); + return NativeStatus.InvalidArgument; + } + /// Records a parse failure; is the layout index, -1 for a short line. internal static void SetParseError(FixedWidthParseException exception, int column) { diff --git a/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs index 1c51aa4..004e3f8 100644 --- a/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs +++ b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs @@ -18,8 +18,6 @@ internal sealed unsafe class ColumnBuilder : IDisposable private const int StackChars = 256; private const long DecimalLimit = 1_000_000_000_000_000_000; private const DateTimeStyles Utc = DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal; - private static readonly int EpochDay = new DateOnly(1970, 1, 1).DayNumber; - private static readonly double[] Pow10 = [.. Enumerable.Range(0, NativeType.MaxScale + 1).Select(p => Math.Pow(10, p))]; private readonly NativeColumn _column; private readonly long _maxTextBytes; @@ -239,7 +237,7 @@ private bool AppendFloat(ReadOnlySpan raw, in FixedWidthContext context, i var text = FixedWidthRuntime.TrimColumn(raw, in parse); if (_column.Scale > 0 && long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out long digits)) { - _doubles.Add(digits / Pow10[_column.Scale]); + _doubles.Add(digits / NativeType.Pow10[_column.Scale]); return true; } if (!FixedWidthRuntime.TryDouble(raw, in context, in parse, out double value)) @@ -248,7 +246,7 @@ private bool AppendFloat(ReadOnlySpan raw, in FixedWidthContext context, i } // Digits too many for a long still carry the implied decimals. bool impliedDecimals = _column.Scale > 0 && text.IndexOfAny(".,eE"u8) < 0; - _doubles.Add(impliedDecimals ? value / Pow10[_column.Scale] : value); + _doubles.Add(impliedDecimals ? value / NativeType.Pow10[_column.Scale] : value); return true; } @@ -262,7 +260,7 @@ private static bool TryDecimal(ReadOnlySpan text, int scale, out long unsc { return false; } - decimal scaled = value * (decimal)Pow10[scale]; + decimal scaled = value * (decimal)NativeType.Pow10[scale]; if (scaled != decimal.Truncate(scaled) || scaled <= -DecimalLimit || scaled >= DecimalLimit) { return false; @@ -280,7 +278,7 @@ private bool AppendTemporal(ReadOnlySpan raw, in FixedWidthContext context bool dateOk = exact ? FixedWidthRuntime.TryDateOnlyExact(raw, in context, in parse, out var date) : FixedWidthRuntime.TryParseDecoded(raw, in context, in parse, out date); if (dateOk) { - _ints.Add(date.DayNumber - EpochDay); + _ints.Add(date.DayNumber - NativeType.EpochDay); } return dateOk; case NativeType.Time: diff --git a/src/FixedWidthParser.Native/Writing/TableFormatter.cs b/src/FixedWidthParser.Native/Writing/TableFormatter.cs index dd19646..297664f 100644 --- a/src/FixedWidthParser.Native/Writing/TableFormatter.cs +++ b/src/FixedWidthParser.Native/Writing/TableFormatter.cs @@ -12,9 +12,6 @@ namespace FixedWidthParser.Native.Writing /// internal sealed unsafe class TableFormatter { - private static readonly int EpochDay = new DateOnly(1970, 1, 1).DayNumber; - private static readonly double[] Pow10 = [.. Enumerable.Range(0, NativeType.MaxScale + 1).Select(p => Math.Pow(10, p))]; - private readonly NativeColumn[] _columns; private readonly NativeColumnRaw* _data; private readonly byte _filler; @@ -98,7 +95,7 @@ private void FormatValue(NativeColumn column, NativeColumnRaw* raw, long row, Sp FormatFloat(column, ((double*)raw->Values)[row], slice); return; case NativeType.Date: - FixedWidthRuntime.FormatUtf8Value(DateOnly.FromDayNumber(EpochDay + ((int*)raw->Values)[row]), slice, in _context, options, column.Name); + FixedWidthRuntime.FormatUtf8Value(DateOnly.FromDayNumber(NativeType.EpochDay + ((int*)raw->Values)[row]), slice, in _context, options, column.Name); return; case NativeType.Time: FixedWidthRuntime.FormatUtf8Value(new TimeOnly(((long*)raw->Values)[row] * TimeSpan.TicksPerMicrosecond), slice, in _context, options, column.Name); @@ -120,7 +117,7 @@ private void FormatFloat(NativeColumn column, double value, Span slice) FixedWidthRuntime.FormatUtf8Value(value, slice, in _context, column.FormatOptions, column.Name); return; } - double scaled = Math.Round(value * Pow10[column.Scale]); + double scaled = Math.Round(value * NativeType.Pow10[column.Scale]); if (!(Math.Abs(scaled) < 9e18)) { throw new InvalidOperationException($"Value {value} of column \"{column.Name}\" does not fit a scaled 64-bit integer."); diff --git a/src/FixedWidthParser.Native/Writing/WriteSettings.cs b/src/FixedWidthParser.Native/Writing/WriteSettings.cs index e7a137f..007c59c 100644 --- a/src/FixedWidthParser.Native/Writing/WriteSettings.cs +++ b/src/FixedWidthParser.Native/Writing/WriteSettings.cs @@ -52,21 +52,15 @@ internal static unsafe int TryCreate(NativeWriteOptionsRaw* options, out WriteSe } if (raw.DegreeOfParallelism < 0 || raw.BatchRows < 0 || raw.NewLineLen < 0 || (raw.NewLine is null && raw.NewLineLen > 0)) { - return Fail("fw_write_options: degree_of_parallelism, batch_rows and newline_len must not be negative, and a newline_len needs a newline."); + return NativeApi.InvalidArgument("fw_write_options: degree_of_parallelism, batch_rows and newline_len must not be negative, and a newline_len needs a newline."); } if (!ReadSettings.TryEncoding(raw.Encoding, out var encoding)) { - return Fail($"fw_write_options: encoding {raw.Encoding} is not a code page this library knows."); + return NativeApi.InvalidArgument($"fw_write_options: encoding {raw.Encoding} is not a code page this library knows."); } byte[] newLine = raw.NewLine is null ? [(byte)'\n'] : new ReadOnlySpan(raw.NewLine, raw.NewLineLen).ToArray(); settings = new WriteSettings(encoding, newLine, raw.DegreeOfParallelism, raw.BatchRows); return NativeStatus.Ok; } - - private static int Fail(string message) - { - NativeApi.SetLastError(message); - return NativeStatus.InvalidArgument; - } } } From d230e3c680953f2e3e7ec3ed78048a0ad51b7a6b Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 17:30:57 -0300 Subject: [PATCH 65/85] refactor(generator): make ColumnInfo a record struct The hand-written Equals and GetHashCode over fourteen fields become the compiler's, with an IsExternalInit polyfill for netstandard2.0. ModelInfo keeps its own equality, since it leaves Location out. Drops a stale comment on ByteLiteral. --- .../FixedWidthModelGenerator.cs | 82 ++++--------------- 1 file changed, 14 insertions(+), 68 deletions(-) diff --git a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs index 5fd3cfe..e172848 100644 --- a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs +++ b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs @@ -692,7 +692,6 @@ private static void AppendColumnKindFormat(StringBuilder sb, string stmt, Column } } - // Trailing trim arguments for a runtime parse call, omitted for the default (space, trailing). private static string ByteLiteral(char value) { return "(byte)" + SymbolDisplay.FormatLiteral(value, true); @@ -729,73 +728,10 @@ private enum ColumnKind { String, Double, Float, Utf8Parsable, DecodedParsable, private enum WriteKind { String, Formattable, Utf8Formattable, Converter, Unsupported } - private readonly struct ColumnInfo( - string name, int start, int length, ColumnKind kind, string typeFqn, string? converterFqn, bool isNullable, - WriteKind writeKind, int alignment, char padding, string? format, int overflow, char trimChar, int trimMode) : IEquatable - { - public readonly string Name = name; - public readonly int Start = start; - public readonly int Length = length; - public readonly ColumnKind Kind = kind; - public readonly string TypeFqn = typeFqn; - public readonly string? ConverterFqn = converterFqn; - public readonly bool IsNullable = isNullable; - public readonly WriteKind WKind = writeKind; - public readonly int Alignment = alignment; - public readonly char Padding = padding; - public readonly string? Format = format; - public readonly int Overflow = overflow; - public readonly char TrimChar = trimChar; - public readonly int TrimMode = trimMode; - - public bool Equals(ColumnInfo other) - { - return Start == other.Start - && Length == other.Length - && Kind == other.Kind - && IsNullable == other.IsNullable - && WKind == other.WKind - && Alignment == other.Alignment - && Padding == other.Padding - && Overflow == other.Overflow - && TrimChar == other.TrimChar - && TrimMode == other.TrimMode - && string.Equals(Format, other.Format, StringComparison.Ordinal) - && string.Equals(Name, other.Name, StringComparison.Ordinal) - && string.Equals(TypeFqn, other.TypeFqn, StringComparison.Ordinal) - && string.Equals(ConverterFqn, other.ConverterFqn, StringComparison.Ordinal); - } - - [ExcludeFromCodeCoverage] - public override bool Equals(object? obj) - { - return obj is ColumnInfo other && Equals(other); - } - - [ExcludeFromCodeCoverage] - public override int GetHashCode() - { - unchecked - { - int hash = 17; - hash = hash * 31 + StringComparer.Ordinal.GetHashCode(Name); - hash = hash * 31 + Start; - hash = hash * 31 + Length; - hash = hash * 31 + (int)Kind; - hash = hash * 31 + StringComparer.Ordinal.GetHashCode(TypeFqn); - hash = hash * 31 + (ConverterFqn is null ? 0 : StringComparer.Ordinal.GetHashCode(ConverterFqn)); - hash = hash * 31 + (IsNullable ? 1 : 0); - hash = hash * 31 + (int)WKind; - hash = hash * 31 + Alignment; - hash = hash * 31 + Padding; - hash = hash * 31 + Overflow; - hash = hash * 31 + TrimChar; - hash = hash * 31 + TrimMode; - hash = hash * 31 + (Format is null ? 0 : StringComparer.Ordinal.GetHashCode(Format)); - return hash; - } - } - } + // Value equality, so the incremental pipeline caches correctly. + private readonly record struct ColumnInfo( + string Name, int Start, int Length, ColumnKind Kind, string TypeFqn, string? ConverterFqn, bool IsNullable, + WriteKind WKind, int Alignment, char Padding, string? Format, int Overflow, char TrimChar, int TrimMode); private sealed class ModelInfo(string? ns, string typeKeywords, string typeName, string fullyQualifiedName, EquatableArray columns, bool isPartial, bool isNested, Location location) : IEquatable @@ -896,3 +832,13 @@ public override int GetHashCode() } } } + +namespace System.Runtime.CompilerServices +{ + // netstandard2.0 lacks it; the records' init accessors need it. +#pragma warning disable S2094 // A marker type, empty by design. + internal static class IsExternalInit + { + } +#pragma warning restore S2094 +} From ecbe0cf665e1969bdee0d61ba2f5c18c6d9a04ab Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 17:49:06 -0300 Subject: [PATCH 66/85] perf: parse digit-only dates and integers straight from the bytes A date or time format made only of fixed-width digit fields (yyyy, MM, dd, HH, mm, ss) and the literals - . T / : is compiled once per column into a FastDate that reads the bytes directly, as ExcelReader's FastDate reads ISO 8601. It steps aside for a non-Gregorian calendar, and for "/" or ":" when the culture's separators differ. DateTime, DateOnly, TimeOnly and native TIMESTAMP columns use it: 85 -> 18 ns per value. int and long columns of plain digits, and the native INT64, implied- decimal FLOAT64 and DECIMAL columns, go through FastInteger, which reuses FastDouble's eight-digit step: 14 -> 9 ns per value. A leading "-" is taken only when the culture's negative sign is "-". Anything either refuses goes to the BCL. New tests check both against it in cultures where the fast paths must not apply (th-TH, de-DE, sv-SE). A single-threaded COTAHIST read through Python drops from 0.920 to 0.678 s; the parallel read barely moves (0.261 -> 0.250 s). --- CHANGELOG.md | 4 + docs/python-benchmarks.md | 37 +++-- .../Reading/ColumnBuilder.cs | 15 +- src/FixedWidthParser/FixedWidthContext.cs | 13 ++ .../Infrastructure/ColumnParseOptions.cs | 4 + .../Infrastructure/FastDate.cs | 130 ++++++++++++++++++ .../Infrastructure/FastDouble.cs | 5 +- .../Infrastructure/FastInteger.cs | 34 +++++ .../Infrastructure/FixedWidthRuntime.cs | 48 ++++++- tests/FixedWidthParser.Tests/CultureTests.cs | 64 +++++++++ 10 files changed, 331 insertions(+), 23 deletions(-) create mode 100644 src/FixedWidthParser/Infrastructure/FastDate.cs create mode 100644 src/FixedWidthParser/Infrastructure/FastInteger.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 12fd99d..974d591 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,6 +130,10 @@ Windows-1252 after registering `CodePagesEncodingProvider`). ([docs/reader-performance.md](docs/reader-performance.md)). Anything the fast path refuses goes to the BCL parser, which decides the result. - **Breaking:** the csFastFloat dependency is gone. +- Dates and times with a format made only of fixed-width digit fields (`yyyyMMdd`, `yyyy-MM-dd`, `HHmmss`, …) + and `int`/`long` columns of plain digits parse straight from the bytes, about 5x and 1.5x faster; any + other text still goes to the BCL. A single-threaded B3 COTAHIST read through Python is 26% faster + ([docs/python-benchmarks.md](docs/python-benchmarks.md)). - Reading with a `StringPool` checks a small per-thread cache before the pool, about halving the cost of a pooled read. - The library, generator and native library build with warnings as errors. diff --git a/docs/python-benchmarks.md b/docs/python-benchmarks.md index 43efc7d..a58cc0b 100644 --- a/docs/python-benchmarks.md +++ b/docs/python-benchmarks.md @@ -16,14 +16,29 @@ pyarrow 26.0.0, polars 2.0.0. `read_fwf` best of 1 run, the others best of 5. | Reader | Seconds | vs read_fwf | |---|---:|---:| -| pandas.read_fwf | 14.955 | 1.0x | -| polars str.slice (reference) | 0.306 | 49x | -| fw.read_pandas | 0.290 | 52x | -| fw.read_pandas, 1 thread | 0.940 | 16x | -| fw.read_polars | 0.273 | 55x | -| fw.read_arrow | 0.253 | 59x | -| fw.read (NumPy views) | 0.263 | 57x | -| fw.read, 1 thread | 0.947 | 16x | - -A single thread is already 16x faster than `read_fwf`; the default, one thread per processor, adds another 3.6x. -`fw.read_pandas` costs about 40 ms over `fw.read_arrow`, pyarrow's conversion to pandas. +| pandas.read_fwf | 15.908 | 1.0x | +| polars str.slice (reference) | 0.351 | 45x | +| fw.read_pandas | 0.293 | 54x | +| fw.read_pandas, 1 thread | 0.747 | 21x | +| fw.read_polars | 0.267 | 60x | +| fw.read_arrow | 0.239 | 67x | +| fw.read (NumPy views) | 0.250 | 64x | +| fw.read, 1 thread | 0.678 | 23x | + +A single thread is already 23x faster than `read_fwf`; the default, one thread per processor, adds another 2.7x. +`fw.read_pandas` costs about 50 ms over `fw.read_arrow`, pyarrow's conversion to pandas. + +## Fast paths for dates and integers + +The two `yyyyMMdd` date columns cost 85 ns each through `DateOnly.TryParseExact` on the decoded text, and the +sixteen integer and implied-decimal columns 12–14 ns each through `long.TryParse`. A format made only of +fixed-width digit fields (`yyyy MM dd HH mm ss` and `- . T / :`) is now compiled once per column and read straight +from the bytes (18 ns), and a digits-only integer goes through fast_float's eight-digit step (9 ns), each falling +back to the BCL for anything else. Same machine, same run order, before and after: + +| Reader | Before | After | +|---|---:|---:| +| fw.read, 1 thread | 0.920 s | 0.678 s | +| fw.read | 0.261 s | 0.250 s | + +The parallel read barely moved, so parsing is not what bounds it. diff --git a/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs index 004e3f8..8ff7528 100644 --- a/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs +++ b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs @@ -235,7 +235,7 @@ private bool Add(bool parsed, long value) private bool AppendFloat(ReadOnlySpan raw, in FixedWidthContext context, in ColumnParseOptions parse) { var text = FixedWidthRuntime.TrimColumn(raw, in parse); - if (_column.Scale > 0 && long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out long digits)) + if (_column.Scale > 0 && TryInteger(text, out long digits)) { _doubles.Add(digits / NativeType.Pow10[_column.Scale]); return true; @@ -250,9 +250,15 @@ private bool AppendFloat(ReadOnlySpan raw, in FixedWidthContext context, i return true; } + private static bool TryInteger(ReadOnlySpan text, out long value) + { + return FastInteger.TryParse(text.Trim((byte)' '), allowMinus: true, maxDigits: 18, out value) + || long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out value); + } + private static bool TryDecimal(ReadOnlySpan text, int scale, out long unscaled) { - if (long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out unscaled)) + if (TryInteger(text, out unscaled)) { return unscaled is > -DecimalLimit and < DecimalLimit; } @@ -285,7 +291,10 @@ private bool AppendTemporal(ReadOnlySpan raw, in FixedWidthContext context bool timeOk = exact ? FixedWidthRuntime.TryTimeOnlyExact(raw, in context, in parse, out var time) : FixedWidthRuntime.TryParseDecoded(raw, in context, in parse, out time); return Add(timeOk, time.Ticks / TimeSpan.TicksPerMicrosecond); default: - bool stampOk = TryTimestamp(FixedWidthRuntime.TrimColumn(raw, in parse), context.Encoding, parse.Format, out var stamp); + // A fast parse has no zone, which the slow path would read as UTC too. + var text = FixedWidthRuntime.TrimColumn(raw, in parse); + bool stampOk = (parse.FastDate is { HasDate: true } fast && fast.TryParse(text, in context, out var stamp)) + || TryTimestamp(text, context.Encoding, parse.Format, out stamp); return Add(stampOk, (stamp.Ticks - DateTime.UnixEpoch.Ticks) / TimeSpan.TicksPerMicrosecond); } } diff --git a/src/FixedWidthParser/FixedWidthContext.cs b/src/FixedWidthParser/FixedWidthContext.cs index 0d8b1fd..22d98bc 100644 --- a/src/FixedWidthParser/FixedWidthContext.cs +++ b/src/FixedWidthParser/FixedWidthContext.cs @@ -27,9 +27,22 @@ public IFormatProvider FormatProvider { _formatProvider = value; DecimalSeparator = PlainDecimalSeparator(value); + if (value is not null) + { + var dates = DateTimeFormatInfo.GetInstance(value); + GregorianDates = dates.Calendar is GregorianCalendar; + StandardDateSeparators = string.Equals(dates.DateSeparator, "/", StringComparison.Ordinal) + && string.Equals(dates.TimeSeparator, ":", StringComparison.Ordinal); + } } } + /// True when the provider's calendar is Gregorian, so a date's digits are its year, month and day. + internal bool GregorianDates { get => _formatProvider is null || field; private init; } + + /// True when the provider's date and time separators are "/" and ":". + internal bool StandardDateSeparators { get => _formatProvider is null || field; private init; } + /// /// The provider's decimal separator as one ASCII byte when its negative sign is "-", so a plain /// number can be written without going through the provider; 0 otherwise. diff --git a/src/FixedWidthParser/Infrastructure/ColumnParseOptions.cs b/src/FixedWidthParser/Infrastructure/ColumnParseOptions.cs index 9a1c635..957e4d1 100644 --- a/src/FixedWidthParser/Infrastructure/ColumnParseOptions.cs +++ b/src/FixedWidthParser/Infrastructure/ColumnParseOptions.cs @@ -22,6 +22,7 @@ public ColumnParseOptions(char trimChar, TrimMode trimMode, string? format) TrimChar = trimChar; TrimMode = trimMode; Format = format; + FastDate = FastDate.TryCreate(format); } /// The ASCII character trimmed from the column before it is parsed, and that a value is written with. @@ -34,5 +35,8 @@ public ColumnParseOptions(char trimChar, TrimMode trimMode, string? format) public string? Format { get; } internal byte TrimByte => (byte)TrimChar; + + /// compiled for , or when it cannot be. + internal FastDate? FastDate { get; } } } diff --git a/src/FixedWidthParser/Infrastructure/FastDate.cs b/src/FixedWidthParser/Infrastructure/FastDate.cs new file mode 100644 index 0000000..ea69742 --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/FastDate.cs @@ -0,0 +1,130 @@ +namespace FixedWidthParser.Infrastructure +{ + /// + /// A date or time format made only of fixed-width digit fields (yyyy, MM, dd, HH, + /// mm, ss) and the literals - . T / :, read straight from the bytes the way ExcelReader's + /// FastDate reads ISO 8601. compiles it once per column and returns + /// for any other format; returns for + /// anything it cannot read exactly as TryParseExact would, and the caller falls back to it. + /// + internal sealed class FastDate + { + private readonly string _format; + private readonly int _year = -1; + private readonly int _month = -1; + private readonly int _day = -1; + private readonly int _hour = -1; + private readonly int _minute = -1; + private readonly int _second = -1; + // "/" and ":" stand for the provider's date and time separators, not for themselves. + private readonly bool _usesSeparators; + private readonly bool _valid; + + private FastDate(string format) + { + _format = format; + for (int i = 0; i < format.Length;) + { + char c = format[i]; + int run = 1; + while (i + run < format.Length && format[i + run] == c) + { + run++; + } + switch (c, run) + { + case ('y', 4) when _year < 0: + _year = i; + break; + case ('M', 2) when _month < 0: + _month = i; + break; + case ('d', 2) when _day < 0: + _day = i; + break; + case ('H', 2) when _hour < 0: + _hour = i; + break; + case ('m', 2) when _minute < 0: + _minute = i; + break; + case ('s', 2) when _second < 0: + _second = i; + break; + case ('-' or '.' or 'T', _): + break; + case ('/' or ':', _): + _usesSeparators = true; + break; + default: + return; + } + i += run; + } + // A date needs all of year, month and day, and a time its hour: TryParseExact fills a missing date + // part from today, which a fixed result cannot match. + bool date = _year >= 0 && _month >= 0 && _day >= 0; + bool noDate = _year < 0 && _month < 0 && _day < 0; + bool time = _hour >= 0 && (_second < 0 || _minute >= 0); + bool noTime = _hour < 0 && _minute < 0 && _second < 0; + _valid = (date || noDate) && (time || noTime) && (date || time); + } + + internal bool HasDate => _year >= 0; + + internal bool HasTime => _hour >= 0; + + internal static FastDate? TryCreate(string? format) + { + if (format is null) + { + return null; + } + var fast = new FastDate(format); + return fast._valid ? fast : null; + } + + /// The date and time in , already trimmed; 0001-01-01 when the format has no date. + internal bool TryParse(ReadOnlySpan s, in FixedWidthContext context, out DateTime value) + { + value = default; + if (s.Length != _format.Length || !context.GregorianDates || (_usesSeparators && !context.StandardDateSeparators)) + { + return false; + } + for (int i = 0; i < s.Length; i++) + { + char f = _format[i]; + bool matches = f is 'y' or 'M' or 'd' or 'H' or 'm' or 's' ? (uint)(s[i] - '0') <= 9 : s[i] == f; + if (!matches) + { + return false; + } + } + int year = _year < 0 ? 1 : Digits(s, _year, 4); + int month = _month < 0 ? 1 : Digits(s, _month, 2); + int day = _day < 0 ? 1 : Digits(s, _day, 2); + int hour = _hour < 0 ? 0 : Digits(s, _hour, 2); + int minute = _minute < 0 ? 0 : Digits(s, _minute, 2); + int second = _second < 0 ? 0 : Digits(s, _second, 2); + if (year < 1 || month is < 1 or > 12 || day < 1 || day > DateTime.DaysInMonth(year, month) + || hour > 23 || minute > 59 || second > 59) + { + return false; + } + // Unspecified, as TryParseExact returns text without a zone. + value = new DateTime(year, month, day, hour, minute, second, DateTimeKind.Unspecified); + return true; + } + + private static int Digits(ReadOnlySpan s, int offset, int count) + { + int value = 0; + foreach (ref readonly byte b in s.Slice(offset, count)) + { + value = (value * 10) + (b - '0'); + } + return value; + } + } +} diff --git a/src/FixedWidthParser/Infrastructure/FastDouble.cs b/src/FixedWidthParser/Infrastructure/FastDouble.cs index 1f45008..fb3a318 100644 --- a/src/FixedWidthParser/Infrastructure/FastDouble.cs +++ b/src/FixedWidthParser/Infrastructure/FastDouble.cs @@ -100,7 +100,7 @@ private static bool TryScan(ReadOnlySpan s, byte decimalSeparator, out ulo if (i < s.Length && s[i] == decimalSeparator) { int fractionStart = ++i; - i = ReadFractionDigits(s, i, ref mantissa); + i = ReadDigits(s, i, ref mantissa); scale = i - fractionStart; } int allDigits = integerDigits + scale; @@ -180,8 +180,9 @@ private static bool EiselLemire(ulong w, int q, bool neg, out double value) // fast_float's digit loop: eight digits per step while the next eight bytes are all digits, then one // four-digit step. Integer parts stay byte by byte, since most are too short for a step to pay off. + // FastInteger reads with it too, as zero-padded fixed-width integers are long. [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int ReadFractionDigits(ReadOnlySpan s, int i, ref ulong mantissa) + internal static int ReadDigits(ReadOnlySpan s, int i, ref ulong mantissa) { while (s.Length - i >= 8) { diff --git a/src/FixedWidthParser/Infrastructure/FastInteger.cs b/src/FixedWidthParser/Infrastructure/FastInteger.cs new file mode 100644 index 0000000..6d5ce27 --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/FastInteger.cs @@ -0,0 +1,34 @@ +using System.Runtime.CompilerServices; + +namespace FixedWidthParser.Infrastructure +{ + /// + /// ExcelReader's ASCII-digit fast path for integers: digits only, after a "-" when the provider's negative + /// sign is "-", and few enough digits that the value cannot overflow. for anything + /// else, and the caller falls back to the BCL, which reads such text the same way in every culture. + /// + internal static class FastInteger + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool TryParse(ReadOnlySpan s, bool allowMinus, int maxDigits, out long value) + { + value = 0; + bool negative = allowMinus && !s.IsEmpty && s[0] == (byte)'-'; + if (negative) + { + s = s[1..]; + } + if (s.IsEmpty || s.Length > maxDigits) + { + return false; + } + ulong accumulated = 0; + if (FastDouble.ReadDigits(s, 0, ref accumulated) != s.Length) + { + return false; + } + value = negative ? -(long)accumulated : (long)accumulated; + return true; + } + } +} diff --git a/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs index 1e05ea8..c390be3 100644 --- a/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs +++ b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs @@ -50,13 +50,13 @@ public static string String(ReadOnlySpan column, in FixedWidthContext cont // The fast parsers need the provider's decimal separator as one ASCII byte and "-" as its negative sign // (FixedWidthContext.DecimalSeparator). What they refuse, thousands separators included, goes to the // BCL over the decoded text: its UTF-8 parser does not take a space for a no-break-space group - // separator, as fr-FR writes one. Leading spaces are trimmed too, so a right-aligned column stays fast. + // separator, as fr-FR writes one. Spaces are trimmed too, so a right-aligned column stays fast. private const NumberStyles FloatStyles = NumberStyles.Float | NumberStyles.AllowThousands; /// Parses a column, honoring the provider's separators. public static bool TryDouble(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out double value) { - var trimmed = TrimColumn(column, in options).TrimStart((byte)' '); + var trimmed = TrimColumn(column, in options).Trim((byte)' '); byte separator = context.DecimalSeparator; if (separator != 0 && FastDouble.TryParse(trimmed, separator, out value)) { @@ -68,7 +68,7 @@ public static bool TryDouble(ReadOnlySpan column, in FixedWidthContext con /// Parses a column, honoring the provider's separators. public static bool TryFloat(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out float value) { - var trimmed = TrimColumn(column, in options).TrimStart((byte)' '); + var trimmed = TrimColumn(column, in options).Trim((byte)' '); byte separator = context.DecimalSeparator; if (separator != 0 && FastDouble.TryParse(trimmed, separator, out value)) { @@ -82,9 +82,26 @@ public static bool TryFloat(ReadOnlySpan column, in FixedWidthContext cont public static bool TryParse(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, [MaybeNullWhen(false)] out TValue value) where TValue : IUtf8SpanParsable { + // The BCL's default styles for these types allow surrounding white space, so trimming spaces for + // the fast paths changes no result. var trimmed = TrimColumn(column, in options); + if (typeof(TValue) == typeof(long) || typeof(TValue) == typeof(int)) + { + bool isInt = typeof(TValue) == typeof(int); + if (FastInteger.TryParse(trimmed.Trim((byte)' '), context.DecimalSeparator != 0, isInt ? 9 : 18, out long integer)) + { + if (isInt) + { + int narrow = (int)integer; + value = Unsafe.As(ref narrow); + return true; + } + value = Unsafe.As(ref integer); + return true; + } + } if (typeof(TValue) == typeof(decimal) && context.DecimalSeparator is var separator and not 0 - && FastDecimal.TryParse(trimmed.TrimStart((byte)' '), separator, out decimal fast)) + && FastDecimal.TryParse(trimmed.Trim((byte)' '), separator, out decimal fast)) { value = Unsafe.As(ref fast); return true; @@ -111,19 +128,36 @@ public static bool TryConvert( /// Parses a column with exactly , decoding it first. public static bool TryDateTimeExact(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out DateTime value) { - return ParseDecoded(TrimColumn(column, in options), in context, options.Format, static (text, format, provider, out result) => DateTime.TryParseExact(text, format, provider, DateTimeStyles.None, out result), out value); + var trimmed = TrimColumn(column, in options); + if (options.FastDate is { HasDate: true } fast && fast.TryParse(trimmed, in context, out value)) + { + return true; + } + return ParseDecoded(trimmed, in context, options.Format, static (text, format, provider, out result) => DateTime.TryParseExact(text, format, provider, DateTimeStyles.None, out result), out value); } /// Parses a column with exactly , decoding it first. public static bool TryDateOnlyExact(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out DateOnly value) { - return ParseDecoded(TrimColumn(column, in options), in context, options.Format, static (text, format, provider, out result) => DateOnly.TryParseExact(text, format, provider, DateTimeStyles.None, out result), out value); + var trimmed = TrimColumn(column, in options); + if (options.FastDate is { HasDate: true, HasTime: false } fast && fast.TryParse(trimmed, in context, out var date)) + { + value = DateOnly.FromDateTime(date); + return true; + } + return ParseDecoded(trimmed, in context, options.Format, static (text, format, provider, out result) => DateOnly.TryParseExact(text, format, provider, DateTimeStyles.None, out result), out value); } /// Parses a column with exactly , decoding it first. public static bool TryTimeOnlyExact(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out TimeOnly value) { - return ParseDecoded(TrimColumn(column, in options), in context, options.Format, static (text, format, provider, out result) => TimeOnly.TryParseExact(text, format, provider, DateTimeStyles.None, out result), out value); + var trimmed = TrimColumn(column, in options); + if (options.FastDate is { HasDate: false } fast && fast.TryParse(trimmed, in context, out var time)) + { + value = TimeOnly.FromDateTime(time); + return true; + } + return ParseDecoded(trimmed, in context, options.Format, static (text, format, provider, out result) => TimeOnly.TryParseExact(text, format, provider, DateTimeStyles.None, out result), out value); } /// Parses a column with exactly , decoding it first. diff --git a/tests/FixedWidthParser.Tests/CultureTests.cs b/tests/FixedWidthParser.Tests/CultureTests.cs index cf2634f..fc3c521 100644 --- a/tests/FixedWidthParser.Tests/CultureTests.cs +++ b/tests/FixedWidthParser.Tests/CultureTests.cs @@ -88,6 +88,70 @@ public void Parse_Double_CommaCulture_MatchesTheBcl(string cultureName) } } + // The digit-only fast path for exact formats must agree with TryParseExact, including where it must + // step aside: a Buddhist calendar (th-TH) and "/" or ":" standing for other separators (de-DE). + [Theory] + [InlineData("")] + [InlineData("pt-BR")] + [InlineData("de-DE")] + [InlineData("th-TH")] + public void Parse_ExactDates_MatchTheBcl(string cultureName) + { + var culture = CultureInfo.GetCultureInfo(cultureName); + var context = CtxWith(culture); + string[] formats = ["yyyyMMdd", "yyyy-MM-dd", "dd/MM/yyyy", "yyyyMMddHHmmss", "yyyy-MM-ddTHH:mm:ss", "HHmmss", "HH:mm:ss", "HH", "MMdd", "yyyyMM", "yyyy.MM.dd"]; + string[] edges = ["00000101", "20230230", "20240229", "20230229", "20231301", "99991231", "2023010", "2023o105", "-0230105", "24:00:00", "23:59:60", "235959", "000000", "2023-01-05", "05/01/2023", "05.01.2023"]; + var random = new Random(42); + + foreach (string format in formats) + { + var options = new Infrastructure.ColumnParseOptions(' ', TrimMode.Both, format); + var samples = Enumerable.Range(0, 300) + .Select(_ => new DateTime(1, 1, 1).AddSeconds(random.NextInt64(0, (long)(DateTime.MaxValue - DateTime.MinValue).TotalSeconds))) + .SelectMany(d => new[] { d.ToString(format, CultureInfo.InvariantCulture), d.ToString(format, culture) }); + foreach (string text in edges.Concat(samples)) + { + byte[] bytes = Bytes(text); + bool expected = DateTime.TryParseExact(text, format, culture, DateTimeStyles.None, out var bclDateTime); + Assert.True(expected == Infrastructure.FixedWidthRuntime.TryDateTimeExact(bytes, in context, in options, out var dateTime) && (!expected || bclDateTime == dateTime && bclDateTime.Kind == dateTime.Kind), + $"DateTime '{text}' as {format}: BCL {expected} {bclDateTime:O}, parser {dateTime:O}"); + expected = DateOnly.TryParseExact(text, format, culture, DateTimeStyles.None, out var bclDate); + Assert.True(expected == Infrastructure.FixedWidthRuntime.TryDateOnlyExact(bytes, in context, in options, out var date) && (!expected || bclDate == date), + $"DateOnly '{text}' as {format}: BCL {expected} {bclDate:O}, parser {date:O}"); + expected = TimeOnly.TryParseExact(text, format, culture, DateTimeStyles.None, out var bclTime); + Assert.True(expected == Infrastructure.FixedWidthRuntime.TryTimeOnlyExact(bytes, in context, in options, out var time) && (!expected || bclTime == time), + $"TimeOnly '{text}' as {format}: BCL {expected} {bclTime:O}, parser {time:O}"); + } + } + } + + // sv-SE's negative sign is U+2212, so a leading "-" must not take the integer fast path there. + [Theory] + [InlineData("")] + [InlineData("de-DE")] + [InlineData("sv-SE")] + public void Parse_Integers_MatchTheBcl(string cultureName) + { + var culture = CultureInfo.GetCultureInfo(cultureName); + var context = CtxWith(culture); + var options = new Infrastructure.ColumnParseOptions(' ', TrimMode.Trailing, format: null); + string[] edges = ["-5", "+5", " 12", "12 ", "0007", "-0", "-", "", "1,000", "1.000", "999999999", "2147483647", "2147483648", "-2147483648", "-2147483649", + "999999999999999999", "9223372036854775807", "9223372036854775808", "-9223372036854775808", "−5", "1e3", "12x"]; + var random = new Random(42); + var values = Enumerable.Range(0, 2000).Select(_ => random.NextInt64(long.MinValue, long.MaxValue) >> random.Next(0, 63)).Select(v => v.ToString(CultureInfo.InvariantCulture)); + + foreach (string text in edges.Concat(values)) + { + byte[] bytes = Bytes(text); + bool expected = long.TryParse(bytes, culture, out long bclLong); + bool actual = Infrastructure.FixedWidthRuntime.TryParse(bytes, in context, in options, out long parsedLong); + Assert.True(expected == actual && (!expected || bclLong == parsedLong), $"long '{text}': BCL {expected} {bclLong}, parser {actual} {parsedLong}"); + expected = int.TryParse(bytes, culture, out int bclInt); + actual = Infrastructure.FixedWidthRuntime.TryParse(bytes, in context, in options, out int parsedInt); + Assert.True(expected == actual && (!expected || bclInt == parsedInt), $"int '{text}': BCL {expected} {bclInt}, parser {actual} {parsedInt}"); + } + } + // The float fast path rounds once in binary32; going through a double could land one ulp off. [Theory] [InlineData("")] From fff8a5ab69775c73139b16e4578f9e3053e28bbc Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 18:01:03 -0300 Subject: [PATCH 67/85] perf(native): cut page faults and serial work from parallel reads Timed inside the native read, every partition stayed busy but slowed as threads were added (1.0x at 2, 1.7x at 8, 3x at 16), with an in-memory source no better than a file. A read writes ~200 MB of columns: 50,000 demand-zero page faults on one thread, 100,000 in parallel because the concatenation copies everything into fresh memory again. - A partition sizes its column buffers once from its byte count, which fixed-width lines turn into a row count, instead of doubling them from 256 elements. - skip_invalid skips a bad line through an internal OnSkip hook instead of building its exception: numbering B3's trailer counted line breaks from the start of the file on one thread (~27 ms). - Publish concatenates one column per thread. COTAHIST through Python, 1M lines: fw.read 0.250 -> 0.169 s, one thread 0.678 -> 0.580 s. --- CHANGELOG.md | 3 ++ docs/python-benchmarks.md | 48 ++++++++++++++----- .../GrowableNativeBuffer.cs | 10 ++++ .../Reading/ColumnBuilder.cs | 20 ++++++-- .../Reading/NativeTableReader.cs | 5 ++ .../Reading/ReadApi.cs | 2 +- .../Reading/ReadSettings.cs | 2 +- .../Reading/TableSink.cs | 42 +++++++++++++--- .../FixedWidthReaderOptions.cs | 5 ++ .../Readers/ParallelAggregator.cs | 10 +++- tests/FixedWidthParser.Tests/LineSinkTests.cs | 8 ++-- 11 files changed, 126 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 974d591..75a7d76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -134,6 +134,9 @@ Windows-1252 after registering `CodePagesEncodingProvider`). and `int`/`long` columns of plain digits parse straight from the bytes, about 5x and 1.5x faster; any other text still goes to the BCL. A single-threaded B3 COTAHIST read through Python is 26% faster ([docs/python-benchmarks.md](docs/python-benchmarks.md)). +- Native and Python whole-file reads size their column buffers once per partition, skip bad lines under + `skip_invalid` without numbering them, and concatenate the partitions' columns in parallel: a parallel + COTAHIST read is about 32% faster. - Reading with a `StringPool` checks a small per-thread cache before the pool, about halving the cost of a pooled read. - The library, generator and native library build with warnings as errors. diff --git a/docs/python-benchmarks.md b/docs/python-benchmarks.md index a58cc0b..a60a148 100644 --- a/docs/python-benchmarks.md +++ b/docs/python-benchmarks.md @@ -16,17 +16,17 @@ pyarrow 26.0.0, polars 2.0.0. `read_fwf` best of 1 run, the others best of 5. | Reader | Seconds | vs read_fwf | |---|---:|---:| -| pandas.read_fwf | 15.908 | 1.0x | -| polars str.slice (reference) | 0.351 | 45x | -| fw.read_pandas | 0.293 | 54x | -| fw.read_pandas, 1 thread | 0.747 | 21x | -| fw.read_polars | 0.267 | 60x | -| fw.read_arrow | 0.239 | 67x | -| fw.read (NumPy views) | 0.250 | 64x | -| fw.read, 1 thread | 0.678 | 23x | - -A single thread is already 23x faster than `read_fwf`; the default, one thread per processor, adds another 2.7x. -`fw.read_pandas` costs about 50 ms over `fw.read_arrow`, pyarrow's conversion to pandas. +| pandas.read_fwf | 14.412 | 1.0x | +| polars str.slice (reference) | 0.328 | 44x | +| fw.read_pandas | 0.207 | 70x | +| fw.read_pandas, 1 thread | 0.618 | 23x | +| fw.read_polars | 0.188 | 77x | +| fw.read_arrow | 0.169 | 85x | +| fw.read (NumPy views) | 0.169 | 85x | +| fw.read, 1 thread | 0.580 | 25x | + +A single thread is already 25x faster than `read_fwf`; the default, one thread per processor, adds another 3.4x. +`fw.read_pandas` costs about 40 ms over `fw.read_arrow`, pyarrow's conversion to pandas. ## Fast paths for dates and integers @@ -42,3 +42,29 @@ back to the BCL for anything else. Same machine, same run order, before and afte | fw.read | 0.261 s | 0.250 s | The parallel read barely moved, so parsing is not what bounds it. + +## The parallel read + +Timed inside the native read at 16 threads, the 0.25 s split into ~170 ms of partitioned parsing and ~40 ms of +concatenating the partitions' columns on one thread. Every partition was busy all along, yet the work took ~2.1 s +of thread time against ~0.7 s on one thread, and more threads made each partition slower (1.0x at 2 threads, +1.3x at 4, 1.7x at 8, 3x at 16). An in-memory source changed little, so it was not file I/O. What grows with +the threads is new memory: a read writes ~200 MB of columns, 50,000 demand-zero page faults on one thread and +twice that in parallel, since the concatenation copies everything into fresh memory again, and Windows takes +those faults under a per-process lock. + +Three changes, together: + +- A partition sizes its column buffers once from its byte count (fixed-width lines make that a row count) + instead of doubling them from 256 elements. +- `skip_invalid` skips a bad line without building its exception. Numbering the line meant counting line + breaks from the start of the file, on one thread, for B3's trailer at the very end: ~27 ms. +- The concatenation runs one column per thread. + +| Reader | Before | After | +|---|---:|---:| +| fw.read | 0.250 s | 0.169 s | +| fw.read, 1 thread | 0.678 s | 0.580 s | + +Not done: the concatenation still copies every column, and its page faults are the next cost. Handing +`read_arrow` the partitions as Arrow chunks would skip that copy for pyarrow, pandas and polars. diff --git a/src/FixedWidthParser.Native/GrowableNativeBuffer.cs b/src/FixedWidthParser.Native/GrowableNativeBuffer.cs index 81f7389..06e7787 100644 --- a/src/FixedWidthParser.Native/GrowableNativeBuffer.cs +++ b/src/FixedWidthParser.Native/GrowableNativeBuffer.cs @@ -23,6 +23,16 @@ internal sealed unsafe class GrowableNativeBuffer : IDisposable where T : unm internal ReadOnlySpan Span => new(_data, Count); + /// Grows once to hold at least elements, clamped to what a buffer can hold. + internal void Reserve(long capacity) + { + capacity = Math.Min(capacity, int.MaxValue); + if (capacity > _capacity) + { + Grow(capacity); + } + } + /// Drops every element from on. internal void Truncate(int count) { diff --git a/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs index 8ff7528..e554f07 100644 --- a/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs +++ b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs @@ -29,13 +29,27 @@ internal sealed unsafe class ColumnBuilder : IDisposable private GrowableNativeBuffer? _validity; #pragma warning restore SS066 - internal ColumnBuilder(NativeColumn column, long maxTextBytes) + /// How many rows to size the buffers for up front, so a partition does not grow them row by row. + internal ColumnBuilder(NativeColumn column, long maxTextBytes, long expectedRows) { _column = column; _maxTextBytes = maxTextBytes; - if (column.Type == NativeType.String) + switch (column.Type) { - _ints.Add(0); + case NativeType.String: + _ints.Reserve(expectedRows + 1); + _bytes.Reserve(Math.Min(expectedRows * column.Length, maxTextBytes)); + _ints.Add(0); + break; + case NativeType.Float64: + _doubles.Reserve(expectedRows); + break; + case NativeType.Date: + _ints.Reserve(expectedRows); + break; + default: + _longs.Reserve(expectedRows); + break; } } diff --git a/src/FixedWidthParser.Native/Reading/NativeTableReader.cs b/src/FixedWidthParser.Native/Reading/NativeTableReader.cs index 044c375..9e4ebbe 100644 --- a/src/FixedWidthParser.Native/Reading/NativeTableReader.cs +++ b/src/FixedWidthParser.Native/Reading/NativeTableReader.cs @@ -92,6 +92,11 @@ private void Fill(ref TableSink sink) } if (!sink.TryAdd(line, in _context)) { + if (_options.OnSkip is { } skip) + { + skip(); + continue; + } ParseErrors.Reject(sink.Diagnose(line, _lines.LineNumber, in _context), _options.OnParseError); } } diff --git a/src/FixedWidthParser.Native/Reading/ReadApi.cs b/src/FixedWidthParser.Native/Reading/ReadApi.cs index 85c1b39..921fcc1 100644 --- a/src/FixedWidthParser.Native/Reading/ReadApi.cs +++ b/src/FixedWidthParser.Native/Reading/ReadApi.cs @@ -51,7 +51,7 @@ private static int Read(IRandomAccessSource source, NativeLayout layout, NativeR var skipped = new StrongBox(headerLines); var readerOptions = settings.ReaderOptions(skipped, settings.DegreeOfParallelism, chunkSize); #pragma warning disable VSTHRD002 // An export is synchronous and runs with no synchronization context. - using var table = ParallelAggregator.RunAsync(source, dataStart, () => new TableSink(layout, limits), readerOptions, CancellationToken.None) + using var table = ParallelAggregator.RunAsync(source, dataStart, bytes => new TableSink(layout, limits, bytes), readerOptions, CancellationToken.None) .GetAwaiter().GetResult(); #pragma warning restore VSTHRD002 table.Publish(outTable); diff --git a/src/FixedWidthParser.Native/Reading/ReadSettings.cs b/src/FixedWidthParser.Native/Reading/ReadSettings.cs index 29501a0..090f448 100644 --- a/src/FixedWidthParser.Native/Reading/ReadSettings.cs +++ b/src/FixedWidthParser.Native/Reading/ReadSettings.cs @@ -60,7 +60,7 @@ internal FixedWidthReaderOptions ReaderOptions(StrongBox skipped, int degr DegreeOfParallelism = degreeOfParallelism, ChunkSize = chunkSize, ByteOrderMark = ByteOrderMark, - OnParseError = SkipInvalid ? _ => Interlocked.Increment(ref skipped.Value) > 0 : null, + OnSkip = SkipInvalid ? () => Interlocked.Increment(ref skipped.Value) : null, }); } diff --git a/src/FixedWidthParser.Native/Reading/TableSink.cs b/src/FixedWidthParser.Native/Reading/TableSink.cs index 43a42ca..c80d5c7 100644 --- a/src/FixedWidthParser.Native/Reading/TableSink.cs +++ b/src/FixedWidthParser.Native/Reading/TableSink.cs @@ -1,3 +1,4 @@ +using System.Runtime.ExceptionServices; using System.Runtime.InteropServices; using FixedWidthParser.Native.Layouts; using FixedWidthParser.Readers; @@ -14,11 +15,13 @@ internal struct TableSink : ILineSink, IDisposable private int _failedColumn; private List? _following; - internal TableSink(NativeLayout layout, TableLimits limits) + /// The bytes this sink will see, if known: fixed-width lines make that a row count. + internal TableSink(NativeLayout layout, TableLimits limits, long expectedBytes = 0) { _layout = layout; _limits = limits; - _columns = [.. layout.Columns.Select(c => new ColumnBuilder(c, limits.MaxTextBytes))]; + long expectedRows = Math.Min(expectedBytes / (layout.LineLength + 1), limits.MaxRows); + _columns = [.. layout.Columns.Select(c => new ColumnBuilder(c, limits.MaxTextBytes, expectedRows))]; _requiredLength = layout.Columns.Max(c => c.Start + c.Length); _failedColumn = -1; _following = null; @@ -90,14 +93,16 @@ internal readonly unsafe void Publish(NativeTableRaw* table) table->ColumnCount = _columns.Length; try { - var parts = new ColumnBuilder[chunks.Length]; - for (int i = 0; i < _columns.Length; i++) + if (chunks.Length == 1) { - for (int c = 0; c < chunks.Length; c++) + for (int i = 0; i < _columns.Length; i++) { - parts[c] = chunks[c][i]; + chunks[0][i].Publish(columns + i); } - ColumnBuilder.Concat(parts, columns + i); + } + else + { + ConcatInParallel(chunks, columns); } table->RowCount = TotalRows; } @@ -108,6 +113,29 @@ internal readonly unsafe void Publish(NativeTableRaw* table) } } + // A concatenated column is all fresh memory, and its page faults are most of the copy's cost: one + // column per thread spreads them out. + private static unsafe void ConcatInParallel(ColumnBuilder[][] chunks, NativeColumnRaw* columns) + { + nint target = (nint)columns; + try + { + Parallel.For(0, chunks[0].Length, i => + { + var parts = new ColumnBuilder[chunks.Length]; + for (int c = 0; c < chunks.Length; c++) + { + parts[c] = chunks[c][i]; + } + ColumnBuilder.Concat(parts, (NativeColumnRaw*)target + i); + }); + } + catch (AggregateException ex) + { + ExceptionDispatchInfo.Throw(ex.InnerExceptions[0]); + } + } + private readonly void CheckLimits(ColumnBuilder[][] chunks) { if (TotalRows > _limits.MaxRows) diff --git a/src/FixedWidthParser/FixedWidthReaderOptions.cs b/src/FixedWidthParser/FixedWidthReaderOptions.cs index 48bdb5a..540d3ee 100644 --- a/src/FixedWidthParser/FixedWidthReaderOptions.cs +++ b/src/FixedWidthParser/FixedWidthReaderOptions.cs @@ -64,6 +64,11 @@ internal int Workers internal FixedWidthContext Context => new() { FormatProvider = FormatProvider!, Encoding = Encoding!, StringPool = StringPool }; + // Skips a line that does not parse and calls this, instead of building the exception OnParseError gets: + // the native library's skip_invalid only counts lines, and numbering one in a partition re-reads the + // source from its start. + internal Action? OnSkip { get; init; } + // Overrides whether a leading UTF-8 byte order mark is skipped; null decides from the encoding. internal bool? ByteOrderMark { get; init; } diff --git a/src/FixedWidthParser/Readers/ParallelAggregator.cs b/src/FixedWidthParser/Readers/ParallelAggregator.cs index f3d305f..c216c21 100644 --- a/src/FixedWidthParser/Readers/ParallelAggregator.cs +++ b/src/FixedWidthParser/Readers/ParallelAggregator.cs @@ -38,13 +38,14 @@ internal static async Task RunAsync /// /// Reads [dataStart, source.Length) in partitions, one each, merged in /// order. must start a line; line numbers count from the start of the source. + /// gets the bytes of the partition its sink will read. /// internal static Task RunAsync( - IRandomAccessSource source, long dataStart, Func create, FixedWidthReaderOptions options, CancellationToken ct) + IRandomAccessSource source, long dataStart, Func create, FixedWidthReaderOptions options, CancellationToken ct) where TSink : struct, ILineSink { return RunPartitionsAsync(source, dataStart, options, - (plan, index) => SinkChunk(source, plan, index, create(), options, ct), + (plan, index) => SinkChunk(source, plan, index, create(plan.End(index) - plan.Start(index)), options, ct), static (merged, following) => { merged.Merge(following); @@ -170,6 +171,11 @@ private static TSink SinkChunk( } if (!sink.TryAdd(line, in context)) { + if (options.OnSkip is { } skip) + { + skip(); + continue; + } ParseErrors.Reject(sink.Diagnose(line, reader.LineNumberAt(lineStart), in context), options.OnParseError); } } diff --git a/tests/FixedWidthParser.Tests/LineSinkTests.cs b/tests/FixedWidthParser.Tests/LineSinkTests.cs index fdd5d9b..f966b07 100644 --- a/tests/FixedWidthParser.Tests/LineSinkTests.cs +++ b/tests/FixedWidthParser.Tests/LineSinkTests.cs @@ -66,7 +66,7 @@ public async Task RunAsync_FromDataStart_SkipsEarlierBytes(int chunkSize) { var source = new ArraySource("HDR\nA1\nA2\n"u8.ToArray()); - var lines = await ParallelAggregator.RunAsync(source, dataStart: 4, () => new Lines(), Options(chunkSize), CancellationToken.None); + var lines = await ParallelAggregator.RunAsync(source, dataStart: 4, _ => new Lines(), Options(chunkSize), CancellationToken.None); Assert.Equal(["A1", "A2"], lines.Values); } @@ -79,7 +79,7 @@ public async Task RunAsync_ErrorLineNumber_CountsFromSourceStart(int chunkSize) var source = new ArraySource("HDR\nA1\nXX\nA3\n"u8.ToArray()); var ex = await Assert.ThrowsAsync(() => - ParallelAggregator.RunAsync(source, dataStart: 4, () => new Lines(), Options(chunkSize), CancellationToken.None)); + ParallelAggregator.RunAsync(source, dataStart: 4, _ => new Lines(), Options(chunkSize), CancellationToken.None)); Assert.Equal(3, ex.LineNumber); } @@ -91,8 +91,8 @@ public async Task RunAsync_MemorySource_EqualsFileSource() await File.WriteAllBytesAsync(_path, content); using var handle = File.OpenHandle(_path); - var fromFile = await ParallelAggregator.RunAsync(new FileSource(handle), 0, () => new Lines(), Options(3), CancellationToken.None); - var fromMemory = await ParallelAggregator.RunAsync(new ArraySource(content), 0, () => new Lines(), Options(3), CancellationToken.None); + var fromFile = await ParallelAggregator.RunAsync(new FileSource(handle), 0, _ => new Lines(), Options(3), CancellationToken.None); + var fromMemory = await ParallelAggregator.RunAsync(new ArraySource(content), 0, _ => new Lines(), Options(3), CancellationToken.None); Assert.Equal(["A1", "A2", "A3"], fromFile.Values); Assert.Equal(fromFile.Values, fromMemory.Values); From d6f30d5af58f385e0a38e85cf46b16d909682c81 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 18:16:59 -0300 Subject: [PATCH 68/85] bench: add a Python write benchmark against savetxt and polars bench_write.py reads a synthetic COTAHIST file once into pandas, polars and pyarrow tables, then times each writer turning the table back into its record lines, asserting every one reproduces the file's body byte for byte. pandas and polars have no fixed-width writer, so the baselines are numpy.savetxt with printf fields and polars pad + concat_str. The generator now caps a day's quantity at 10^9 shares: its 16-digit volumes did not survive a float64, so no writer could reproduce them. 1M lines: fw.write (pyarrow) 0.259 s, polars 0.722 s, savetxt 4.285 s. --- docs/python-benchmarks.md | 59 ++++++++++---- python/benchmarks/bench_write.py | 129 +++++++++++++++++++++++++++++++ python/benchmarks/cotahist.py | 3 +- 3 files changed, 177 insertions(+), 14 deletions(-) create mode 100644 python/benchmarks/bench_write.py diff --git a/docs/python-benchmarks.md b/docs/python-benchmarks.md index a60a148..fe8645b 100644 --- a/docs/python-benchmarks.md +++ b/docs/python-benchmarks.md @@ -1,33 +1,66 @@ -# Python: fixedwidth against pandas.read_fwf +# Python: fixedwidth against pandas and polars `python/benchmarks/bench_read.py` generates a synthetic file shaped like B3's COTAHIST quote history (245-byte lines, all 26 fields of record type 01, a 00 header and a 99 trailer; layout in `python/benchmarks/cotahist.py`), checks that every reader returns the same values as `pandas.read_fwf` given the same column specs (fixedwidth exactly, the polars reference within floating-point rounding), and times each one, best of several runs. +`python/benchmarks/bench_write.py` writes the same table back out (see [Writing](#writing)). python python/benchmarks/bench_read.py --lines 1000000 --runs 5 + python python/benchmarks/bench_write.py --lines 1000000 --runs 5 -`read_fwf` uses its Python engine, which `skipfooter` needs to drop the trailer. +`read_fwf` uses its Python engine, which `skipfooter` needs to drop the trailer. The generator caps a day's +quantity at 10^9 shares, so a volume has at most 15 digits and survives a FLOAT64 column exactly; the writers +could not reproduce a 16-digit one from a float64 either. -## Results +## Reading 1,000,000 lines (235 MiB). AMD Ryzen 7 5700X (8 cores, 16 threads), Windows 10, Python 3.14.7, pandas 3.1.0, pyarrow 26.0.0, polars 2.0.0. `read_fwf` best of 1 run, the others best of 5. | Reader | Seconds | vs read_fwf | |---|---:|---:| -| pandas.read_fwf | 14.412 | 1.0x | -| polars str.slice (reference) | 0.328 | 44x | -| fw.read_pandas | 0.207 | 70x | -| fw.read_pandas, 1 thread | 0.618 | 23x | -| fw.read_polars | 0.188 | 77x | -| fw.read_arrow | 0.169 | 85x | -| fw.read (NumPy views) | 0.169 | 85x | -| fw.read, 1 thread | 0.580 | 25x | - -A single thread is already 25x faster than `read_fwf`; the default, one thread per processor, adds another 3.4x. +| pandas.read_fwf | 14.751 | 1.0x | +| polars str.slice (reference) | 0.349 | 42x | +| fw.read_pandas | 0.212 | 70x | +| fw.read_pandas, 1 thread | 0.655 | 23x | +| fw.read_polars | 0.198 | 74x | +| fw.read_arrow | 0.195 | 76x | +| fw.read (NumPy views) | 0.166 | 89x | +| fw.read, 1 thread | 0.600 | 25x | + +A single thread is already 25x faster than `read_fwf`; the default, one thread per processor, adds another 3.6x. `fw.read_pandas` costs about 40 ms over `fw.read_arrow`, pyarrow's conversion to pandas. +## Writing + +`bench_write.py` reads the generated file once, before any timing, into pandas, polars and pyarrow tables, and +times each writer turning that table back into the record lines. The expected output is the generated file's +body, so every writer must produce it byte for byte; the script asserts that. Neither pandas nor polars writes +fixed-width files, so the baselines are what their users write instead: + +- `numpy.savetxt` over the pandas columns with one printf field per column (`%-12s`, `%013d`), floats scaled to + their implied decimals and dates turned into `yyyymmdd` integers first. +- polars string expressions: `pad_end` for text, the scaled integers zero-padded with the sign kept leftmost, + `strftime` for dates, joined with `concat_str` into one column written by `write_csv`. + +Same machine and versions; `numpy.savetxt` best of 1 run, the others best of 5. + +| Writer | Seconds | vs savetxt | +|---|---:|---:| +| numpy.savetxt (pandas) | 4.285 | 1.0x | +| polars pad + concat_str | 0.722 | 5.9x | +| fw.write (pandas) | 0.329 | 13x | +| fw.write (polars) | 0.314 | 14x | +| fw.write (pyarrow) | 0.259 | 17x | +| fw.write (pyarrow), 1 thread | 0.838 | 5.1x | +| fw.write_bytes (pyarrow) | 0.404 | 11x | + +`fw.write` is 2.8x faster than the polars pipeline. A pandas frame costs ~70 ms more than a pyarrow table, its +conversion to Arrow; `write_bytes` timing includes copying the 235 MB into a `bytes` and writing it to the file. +Of `fw.write`'s 0.26 s, ~20 ms is the Python side handing the columns over and ~240 ms the native write, which +16 threads speed up only 3.5x over one (0.84 s). + ## Fast paths for dates and integers The two `yyyyMMdd` date columns cost 85 ns each through `DateOnly.TryParseExact` on the decoded text, and the diff --git a/python/benchmarks/bench_write.py b/python/benchmarks/bench_write.py new file mode 100644 index 0000000..9c63500 --- /dev/null +++ b/python/benchmarks/bench_write.py @@ -0,0 +1,129 @@ +"""Writes a synthetic COTAHIST table with fixedwidth and with the ways pandas and polars users write one, checks +every writer produces the same bytes, and times them. + + python python/benchmarks/bench_write.py [--lines 1000000] [--runs 5] [--savetxt-runs 1] + +The table is read once, before any timing, from a generated file; every writer then turns that same table back +into its record lines, so the expected output is the file's body, byte for byte. pandas and polars have no +fixed-width writer: the baselines are numpy.savetxt with one printf field per column, and polars string +expressions padded and concatenated into one column written as CSV. +""" + +from __future__ import annotations + +import argparse +import platform +import sys +import tempfile +import time +from pathlib import Path + +import fixedwidth as fw +import numpy as np +import pandas as pd +import polars as pl +import pyarrow as pa + +sys.path.insert(0, str(Path(__file__).parent)) +import cotahist # noqa: E402 + + +def savetxt(frame: pd.DataFrame, path: Path) -> None: + columns, formats = [], [] + for name, _, length, kind, scale in cotahist.FIELDS: + column = frame[name] + if kind is fw.String: + columns.append(column.to_numpy(dtype=object)) + formats.append(f"%-{length}s") + elif kind is fw.Date: + dates = pd.to_datetime(column) + columns.append((dates.dt.year * 10000 + dates.dt.month * 100 + dates.dt.day).to_numpy()) + formats.append(f"%0{length}d") + elif kind is fw.Float64: + columns.append(np.round(column.to_numpy() * 10**scale).astype(np.int64)) + formats.append(f"%0{length}d") + else: + columns.append(column.to_numpy()) + formats.append(f"%0{length}d") + rows = np.empty((len(frame), len(columns)), dtype=object) + for index, values in enumerate(columns): + rows[:, index] = values + # A binary file, or Windows turns every "\n" into "\r\n". + with path.open("wb") as file: + np.savetxt(file, rows, fmt="".join(formats), newline="\n") + + +def zero_padded(integers: pl.Expr, length: int) -> pl.Expr: + # The sign stays leftmost, ahead of the zeros, as printf's %0Nd and fixedwidth write it. + digits = integers.abs().cast(pl.String) + return pl.when(integers < 0).then(pl.lit("-") + digits.str.pad_start(length - 1, "0")).otherwise(digits.str.pad_start(length, "0")) + + +def polars_strings(frame: pl.DataFrame, path: Path) -> None: + fields = [] + for name, _, length, kind, scale in cotahist.FIELDS: + column = pl.col(name) + if kind is fw.String: + fields.append(column.str.pad_end(length)) + elif kind is fw.Date: + fields.append(column.dt.strftime("%Y%m%d")) + elif kind is fw.Float64: + fields.append(zero_padded((column * 10**scale).round().cast(pl.Int64), length)) + else: + fields.append(zero_padded(column, length)) + frame.select(pl.concat_str(fields).alias("line")).write_csv(path, include_header=False, quote_style="never") + + +def best(runs: int, work, path: Path, expected: bytes) -> float: + times = [] + for _ in range(runs): + path.unlink(missing_ok=True) + start = time.perf_counter() + work() + times.append(time.perf_counter() - start) + assert path.read_bytes() == expected, f"{path.name} differs from the expected record lines" + return min(times) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--lines", type=int, default=1_000_000) + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--savetxt-runs", type=int, default=1, help="numpy.savetxt is slow; one run is usually enough") + args = parser.parse_args() + + layout = cotahist.layout() + read = fw.ReadOptions(header_lines=1, skip_invalid=True) + sequential = fw.WriteOptions(degree_of_parallelism=1) + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "COTAHIST_SYNTHETIC.TXT" + cotahist.generate(source, args.lines) + lines = source.read_bytes().splitlines(keepends=True) + expected = b"".join(lines[1:-1]) + arrow = fw.read_arrow(source, layout, read) + frame = fw.read_pandas(source, layout, read) + polars = pl.from_arrow(arrow) + size = len(expected) / 2**20 + print(f"{args.lines:,} lines, {size:.0f} MiB, Python {platform.python_version()}, pandas {pd.__version__}, " + f"polars {pl.__version__}, pyarrow {pa.__version__}") + + out = Path(directory) / "out.txt" + rows = [ + ("numpy.savetxt (pandas)", best(args.savetxt_runs, lambda: savetxt(frame, out), out, expected)), + ("polars pad + concat_str", best(args.runs, lambda: polars_strings(polars, out), out, expected)), + ("fw.write (pandas)", best(args.runs, lambda: fw.write(frame, out, layout), out, expected)), + ("fw.write (polars)", best(args.runs, lambda: fw.write(polars, out, layout), out, expected)), + ("fw.write (pyarrow)", best(args.runs, lambda: fw.write(arrow, out, layout), out, expected)), + ("fw.write (pyarrow), 1 thread", best(args.runs, lambda: fw.write(arrow, out, layout, sequential), out, expected)), + ("fw.write_bytes (pyarrow)", best(args.runs, lambda: out.write_bytes(fw.write_bytes(arrow, layout)), out, expected)), + ] + print("every writer produced the generated file's record lines, byte for byte") + baseline = rows[0][1] + print(f"{'writer':<32}{'seconds':>10}{'vs savetxt':>13}") + for name, seconds in rows: + print(f"{name:<32}{seconds:>10.3f}{baseline / seconds:>12.1f}x") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/benchmarks/cotahist.py b/python/benchmarks/cotahist.py index f82caae..173bb55 100644 --- a/python/benchmarks/cotahist.py +++ b/python/benchmarks/cotahist.py @@ -80,7 +80,8 @@ def _line(rng: random.Random, day: int) -> str: open_ = rng.randint(100, 9_999_999) prices = [open_ + rng.randint(-500, 500) for _ in range(6)] trades = rng.randint(1, 99_999) - quantity = rng.randint(100, 10**12) + # At most 10**9 shares keeps the volume (quantity * price) within the 15 digits a float64 round-trips. + quantity = rng.randint(100, 10**9) return ( "01" + f"202310{day:02d}" From b4d22cfc7677a3586bba9aa5521dd479b76670af Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 18:43:14 -0300 Subject: [PATCH 69/85] perf: write integers and digit-only dates directly, and write files in parallel At 16 threads fw_write_file took 240 ms where fw_write_memory took 141, with the two equal on one thread: Windows runs one synchronous I/O at a time on a handle, so the parallel batches queued. The file is now opened for asynchronous I/O; each batch still waits for its write, but they run together (240 -> 170 ms). A right-aligned int or long with no format writes its digits from the right straight into the column, then the padding, with "-" kept leftmost (19 -> 9 ns): formatting at the start and aligning after cost a copy. It applies only when the provider's negative sign is "-". A date or time whose format is only digit fields reuses the reader's compiled FastDate to write its digits (47 -> 23 ns). A test checks both against the BCL's char path in four cultures, both alignments, and zero and space padding. COTAHIST through Python, 1M lines: fw.write 0.259 -> 0.162 s, one thread 0.838 -> 0.550 s. --- CHANGELOG.md | 3 + docs/python-benchmarks.md | 43 ++++++++++---- .../Writing/WriteApi.cs | 4 +- .../Infrastructure/ColumnFormatOptions.cs | 4 ++ .../Infrastructure/FastDate.cs | 46 ++++++++++++++- .../Infrastructure/FastInteger.cs | 42 ++++++++++++++ .../Infrastructure/FixedWidthRuntime.cs | 23 ++++++++ tests/FixedWidthParser.Tests/CultureTests.cs | 57 +++++++++++++++++++ 8 files changed, 207 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75a7d76..8f6a4c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -137,6 +137,9 @@ Windows-1252 after registering `CodePagesEncodingProvider`). - Native and Python whole-file reads size their column buffers once per partition, skip bad lines under `skip_invalid` without numbering them, and concatenate the partitions' columns in parallel: a parallel COTAHIST read is about 32% faster. +- Writing: right-aligned `int`/`long` columns without a format put their digits straight into the line, + digit-only date and time formats reuse the reader's fast path, and `fw_write_file` opens its file for + asynchronous I/O so parallel batches write at once. A COTAHIST write through Python is about 37% faster. - Reading with a `StringPool` checks a small per-thread cache before the pool, about halving the cost of a pooled read. - The library, generator and native library build with warnings as errors. diff --git a/docs/python-benchmarks.md b/docs/python-benchmarks.md index fe8645b..d1d7b80 100644 --- a/docs/python-benchmarks.md +++ b/docs/python-benchmarks.md @@ -48,18 +48,39 @@ Same machine and versions; `numpy.savetxt` best of 1 run, the others best of 5. | Writer | Seconds | vs savetxt | |---|---:|---:| -| numpy.savetxt (pandas) | 4.285 | 1.0x | -| polars pad + concat_str | 0.722 | 5.9x | -| fw.write (pandas) | 0.329 | 13x | -| fw.write (polars) | 0.314 | 14x | -| fw.write (pyarrow) | 0.259 | 17x | -| fw.write (pyarrow), 1 thread | 0.838 | 5.1x | -| fw.write_bytes (pyarrow) | 0.404 | 11x | - -`fw.write` is 2.8x faster than the polars pipeline. A pandas frame costs ~70 ms more than a pyarrow table, its +| numpy.savetxt (pandas) | 4.302 | 1.0x | +| polars pad + concat_str | 0.707 | 6.1x | +| fw.write (pandas) | 0.219 | 20x | +| fw.write (polars) | 0.190 | 23x | +| fw.write (pyarrow) | 0.162 | 27x | +| fw.write (pyarrow), 1 thread | 0.550 | 7.8x | +| fw.write_bytes (pyarrow) | 0.342 | 13x | + +`fw.write` is 4.4x faster than the polars pipeline. A pandas frame costs ~60 ms more than a pyarrow table, its conversion to Arrow; `write_bytes` timing includes copying the 235 MB into a `bytes` and writing it to the file. -Of `fw.write`'s 0.26 s, ~20 ms is the Python side handing the columns over and ~240 ms the native write, which -16 threads speed up only 3.5x over one (0.84 s). +Of `fw.write`'s 0.16 s, ~20 ms is the Python side handing the columns over and ~140 ms the native write. + +### Making the native write faster + +The first run measured 0.259 s, ~240 ms of it native. Per value, a zero-padded integer (16 of the 26 columns) +cost 19 ns and a `yyyyMMdd` date 47 ns. At 16 threads, `fw_write_file` took 240 ms where `fw_write_memory` +took 141 ms, with the two equal on one thread. Windows runs one synchronous I/O at a time on a file handle, so +the parallel batches queued for it. + +- The output file is opened for asynchronous I/O. Each batch still waits for its own write, but they run + together: 240 → 170 ms. +- A right-aligned integer with no format writes its digits from the right straight into the column, then the + padding, with "-" kept leftmost: 19 → 9 ns. Formatting at the start and aligning afterwards, as before, cost + a copy, and counting the digits first cost a second pass. +- A date or time whose format is only digit fields and `- . T / :` reuses the reader's compiled `FastDate` + to write its digits: 47 → 23 ns. + +A test checks both against the BCL's char path in four cultures, both alignments, and zero and space padding. + +| Writer | Before | After | +|---|---:|---:| +| fw.write (pyarrow) | 0.259 s | 0.162 s | +| fw.write (pyarrow), 1 thread | 0.838 s | 0.550 s | ## Fast paths for dates and integers diff --git a/src/FixedWidthParser.Native/Writing/WriteApi.cs b/src/FixedWidthParser.Native/Writing/WriteApi.cs index 39c88ee..bc28f8c 100644 --- a/src/FixedWidthParser.Native/Writing/WriteApi.cs +++ b/src/FixedWidthParser.Native/Writing/WriteApi.cs @@ -47,7 +47,9 @@ internal static int WriteFile(ReadOnlySpan path, NativeTableRaw* table, Na try { long total = formatter!.Rows * lineBytes; - using var handle = File.OpenHandle(Encoding.UTF8.GetString(path), FileMode.Create, FileAccess.Write, FileShare.None, FileOptions.None, total); + // Asynchronous, though every write waits: Windows runs one synchronous I/O at a time on a handle, + // which would serialize the parallel batches. + using var handle = File.OpenHandle(Encoding.UTF8.GetString(path), FileMode.Create, FileAccess.Write, FileShare.None, FileOptions.Asynchronous, total); RandomAccess.SetLength(handle, total); return Run(formatter, settings!, lineBytes, (first, count) => WriteBatch(handle, formatter, settings!, layout, first, count, lineBytes)); } diff --git a/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs b/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs index e052a74..450c055 100644 --- a/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs +++ b/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs @@ -26,6 +26,7 @@ public ColumnFormatOptions(Alignment alignment, char padding, string? format, Ov Padding = padding; Format = format; Overflow = overflow; + FastDate = FastDate.TryCreate(format); } /// Which side a written value is aligned to. @@ -40,6 +41,9 @@ public ColumnFormatOptions(Alignment alignment, char padding, string? format, Ov /// The resolved overflow policy: or . public OverflowBehavior Overflow { get; } + /// compiled for , or when it cannot be. + internal FastDate? FastDate { get; } + /// /// Writes the encoded into the column slice , /// applying alignment, padding and overflow. diff --git a/src/FixedWidthParser/Infrastructure/FastDate.cs b/src/FixedWidthParser/Infrastructure/FastDate.cs index ea69742..7175756 100644 --- a/src/FixedWidthParser/Infrastructure/FastDate.cs +++ b/src/FixedWidthParser/Infrastructure/FastDate.cs @@ -3,9 +3,10 @@ namespace FixedWidthParser.Infrastructure /// /// A date or time format made only of fixed-width digit fields (yyyy, MM, dd, HH, /// mm, ss) and the literals - . T / :, read straight from the bytes the way ExcelReader's - /// FastDate reads ISO 8601. compiles it once per column and returns - /// for any other format; returns for - /// anything it cannot read exactly as TryParseExact would, and the caller falls back to it. + /// FastDate reads ISO 8601, and written the same way. compiles it once per column + /// and returns for any other format; and + /// return for anything they cannot do exactly as TryParseExact and ToString + /// would, and the caller falls back to those. /// internal sealed class FastDate { @@ -117,6 +118,45 @@ internal bool TryParse(ReadOnlySpan s, in FixedWidthContext context, out D return true; } + /// + /// Writes as ToString with the format would, at the start of + /// ; when it does not fit or the provider's calendar or + /// separators rule the fast path out. + /// + internal bool TryFormat(DateTime value, Span destination, in FixedWidthContext context, out int written) + { + written = 0; + if (destination.Length < _format.Length || !context.GregorianDates || (_usesSeparators && !context.StandardDateSeparators)) + { + return false; + } + for (int i = 0; i < _format.Length; i++) + { + destination[i] = (byte)_format[i]; + } + WriteDigits(destination, _year, 4, value.Year); + WriteDigits(destination, _month, 2, value.Month); + WriteDigits(destination, _day, 2, value.Day); + WriteDigits(destination, _hour, 2, value.Hour); + WriteDigits(destination, _minute, 2, value.Minute); + WriteDigits(destination, _second, 2, value.Second); + written = _format.Length; + return true; + } + + private static void WriteDigits(Span destination, int offset, int count, int value) + { + if (offset < 0) + { + return; + } + for (int i = offset + count - 1; i >= offset; i--) + { + destination[i] = (byte)('0' + (value % 10)); + value /= 10; + } + } + private static int Digits(ReadOnlySpan s, int offset, int count) { int value = 0; diff --git a/src/FixedWidthParser/Infrastructure/FastInteger.cs b/src/FixedWidthParser/Infrastructure/FastInteger.cs index 6d5ce27..db50bdb 100644 --- a/src/FixedWidthParser/Infrastructure/FastInteger.cs +++ b/src/FixedWidthParser/Infrastructure/FastInteger.cs @@ -30,5 +30,47 @@ public static bool TryParse(ReadOnlySpan s, bool allowMinus, int maxDigits value = negative ? -(long)accumulated : (long)accumulated; return true; } + + /// + /// Writes right-aligned into the whole of in one pass, as + /// the default format and would together: digits from the + /// right, then , with "-" ahead of the padding unless it is a space. + /// when it does not fit, leaving partly written. + /// + public static bool TryFormatRight(long value, Span slice, byte padding) + { + // Unsigned, so long.MinValue negates. + ulong magnitude = value < 0 ? 0 - (ulong)value : (ulong)value; + int i = slice.Length; + do + { + if (i == 0) + { + return false; + } + ulong quotient = magnitude / 10; + slice[--i] = (byte)('0' + (magnitude - (quotient * 10))); + magnitude = quotient; + } + while (magnitude != 0); + if (value >= 0) + { + slice[..i].Fill(padding); + return true; + } + if (i == 0) + { + return false; + } + if (padding == (byte)' ') + { + slice[--i] = (byte)'-'; + slice[..i].Fill(padding); + return true; + } + slice[0] = (byte)'-'; + slice[1..i].Fill(padding); + return true; + } } } diff --git a/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs index c390be3..23bf281 100644 --- a/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs +++ b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs @@ -261,6 +261,17 @@ public static void FormatValue(TValue value, Span slice, in FixedW public static void FormatUtf8Value(TValue value, Span slice, in FixedWidthContext context, ColumnFormatOptions options, string columnName) where TValue : IUtf8SpanFormattable, ISpanFormattable { + // The default format writes ASCII digits after the provider's negative sign, which DecimalSeparator + // being set guarantees is "-". + if ((typeof(TValue) == typeof(long) || typeof(TValue) == typeof(int)) && options.Format is null + && options.Alignment == Alignment.Right && context.DecimalSeparator != 0) + { + long integer = typeof(TValue) == typeof(int) ? Unsafe.As(ref value) : Unsafe.As(ref value); + if (FastInteger.TryFormatRight(integer, slice, (byte)options.Padding)) + { + return; + } + } if (TryFormatInto(value, slice, in context, options, out int written) && (context.IsUtf8 || Ascii.IsValid(slice[..written]))) { options.AlignInPlace(slice, written); @@ -276,6 +287,18 @@ private static bool TryFormatInto(TValue value, Span destination, { return TryFormatDouble(Unsafe.As(ref value), destination, in context, out written); } + if (typeof(TValue) == typeof(DateOnly) && options.FastDate is { HasDate: true, HasTime: false } dateFormat) + { + return dateFormat.TryFormat(Unsafe.As(ref value).ToDateTime(TimeOnly.MinValue), destination, in context, out written); + } + if (typeof(TValue) == typeof(TimeOnly) && options.FastDate is { HasDate: false } timeFormat) + { + return timeFormat.TryFormat(DateTime.MinValue + Unsafe.As(ref value).ToTimeSpan(), destination, in context, out written); + } + if (typeof(TValue) == typeof(DateTime) && options.FastDate is { } dateTimeFormat) + { + return dateTimeFormat.TryFormat(Unsafe.As(ref value), destination, in context, out written); + } return value.TryFormat(destination, out written, options.Format, context.FormatProvider); } diff --git a/tests/FixedWidthParser.Tests/CultureTests.cs b/tests/FixedWidthParser.Tests/CultureTests.cs index fc3c521..f6ca75a 100644 --- a/tests/FixedWidthParser.Tests/CultureTests.cs +++ b/tests/FixedWidthParser.Tests/CultureTests.cs @@ -1,4 +1,5 @@ using System.Globalization; +using System.Text; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -152,6 +153,62 @@ public void Parse_Integers_MatchTheBcl(string cultureName) } } + // The integer and date fast paths for writing must lay out the same bytes as the BCL's char path. + [Theory] + [InlineData("")] + [InlineData("de-DE")] + [InlineData("th-TH")] + [InlineData("sv-SE")] + public void Format_IntegersAndDates_MatchTheBcl(string cultureName) + { + var context = CtxWith(CultureInfo.GetCultureInfo(cultureName)); + var random = new Random(42); + long[] edges = [0, 1, -1, 9, -9, 10, 123, -171, int.MaxValue, int.MinValue, long.MaxValue, long.MinValue]; + var longs = edges.Concat(Enumerable.Range(0, 1000).Select(_ => random.NextInt64(long.MinValue, long.MaxValue) >> random.Next(0, 63))); + var dates = Enumerable.Range(0, 300).Select(_ => new DateTime(1, 1, 1).AddSeconds(random.NextInt64(0, (long)(DateTime.MaxValue - DateTime.MinValue).TotalSeconds))); + string[] formats = ["yyyyMMdd", "yyyy-MM-dd", "dd/MM/yyyy", "yyyy-MM-ddTHH:mm:ss", "HHmmss", "HH:mm:ss"]; + + foreach (var alignment in new[] { Alignment.Left, Alignment.Right }) + { + foreach (char padding in new[] { ' ', '0' }) + { + var plain = new Infrastructure.ColumnFormatOptions(alignment, padding, null, OverflowBehavior.Throw); + foreach (long value in longs) + { + // Room for sv-SE's three-byte minus sign too. + Same(value, plain, 23); + Same((int)value, plain, 14); + } + foreach (string format in formats) + { + var options = new Infrastructure.ColumnFormatOptions(alignment, padding, format, OverflowBehavior.Throw); + foreach (var date in dates) + { + Same(date, options, 24); + if (!format.Contains('H', StringComparison.Ordinal)) + { + Same(DateOnly.FromDateTime(date), options, 24); + } + if (!format.Contains('y', StringComparison.Ordinal)) + { + Same(TimeOnly.FromDateTime(date), options, 24); + } + } + } + } + } + + void Same(T value, Infrastructure.ColumnFormatOptions options, int width) + where T : IUtf8SpanFormattable, ISpanFormattable + { + byte[] fast = new byte[width]; + byte[] bcl = new byte[width]; + Infrastructure.FixedWidthRuntime.FormatUtf8Value(value, fast, in context, options, "c"); + Infrastructure.FixedWidthRuntime.FormatValue(value, bcl, in context, options, "c"); + Assert.True(fast.AsSpan().SequenceEqual(bcl), $"{typeof(T).Name} {value} ({options.Format}, {options.Alignment}, '{options.Padding}'): fast \"{Encoding.UTF8.GetString(fast)}\", BCL \"{Encoding.UTF8.GetString(bcl)}\""); + } + } + // The float fast path rounds once in binary32; going through a double could land one ulp off. [Theory] [InlineData("")] From a31350436bffcc7e3dafe438c24822a5bc8c0e6c Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 20:00:27 -0300 Subject: [PATCH 70/85] refactor: reorganize import statements for clarity --- python/benchmarks/bench_read.py | 3 +-- python/src/fixedwidth/writer.py | 8 +++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/python/benchmarks/bench_read.py b/python/benchmarks/bench_read.py index 5055c96..1d372b1 100644 --- a/python/benchmarks/bench_read.py +++ b/python/benchmarks/bench_read.py @@ -12,12 +12,11 @@ import time from pathlib import Path +import fixedwidth as fw import numpy as np import pandas as pd import polars as pl -import fixedwidth as fw - sys.path.insert(0, str(Path(__file__).parent)) import cotahist # noqa: E402 diff --git a/python/src/fixedwidth/writer.py b/python/src/fixedwidth/writer.py index 87a5640..2c28d44 100644 --- a/python/src/fixedwidth/writer.py +++ b/python/src/fixedwidth/writer.py @@ -8,7 +8,13 @@ from fixedwidth import _native from fixedwidth.layout import Layout, check -from fixedwidth.types import Column, ColumnType, FixedWidthError, WriteOptions, code_page +from fixedwidth.types import ( + Column, + ColumnType, + FixedWidthError, + WriteOptions, + code_page, +) def write(data: Any, path: str | os.PathLike[str], layout: Layout, options: WriteOptions | None = None) -> None: From c8d1d5a5ef4914b1255f9a75dc490389a7536c9c Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 21:32:09 -0300 Subject: [PATCH 71/85] perf(native): parse partitions straight into the final columns A whole-source read allocates the final columns once and lends each partition the slice its byte offset gives, with the first line's length as the stride. Numeric, date and time values and STRING bytes are parsed in place; publishing moves each partition down to close the gaps skipped lines and trimmed text leave, and hands the buffers over. A partition that outgrows its slice copies its column out, and that column is concatenated as before. STRING offsets are still copied. A parallel COTAHIST read through Python: fw.read 0.184 -> 0.153 s, fw.read_polars 0.198 -> 0.165 s. --- CHANGELOG.md | 4 + docs/python-benchmarks.md | 20 ++- .../GrowableNativeBuffer.cs | 20 +++ .../Reading/ColumnBuilder.cs | 118 +++++++++++++----- .../Reading/ReadApi.cs | 3 +- .../Reading/TableArena.cs | 104 +++++++++++++++ .../Reading/TableSink.cs | 32 +++-- .../Readers/ParallelAggregator.cs | 7 +- tests/FixedWidthParser.Tests/LineSinkTests.cs | 8 +- .../Native/NativeParallelReadTests.cs | 32 +++++ 10 files changed, 302 insertions(+), 46 deletions(-) create mode 100644 src/FixedWidthParser.Native/Reading/TableArena.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f6a4c6..7f7eabc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -137,6 +137,10 @@ Windows-1252 after registering `CodePagesEncodingProvider`). - Native and Python whole-file reads size their column buffers once per partition, skip bad lines under `skip_invalid` without numbering them, and concatenate the partitions' columns in parallel: a parallel COTAHIST read is about 32% faster. +- Native and Python whole-file reads parse every partition straight into the final columns, at the row its + byte offset gives when all lines have the first line's length, instead of copying the partitions together + afterwards. Numeric, date and time values and STRING bytes skip the copy, and a parallel COTAHIST read is + another 17% faster. A file whose lines vary in length still reads correctly; its columns are copied as before. - Writing: right-aligned `int`/`long` columns without a format put their digits straight into the line, digit-only date and time formats reuse the reader's fast path, and `fw_write_file` opens its file for asynchronous I/O so parallel batches write at once. A COTAHIST write through Python is about 37% faster. diff --git a/docs/python-benchmarks.md b/docs/python-benchmarks.md index d1d7b80..b8a019f 100644 --- a/docs/python-benchmarks.md +++ b/docs/python-benchmarks.md @@ -120,5 +120,21 @@ Three changes, together: | fw.read | 0.250 s | 0.169 s | | fw.read, 1 thread | 0.678 s | 0.580 s | -Not done: the concatenation still copies every column, and its page faults are the next cost. Handing -`read_arrow` the partitions as Arrow chunks would skip that copy for pyarrow, pandas and polars. +### Parsing into the final columns + +The concatenation still wrote every column into fresh memory a second time. Fixed-width lines fix where each +partition's rows land in the result: with the first line's length as the stride, a partition starting at byte +`b` starts at row `(b - dataStart) / stride`, rounded up. The read now allocates the final columns once and lends +each partition its slice; numeric, date and time values go straight there, and so do STRING bytes, with room +for the column's full width on every row. Publishing moves each partition down to close the gaps that skipped +lines and trimmed text leave, in memory already touched, and hands the buffers over. A partition with more +rows than its slice, as when line lengths vary, copies its column out, and that column is concatenated as +before. STRING offsets are still copied: each slice's leading 0 would fall on the previous slice's last offset. + +Same machine, same run order, best of 9: + +| Reader | Before | After | +|---|---:|---:| +| fw.read | 0.184 s | 0.153 s | +| fw.read_polars | 0.198 s | 0.165 s | +| fw.read, 1 thread | 0.588 s | 0.593 s | diff --git a/src/FixedWidthParser.Native/GrowableNativeBuffer.cs b/src/FixedWidthParser.Native/GrowableNativeBuffer.cs index 06e7787..e6a6b2e 100644 --- a/src/FixedWidthParser.Native/GrowableNativeBuffer.cs +++ b/src/FixedWidthParser.Native/GrowableNativeBuffer.cs @@ -23,6 +23,21 @@ internal sealed unsafe class GrowableNativeBuffer : IDisposable where T : unm internal ReadOnlySpan Span => new(_data, Count); + internal T* Pointer => _data; + + /// Whether the elements still sit in memory lent by . + internal bool Borrowed => _block is null && _data is not null; + + /// + /// Fills elements at , which the caller owns and frees, and + /// copies them into a block of its own if it outgrows them. + /// + internal void Borrow(T* data, int capacity) + { + _data = data; + _capacity = capacity; + } + /// Grows once to hold at least elements, clamped to what a buffer can hold. internal void Reserve(long capacity) { @@ -111,9 +126,14 @@ private void Grow(long minimum) { int capacity = NextCapacity(_capacity, minimum); nint bytes = checked((nint)capacity * sizeof(T)); + T* lent = Borrowed ? _data : null; _block ??= new Block(); _data = (T*)_block.Resize(bytes); _capacity = capacity; + if (lent is not null) + { + new ReadOnlySpan(lent, Count).CopyTo(new Span(_data, Count)); + } } private sealed class Block : SafeHandle diff --git a/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs index e554f07..50800a5 100644 --- a/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs +++ b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs @@ -29,8 +29,14 @@ internal sealed unsafe class ColumnBuilder : IDisposable private GrowableNativeBuffer? _validity; #pragma warning restore SS066 + /// + /// Initializes a new instance of the class. + /// + /// The native column this builder is for. + /// The maximum number of text bytes expected for this column. /// How many rows to size the buffers for up front, so a partition does not grow them row by row. - internal ColumnBuilder(NativeColumn column, long maxTextBytes, long expectedRows) + /// Room for values that a owns, or null. + internal ColumnBuilder(NativeColumn column, long maxTextBytes, long expectedRows, void* lent = null) { _column = column; _maxTextBytes = maxTextBytes; @@ -38,21 +44,42 @@ internal ColumnBuilder(NativeColumn column, long maxTextBytes, long expectedRows { case NativeType.String: _ints.Reserve(expectedRows + 1); - _bytes.Reserve(Math.Min(expectedRows * column.Length, maxTextBytes)); + Size(_bytes, lent, Math.Min(expectedRows * column.Length, maxTextBytes)); _ints.Add(0); break; case NativeType.Float64: - _doubles.Reserve(expectedRows); + Size(_doubles, lent, expectedRows); break; case NativeType.Date: - _ints.Reserve(expectedRows); + Size(_ints, lent, expectedRows); break; default: - _longs.Reserve(expectedRows); + Size(_longs, lent, expectedRows); break; } } + private static void Size(GrowableNativeBuffer values, void* lent, long rows) where T : unmanaged + { + if (lent is null) + { + values.Reserve(rows); + return; + } + values.Borrow((T*)lent, (int)rows); + } + + /// The bytes of one value of a column of ; 0 for STRING, which has no fixed size. + internal static int ValueSize(int type) + { + return type switch + { + NativeType.String => 0, + NativeType.Date => sizeof(int), + _ => sizeof(long), + }; + } + internal int Rows { get; private set; } /// The UTF-8 bytes a STRING column holds. @@ -113,18 +140,23 @@ internal void Publish(NativeColumnRaw* target) }); } - /// Publishes the same column of several partitions, in order, as one column: the one copy a partitioned read makes. - internal static void Concat(ReadOnlySpan parts, NativeColumnRaw* target) + /// + /// Publishes the same column of several partitions, in order, as one column. Partitions that parsed into slices + /// of are moved down in it to close the gaps between them, and it becomes the column; + /// otherwise, or once a partition outgrew its slice, they are copied into fresh memory. + /// + internal static void Concat(ReadOnlySpan parts, NativeColumnRaw* target, GrowableNativeBuffer? lent = null) { - if (parts.Length == 1) - { - parts[0].Publish(target); - return; - } long rows = 0; foreach (ref readonly var part in parts) { rows += part.Rows; + lent = part.Lent ? lent : null; + } + if (parts.Length == 1 && lent is null) + { + parts[0].Publish(target); + return; } var column = parts[0]._column; target->Type = column.Type; @@ -135,12 +167,21 @@ internal static void Concat(ReadOnlySpan parts, NativeColumnRaw* target->DataLen = 0; if (column.Type == NativeType.String) { - ConcatStrings(parts, rows, target); + ConcatStrings(parts, rows, target, lent); return; } - target->Values = ConcatValues(parts); + target->Values = ConcatValues(parts, lent); } + /// Whether this partition's values, or a STRING column's bytes, still sit where a lent them. + private bool Lent => _column.Type switch + { + NativeType.String => _bytes.Borrowed, + NativeType.Float64 => _doubles.Borrowed, + NativeType.Date => _ints.Borrowed, + _ => _longs.Borrowed, + }; + private ReadOnlySpan ValueBytes => _column.Type switch { NativeType.Float64 => MemoryMarshal.AsBytes(_doubles.Span), @@ -148,25 +189,25 @@ internal static void Concat(ReadOnlySpan parts, NativeColumnRaw* _ => MemoryMarshal.AsBytes(_longs.Span), }; - private static void* ConcatValues(ReadOnlySpan parts) + private static void* ConcatValues(ReadOnlySpan parts, GrowableNativeBuffer? lent) { long total = 0; foreach (ref readonly var part in parts) { total += part.ValueBytes.Length; } - byte* values = (byte*)Marshal.AllocHGlobal((nint)total); + byte* values = lent is null ? (byte*)Marshal.AllocHGlobal((nint)total) : lent.Pointer; long at = 0; foreach (ref readonly var part in parts) { var bytes = part.ValueBytes; - bytes.CopyTo(new Span(values + at, bytes.Length)); + MoveTo(bytes, values + at); at += bytes.Length; } - return values; + return lent is null ? values : (void*)lent.Detach(); } - private static void ConcatStrings(ReadOnlySpan parts, long rows, NativeColumnRaw* target) + private static void ConcatStrings(ReadOnlySpan parts, long rows, NativeColumnRaw* target, GrowableNativeBuffer? lent) { long total = 0; foreach (ref readonly var part in parts) @@ -178,26 +219,47 @@ private static void ConcatStrings(ReadOnlySpan parts, long rows, throw new InvalidOperationException( $"Column '{parts[0]._column.Name}' holds more than 2 GiB of text; read in batches (fw_reader_open_*)."); } - int* offsets = (int*)Marshal.AllocHGlobal((nint)((rows + 1) * sizeof(int))); - byte* data = (byte*)Marshal.AllocHGlobal((nint)total); - offsets[0] = 0; + byte* data = lent is null ? (byte*)Marshal.AllocHGlobal((nint)total) : lent.Pointer; + // One partition's offsets already start at 0. + int* offsets = parts.Length == 1 ? null : (int*)Marshal.AllocHGlobal((nint)((rows + 1) * sizeof(int))); + if (offsets is not null) + { + offsets[0] = 0; + } long row = 1; int baseOffset = 0; foreach (ref readonly var part in parts) { - var partOffsets = part._ints.Span; - foreach (ref readonly int offset in partOffsets[1..]) + if (offsets is not null) { - offsets[row++] = baseOffset + offset; + foreach (ref readonly int offset in part._ints.Span[1..]) + { + offsets[row++] = baseOffset + offset; + } } - part._bytes.Span.CopyTo(new Span(data + baseOffset, part._bytes.Count)); + MoveTo(part._bytes.Span, data + baseOffset); baseOffset += part._bytes.Count; } - target->Values = offsets; - target->Data = data; + target->Values = offsets is null ? (void*)parts[0]._ints.Detach() : offsets; + target->Data = lent is null ? data : (byte*)lent.Detach(); target->DataLen = total; } + /// + /// Copies to , which may overlap them: partitions lent slices of one + /// buffer move down in order, each staying inside its own slice, so none overwrites one not yet moved. + /// + private static void MoveTo(ReadOnlySpan bytes, byte* to) + { + fixed (byte* from = bytes) + { + if (from != to) + { + bytes.CopyTo(new Span(to, bytes.Length)); + } + } + } + private static byte* ConcatValidity(ReadOnlySpan parts, long rows) { bool any = false; diff --git a/src/FixedWidthParser.Native/Reading/ReadApi.cs b/src/FixedWidthParser.Native/Reading/ReadApi.cs index 921fcc1..8595308 100644 --- a/src/FixedWidthParser.Native/Reading/ReadApi.cs +++ b/src/FixedWidthParser.Native/Reading/ReadApi.cs @@ -51,7 +51,8 @@ private static int Read(IRandomAccessSource source, NativeLayout layout, NativeR var skipped = new StrongBox(headerLines); var readerOptions = settings.ReaderOptions(skipped, settings.DegreeOfParallelism, chunkSize); #pragma warning disable VSTHRD002 // An export is synchronous and runs with no synchronization context. - using var table = ParallelAggregator.RunAsync(source, dataStart, bytes => new TableSink(layout, limits, bytes), readerOptions, CancellationToken.None) + using var arena = TableArena.TryCreate(source, dataStart, layout, limits); + using var table = ParallelAggregator.RunAsync(source, dataStart, (start, end) => new TableSink(layout, limits, arena, start, end), readerOptions, CancellationToken.None) .GetAwaiter().GetResult(); #pragma warning restore VSTHRD002 table.Publish(outTable); diff --git a/src/FixedWidthParser.Native/Reading/TableArena.cs b/src/FixedWidthParser.Native/Reading/TableArena.cs new file mode 100644 index 0000000..64be7ea --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/TableArena.cs @@ -0,0 +1,104 @@ +using System.Buffers; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Readers; + +namespace FixedWidthParser.Native.Reading +{ + /// + /// The final buffers of a whole-source read, lent a slice per partition so the partitions parse straight into them + /// and publishing has nothing to copy. Every line is taken to be as long as the first, so a partition's first row + /// is its offset over that length. A partition with fewer rows leaves a gap that publishing closes in place; one + /// with more copies its column out of the slice, and that column is copied as before. A STRING column is lent + /// room for its whole width on every row, for its bytes. + /// + // ponytail: STRING offsets are still copied once, rebased; a slice's leading 0 would land on the last offset of + // the slice before. Lending one extra offset per partition would cut that copy too. + internal sealed unsafe class TableArena : IDisposable + { + private const int ProbeBytes = 64 * 1024; + + private readonly long _dataStart; + private readonly long _stride; + private readonly int _rows; + private readonly int[] _sizes; + private readonly GrowableNativeBuffer?[] _blocks; + + private TableArena(long dataStart, long stride, int rows, NativeLayout layout, TableLimits limits) + { + _dataStart = dataStart; + _stride = stride; + _rows = rows; + _sizes = [.. layout.Columns.Select(c => BytesPerRow(c, rows, limits))]; + _blocks = new GrowableNativeBuffer?[_sizes.Length]; + for (int i = 0; i < _blocks.Length; i++) + { + if (_sizes[i] > 0) + { + var block = new GrowableNativeBuffer(); + block.Reserve((long)rows * _sizes[i]); + _blocks[i] = block; + } + } + } + + /// + /// An arena sized from the length of the first line at ; null when there is no line + /// break in the first 64 KiB, or the rows would not fit one buffer or . + /// + internal static TableArena? TryCreate(IRandomAccessSource source, long dataStart, NativeLayout layout, TableLimits limits) + { + byte[] probe = ArrayPool.Shared.Rent(ProbeBytes); + try + { + int read = source.Read(probe.AsSpan(0, ProbeBytes), dataStart); + long stride = probe.AsSpan(0, read).IndexOf((byte)'\n') + 1L; + if (stride == 0) + { + return null; + } + long rows = (source.Length - dataStart + stride - 1) / stride; + bool fits = rows > 0 && rows <= limits.MaxRows && rows * sizeof(long) <= int.MaxValue; + return fits ? new TableArena(dataStart, stride, (int)rows, layout, limits) : null; + } + finally + { + ArrayPool.Shared.Return(probe); + } + } + + /// What the arena lends a column per row; 0 for a STRING column whose full width would break the text limit. + private static int BytesPerRow(NativeColumn column, int rows, TableLimits limits) + { + if (column.Type != NativeType.String) + { + return ColumnBuilder.ValueSize(column.Type); + } + return (long)rows * column.Length <= Math.Min(limits.MaxTextBytes, int.MaxValue) ? column.Length : 0; + } + + /// The row of the first line starting at or after , if every line is as long as the first. + internal int RowAt(long offset) + { + return (int)Math.Min(_rows, (offset - _dataStart + _stride - 1) / _stride); + } + + /// Where column 's value for goes; null for a column the arena does not hold. + internal void* Lend(int column, int row) + { + return _blocks[column] is { } block ? block.Pointer + ((long)row * _sizes[column]) : null; + } + + internal GrowableNativeBuffer? Block(int column) + { + return _blocks[column]; + } + + public void Dispose() + { + foreach (var block in _blocks) + { + block?.Dispose(); + } + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/TableSink.cs b/src/FixedWidthParser.Native/Reading/TableSink.cs index c80d5c7..ea1fb4a 100644 --- a/src/FixedWidthParser.Native/Reading/TableSink.cs +++ b/src/FixedWidthParser.Native/Reading/TableSink.cs @@ -12,16 +12,32 @@ internal struct TableSink : ILineSink, IDisposable private readonly ColumnBuilder[] _columns; private readonly int _requiredLength; private readonly TableLimits _limits; +#pragma warning disable SS066 // Shared by every partition; the read that made it disposes it. + private readonly TableArena? _arena; +#pragma warning restore SS066 private int _failedColumn; private List? _following; - /// The bytes this sink will see, if known: fixed-width lines make that a row count. - internal TableSink(NativeLayout layout, TableLimits limits, long expectedBytes = 0) + /// + /// Initializes a new instance of the struct. + /// + /// The final buffers to parse into, shared by every partition; null for buffers of its own. + /// Where the bytes this sink will see start, if known: fixed-width lines make them a row count. + /// Where those bytes end. + internal unsafe TableSink(NativeLayout layout, TableLimits limits, TableArena? arena = null, long start = 0, long end = 0) { _layout = layout; _limits = limits; - long expectedRows = Math.Min(expectedBytes / (layout.LineLength + 1), limits.MaxRows); - _columns = [.. layout.Columns.Select(c => new ColumnBuilder(c, limits.MaxTextBytes, expectedRows))]; + _arena = arena; + int firstRow = arena?.RowAt(start) ?? 0; + long expectedRows = arena is null + ? Math.Min((end - start) / (layout.LineLength + 1), limits.MaxRows) + : arena.RowAt(end) - firstRow; + _columns = new ColumnBuilder[layout.Columns.Length]; + for (int i = 0; i < _columns.Length; i++) + { + _columns[i] = new ColumnBuilder(layout.Columns[i], limits.MaxTextBytes, expectedRows, arena is null ? null : arena.Lend(i, firstRow)); + } _requiredLength = layout.Columns.Max(c => c.Start + c.Length); _failedColumn = -1; _following = null; @@ -93,7 +109,7 @@ internal readonly unsafe void Publish(NativeTableRaw* table) table->ColumnCount = _columns.Length; try { - if (chunks.Length == 1) + if (chunks.Length == 1 && _arena is null) { for (int i = 0; i < _columns.Length; i++) { @@ -102,7 +118,7 @@ internal readonly unsafe void Publish(NativeTableRaw* table) } else { - ConcatInParallel(chunks, columns); + ConcatInParallel(chunks, columns, _arena); } table->RowCount = TotalRows; } @@ -115,7 +131,7 @@ internal readonly unsafe void Publish(NativeTableRaw* table) // A concatenated column is all fresh memory, and its page faults are most of the copy's cost: one // column per thread spreads them out. - private static unsafe void ConcatInParallel(ColumnBuilder[][] chunks, NativeColumnRaw* columns) + private static unsafe void ConcatInParallel(ColumnBuilder[][] chunks, NativeColumnRaw* columns, TableArena? arena) { nint target = (nint)columns; try @@ -127,7 +143,7 @@ private static unsafe void ConcatInParallel(ColumnBuilder[][] chunks, NativeColu { parts[c] = chunks[c][i]; } - ColumnBuilder.Concat(parts, (NativeColumnRaw*)target + i); + ColumnBuilder.Concat(parts, (NativeColumnRaw*)target + i, arena?.Block(i)); }); } catch (AggregateException ex) diff --git a/src/FixedWidthParser/Readers/ParallelAggregator.cs b/src/FixedWidthParser/Readers/ParallelAggregator.cs index c216c21..2ad9148 100644 --- a/src/FixedWidthParser/Readers/ParallelAggregator.cs +++ b/src/FixedWidthParser/Readers/ParallelAggregator.cs @@ -38,14 +38,15 @@ internal static async Task RunAsync /// /// Reads [dataStart, source.Length) in partitions, one each, merged in /// order. must start a line; line numbers count from the start of the source. - /// gets the bytes of the partition its sink will read. + /// gets where the bytes of the partition its sink will read start and end; a line + /// belongs to the partition its first byte falls in. /// internal static Task RunAsync( - IRandomAccessSource source, long dataStart, Func create, FixedWidthReaderOptions options, CancellationToken ct) + IRandomAccessSource source, long dataStart, Func create, FixedWidthReaderOptions options, CancellationToken ct) where TSink : struct, ILineSink { return RunPartitionsAsync(source, dataStart, options, - (plan, index) => SinkChunk(source, plan, index, create(plan.End(index) - plan.Start(index)), options, ct), + (plan, index) => SinkChunk(source, plan, index, create(plan.Start(index), plan.End(index)), options, ct), static (merged, following) => { merged.Merge(following); diff --git a/tests/FixedWidthParser.Tests/LineSinkTests.cs b/tests/FixedWidthParser.Tests/LineSinkTests.cs index f966b07..a7c8639 100644 --- a/tests/FixedWidthParser.Tests/LineSinkTests.cs +++ b/tests/FixedWidthParser.Tests/LineSinkTests.cs @@ -66,7 +66,7 @@ public async Task RunAsync_FromDataStart_SkipsEarlierBytes(int chunkSize) { var source = new ArraySource("HDR\nA1\nA2\n"u8.ToArray()); - var lines = await ParallelAggregator.RunAsync(source, dataStart: 4, _ => new Lines(), Options(chunkSize), CancellationToken.None); + var lines = await ParallelAggregator.RunAsync(source, dataStart: 4, (_, _) => new Lines(), Options(chunkSize), CancellationToken.None); Assert.Equal(["A1", "A2"], lines.Values); } @@ -79,7 +79,7 @@ public async Task RunAsync_ErrorLineNumber_CountsFromSourceStart(int chunkSize) var source = new ArraySource("HDR\nA1\nXX\nA3\n"u8.ToArray()); var ex = await Assert.ThrowsAsync(() => - ParallelAggregator.RunAsync(source, dataStart: 4, _ => new Lines(), Options(chunkSize), CancellationToken.None)); + ParallelAggregator.RunAsync(source, dataStart: 4, (_, _) => new Lines(), Options(chunkSize), CancellationToken.None)); Assert.Equal(3, ex.LineNumber); } @@ -91,8 +91,8 @@ public async Task RunAsync_MemorySource_EqualsFileSource() await File.WriteAllBytesAsync(_path, content); using var handle = File.OpenHandle(_path); - var fromFile = await ParallelAggregator.RunAsync(new FileSource(handle), 0, _ => new Lines(), Options(3), CancellationToken.None); - var fromMemory = await ParallelAggregator.RunAsync(new ArraySource(content), 0, _ => new Lines(), Options(3), CancellationToken.None); + var fromFile = await ParallelAggregator.RunAsync(new FileSource(handle), 0, (_, _) => new Lines(), Options(3), CancellationToken.None); + var fromMemory = await ParallelAggregator.RunAsync(new ArraySource(content), 0, (_, _) => new Lines(), Options(3), CancellationToken.None); Assert.Equal(["A1", "A2", "A3"], fromFile.Values); Assert.Equal(fromFile.Values, fromMemory.Values); diff --git a/tests/FixedWidthParser.Tests/Native/NativeParallelReadTests.cs b/tests/FixedWidthParser.Tests/Native/NativeParallelReadTests.cs index 125cfa2..13af12a 100644 --- a/tests/FixedWidthParser.Tests/Native/NativeParallelReadTests.cs +++ b/tests/FixedWidthParser.Tests/Native/NativeParallelReadTests.cs @@ -72,6 +72,38 @@ public void ParallelRead_EqualsSequential(int chunkSize, int degree, bool withBo Assert.Equal(200 - (200 / 13) + 1, parallel.RowCount); } + // Lines of one length with invalid ones among them leave gaps in the shared columns that publishing closes in + // place; a longer first line makes every partition outgrow the room it was lent. + [Theory] + [InlineData(1, false)] + [InlineData(7, false)] + [InlineData(64, false)] + [InlineData(4096, false)] + [InlineData(7, true)] + [InlineData(64, true)] + public void ParallelRead_UniformLines_EqualsSequential(int chunkSize, bool longFirstLine) + { + var layout = Layout(); + var text = new StringBuilder(longFirstLine ? "AB000001000003 trailing\n" : ""); + for (int i = 1; i <= 300; i++) + { + string price = i % 11 == 0 ? "XXXXXX" : (i * 3).ToString("D6", System.Globalization.CultureInfo.InvariantCulture); + text.Append(System.Globalization.CultureInfo.InvariantCulture, $"AB{i % 100:D2}{i % 10000:D4}{price}\n"); + } + byte[] content = Encoding.UTF8.GetBytes(text.ToString()); + + var sequential = FromMemory(content, layout, Options(skipInvalid: true, degreeOfParallelism: 1)); + var parallel = FromMemory(content, layout, Options(skipInvalid: true, degreeOfParallelism: 4), chunkSize); + + Assert.True(parallel.Status == NativeStatus.Ok, parallel.Error); + Assert.Equal(300 - (300 / 11) + (longFirstLine ? 1 : 0), parallel.RowCount); + Assert.Equal(sequential.RowCount, parallel.RowCount); + for (int column = 0; column < 3; column++) + { + Assert.Equal(sequential.Column(column), parallel.Column(column)); + } + } + [Theory] [InlineData(1)] [InlineData(7)] From 3f3b55a769f15ea54580f3353b9c5439eaa293d3 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 22:03:12 -0300 Subject: [PATCH 72/85] ci(release): publish Python wheels to PyPI as fixedwidthparser Follows ExcelReader's release workflow. A v* tag now builds the native library on Windows x64, macOS arm64 and x64, and Linux glibc and musl on x64 and arm64; tests and builds one wheel per platform with the tag's version; publishes the NuGet package, the wheels to PyPI through Trusted Publishing, and the native libraries and an SBOM to the GitHub Release. A tag with a "-" is a prerelease. check_wheel_tag.py and build_musl_wheel.sh come from ExcelReader. pyarrow has no musllinux wheels, so the pandas, polars, Arrow and write tests skip without it and the musl job tests the NumPy read. --- .github/workflows/release.yml | 237 +++++++++++++++++++++++++++-- CHANGELOG.md | 3 + CONTRIBUTING.md | 13 ++ python/README.md | 15 +- python/pyproject.toml | 2 +- python/scripts/build_musl_wheel.sh | 13 ++ python/scripts/check_wheel_tag.py | 29 ++++ python/tests/test_arrow.py | 7 +- python/tests/test_read.py | 5 +- python/tests/test_write.py | 8 +- 10 files changed, 308 insertions(+), 24 deletions(-) create mode 100644 python/scripts/build_musl_wheel.sh create mode 100644 python/scripts/check_wheel_tag.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 12fcb4a..b819e64 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,28 +1,31 @@ name: Release -# Publica no NuGet ao empurrar uma tag de versão (ex.: v1.0.0 -> versão 1.0.0). -# Usa Trusted Publishing (OIDC): nenhuma API key fica armazenada como secret — o GitHub -# troca um token OIDC de curta duração por uma chave temporária via NuGet/login. -# Pré-requisitos no nuget.org: uma policy de Trusted Publishing apontando para este repo + -# workflow, e a variável de repositório NUGET_USER com o usuário do nuget.org. +# Pushing a version tag (v2.0.0, v2.0.0-rc.1) publishes that version everywhere: +# - FixedWidthParser.NET to NuGet, through Trusted Publishing (OIDC): nuget.org needs a policy for this +# repository and workflow, and the repository variable NUGET_USER its user name. +# - fixedwidthparser wheels to PyPI, through Trusted Publishing: PyPI needs a trusted publisher for +# this repository, workflow release.yml. +# - The native library for each platform, attached to the GitHub Release. +# A tag with a "-" is published as a prerelease. on: push: tags: [ 'v*' ] permissions: contents: read - id-token: write # necessário para o OIDC do Trusted Publishing jobs: - publish: - name: Pack & Publish to NuGet + build: + name: Build, test & pack (.NET) runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.value }} steps: - name: Checkout uses: actions/checkout@v7 with: - fetch-depth: 0 # histórico completo para o SourceLink + fetch-depth: 0 # full history for SourceLink - name: Setup .NET uses: actions/setup-dotnet@v6 @@ -60,6 +63,187 @@ jobs: -p:Version=${{ steps.version.outputs.value }} --output ./artifacts + - name: Generate SBOM + uses: anchore/sbom-action@v0 + with: + path: ./artifacts + format: spdx-json + output-file: ./artifacts/fixedwidthparser.spdx.json + upload-release-assets: false + + - name: Upload package artifacts + uses: actions/upload-artifact@v7 + with: + name: nuget-packages + path: ./artifacts/* + if-no-files-found: error + + native: + name: Build native library (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup .NET + uses: actions/setup-dotnet@v6 + with: + dotnet-version: '10.0.x' + + - name: Build the native library + shell: bash + env: + RunAnalyzers: 'false' + run: python python/scripts/build_native.py + + - name: Rename asset for this platform + shell: bash + run: | + case "${{ runner.os }}" in + Windows) src=FixedWidthParser.Native.dll; os=win; ext=dll ;; + macOS) src=FixedWidthParser.Native.dylib; os=osx; ext=dylib ;; + *) src=FixedWidthParser.Native.so; os=linux; ext=so ;; + esac + case "$(uname -m)" in arm64|aarch64) arch=arm64 ;; *) arch=x64 ;; esac + mkdir -p release-assets + cp "python/src/fixedwidth/_lib/${src}" "release-assets/fixedwidth-native-${os}-${arch}.${ext}" + + - name: Upload release asset + uses: actions/upload-artifact@v7 + with: + name: release-native-${{ matrix.os }} + path: release-assets/* + if-no-files-found: error + + - name: Upload library for the Python wheel + uses: actions/upload-artifact@v7 + with: + name: python-lib-${{ matrix.os }} + path: python/src/fixedwidth/_lib/* + if-no-files-found: error + + python: + name: Build Python wheel (${{ matrix.os }}) + needs: [build, native] + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: '3.12' + + - name: Download the native library + uses: actions/download-artifact@v8 + with: + name: python-lib-${{ matrix.os }} + path: python/src/fixedwidth/_lib + + - name: Restore executable bit (Linux/macOS) + if: runner.os != 'Windows' + shell: bash + run: chmod +x python/src/fixedwidth/_lib/* + + - name: Set package version + shell: bash + run: | + sed -i.bak "s/^version = \".*\"/version = \"${{ needs.build.outputs.version }}\"/" python/pyproject.toml + rm -f python/pyproject.toml.bak + + - name: Install test dependencies + run: pip install -e "python[dev,pandas,polars]" + + - name: Test (Python) + run: pytest python/tests + + - name: Build the wheel + run: | + pip install build + python -m build --wheel python/ --outdir dist/ + + - name: Check the wheel installs here + run: python python/scripts/check_wheel_tag.py dist + + - name: Upload wheel artifact + uses: actions/upload-artifact@v7 + with: + name: wheel-${{ matrix.os }} + path: dist/*.whl + if-no-files-found: error + + musl: + name: Build musl library and wheel (${{ matrix.arch }}) + needs: build + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - { arch: x64, runner: ubuntu-latest } + - { arch: arm64, runner: ubuntu-24.04-arm } + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set package version + run: sed -i "s/^version = \".*\"/version = \"${{ needs.build.outputs.version }}\"/" python/pyproject.toml + + - name: Build, test and wheel in Alpine + env: + RunAnalyzers: 'false' + run: sh python/scripts/build_musl_wheel.sh + + - name: Rename asset for this platform + run: | + mkdir -p release-assets + cp python/src/fixedwidth/_lib/FixedWidthParser.Native.so "release-assets/fixedwidth-native-linux-musl-${{ matrix.arch }}.so" + + - name: Upload release asset + uses: actions/upload-artifact@v7 + with: + name: release-native-musl-${{ matrix.arch }} + path: release-assets/* + if-no-files-found: error + + - name: Upload wheel artifact + uses: actions/upload-artifact@v7 + with: + name: wheel-musl-${{ matrix.arch }} + path: dist/*.whl + if-no-files-found: error + + publish: + name: Publish to NuGet & GitHub Release + needs: [build, native, python, musl] + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write + + steps: + - name: Download NuGet packages + uses: actions/download-artifact@v8 + with: + name: nuget-packages + path: artifacts + + - name: Download release assets + uses: actions/download-artifact@v8 + with: + pattern: release-* + path: release-assets + merge-multiple: true + - name: NuGet login (Trusted Publishing / OIDC) uses: NuGet/login@v1 id: nuget-login @@ -73,10 +257,33 @@ jobs: --source https://api.nuget.org/v3/index.json --skip-duplicate - - name: Upload package artifacts - if: always() - uses: actions/upload-artifact@v7 + - name: Upload to the GitHub Release + uses: softprops/action-gh-release@v3 with: - name: nuget-packages - path: ./artifacts/* - if-no-files-found: warn + files: | + release-assets/* + artifacts/*.spdx.json + fail_on_unmatched_files: true + prerelease: ${{ contains(github.ref_name, '-') }} + + publish-pypi: + name: Publish to PyPI + needs: publish + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + + steps: + - name: Download wheel artifacts + uses: actions/download-artifact@v8 + with: + pattern: wheel-* + path: dist/ + merge-multiple: true + + - name: Publish to PyPI (Trusted Publishing) + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ + skip-existing: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f7eabc..40c05f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,9 @@ Windows-1252 after registering `CodePagesEncodingProvider`). - The `fixedwidth` Python package over it (NumPy views, pyarrow, pandas, polars), 52 times faster than `pandas.read_fwf` on a 1,000,000-line COTAHIST-shaped file, and a header-only C++23 wrapper (`fixedwidth.hpp`) that also maps structs to columns through `FixedWidthMapper`. +- Releases publish the Python package to PyPI as `fixedwidthparser`, with wheels for Windows x64, + macOS arm64 and x64, and Linux glibc and musl on x64 and arm64, and attach each platform's native library + to the GitHub Release. - `ref struct` models on `Read`, `ReadFile`, `ReadAsync`, `ReadFileAsync`, `AggregateFileAsync` and `AggregateAsync`, generated and through a layout. The enumerators parse a line when `Current` is read, so reading it twice parses twice; with `OnParseError` set, each line is parsed once more to diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7d6fa88..5b04957 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,6 +92,19 @@ mood, lowercase summary, no trailing period. Common types in this repo: `feat`, dependencies, publishes and runs the NativeAOT smoke app, and runs CodeQL and the benchmark comparison. A change that only builds on one OS isn't ready to merge. +## Releasing + +Push a `v*` tag (`v2.0.0`, or `v2.0.0-rc.1` for a prerelease). [release.yml](.github/workflows/release.yml) +takes the version from the tag, tests and packs the NuGet package, builds the native library on Windows x64, +macOS arm64 and x64, Linux glibc and musl x64 and arm64, tests one Python wheel per platform, then publishes +the package to NuGet, the wheels to PyPI as `fixedwidthparser`, and the native libraries and an SBOM +to the GitHub Release. Both registries publish through Trusted Publishing, so no API keys are stored: + +- nuget.org: a trusted publishing policy for this repository and `release.yml`, and the repository variable + `NUGET_USER` set to the nuget.org user. +- PyPI: a trusted publisher for this repository and `release.yml`. Before the first release, add it as a + pending publisher for the project name `fixedwidthparser`. + ## Reporting bugs / requesting features Use the [issue templates](https://github.com/GabrielMarquezMatte/FixedWidthParser/issues/new/choose). diff --git a/python/README.md b/python/README.md index 1482ccf..445e72b 100644 --- a/python/README.md +++ b/python/README.md @@ -1,8 +1,17 @@ -# fixedwidthparser-native +# fixedwidthparser Read and write fixed-width files from Python through FixedWidthParser's NativeAOT library: no .NET runtime, parallel reads, results as NumPy views, Arrow, pandas or polars. +```bash +pip install "fixedwidthparser[numpy]" # fw.read: NumPy views +pip install "fixedwidthparser[pandas]" # or [pyarrow], [polars] +``` + +Wheels ship for Windows x64, macOS (arm64, x64) and Linux (glibc and musl, x64 and arm64). The package installs +as `fixedwidthparser` and imports as `fixedwidth`. pyarrow has no musllinux wheels, so on Alpine and other +musl systems Arrow, pandas, polars and writing need a pyarrow built for them. + ```python import fixedwidth as fw @@ -72,6 +81,8 @@ A value that is NaN, infinite or outside int64 once scaled raises before anythin ## Building from source - python python/scripts/build_native.py # publishes the library into src/fixedwidth/_lib + python python/scripts/build_native.py # requires the .NET 10 SDK; publishes the library into src/fixedwidth/_lib pip install -e "python[dev,pandas,polars]" pytest python/tests + +To point at a library built elsewhere, set `FIXEDWIDTH_NATIVE_LIB` to its full path. diff --git a/python/pyproject.toml b/python/pyproject.toml index 3f446b4..1cbe236 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -3,7 +3,7 @@ requires = ["hatchling", "packaging"] build-backend = "hatchling.build" [project] -name = "fixedwidthparser-native" +name = "fixedwidthparser" version = "0.1.0" description = "Read and write fixed-width files through the FixedWidthParser NativeAOT library" readme = "README.md" diff --git a/python/scripts/build_musl_wheel.sh b/python/scripts/build_musl_wheel.sh new file mode 100644 index 0000000..b6f8358 --- /dev/null +++ b/python/scripts/build_musl_wheel.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Builds the linux-musl native library and its musllinux wheel in Alpine containers, running the +# Python tests against it. Needs a Linux host with Docker; the wheel lands in dist/. pyarrow has no +# musllinux wheels, so the pandas, polars, Arrow and write tests skip here. +# +# Run from the repository root: sh python/scripts/build_musl_wheel.sh +set -eu + +docker run --rm -e RunAnalyzers -v "$PWD:/repo" -w /repo mcr.microsoft.com/dotnet/sdk:10.0-alpine \ + sh -c 'apk add --no-cache clang build-base zlib-dev python3 && python3 python/scripts/build_native.py' + +docker run --rm -v "$PWD:/repo" -w /repo python:3.12-alpine \ + sh -c 'pip install -e "python[dev,numpy]" build && pytest python/tests && python -m build --wheel python/ --outdir dist/ && python python/scripts/check_wheel_tag.py dist' diff --git a/python/scripts/check_wheel_tag.py b/python/scripts/check_wheel_tag.py new file mode 100644 index 0000000..d5874a0 --- /dev/null +++ b/python/scripts/check_wheel_tag.py @@ -0,0 +1,29 @@ +"""Fails unless every wheel in the given directory installs on this machine. + +Run after building a wheel: python python/scripts/check_wheel_tag.py dist +""" + +import sys +from pathlib import Path + +from packaging.tags import sys_tags +from packaging.utils import parse_wheel_filename + + +def main() -> int: + supported = set(sys_tags()) + wheels = sorted(Path(sys.argv[1]).glob("*.whl")) + if not wheels: + print(f"error: no wheel in {sys.argv[1]}", file=sys.stderr) + return 1 + for wheel in wheels: + tags = parse_wheel_filename(wheel.name)[3] + if tags.isdisjoint(supported): + print(f"error: {wheel.name} does not install on this machine", file=sys.stderr) + return 1 + print(f"ok: {wheel.name}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/tests/test_arrow.py b/python/tests/test_arrow.py index 7ef4713..7f18545 100644 --- a/python/tests/test_arrow.py +++ b/python/tests/test_arrow.py @@ -2,12 +2,15 @@ import datetime as dt -import polars as pl -import pyarrow as pa +import pytest import fixedwidth as fw from conftest import CONTENT, layout +# No pyarrow wheels for musllinux; the musl wheel job tests without it. +pl = pytest.importorskip("polars") +pa = pytest.importorskip("pyarrow") + def test_read_arrow(cotahist, options): table = fw.read_arrow(cotahist, layout(), options) diff --git a/python/tests/test_read.py b/python/tests/test_read.py index d1448d0..edf4772 100644 --- a/python/tests/test_read.py +++ b/python/tests/test_read.py @@ -5,7 +5,6 @@ import io import numpy as np -import pandas as pd import pytest import fixedwidth as fw @@ -84,6 +83,8 @@ def test_views_keep_table_alive(cotahist, options): def test_read_pandas_matches_read_fwf(cotahist, options): + pd = pytest.importorskip("pandas") + pytest.importorskip("pyarrow") frame = fw.read_pandas(cotahist, layout(), options) expected = pd.read_fwf( @@ -105,6 +106,8 @@ def test_read_pandas_matches_read_fwf(cotahist, options): def test_read_polars(cotahist, options): + pytest.importorskip("polars") + pytest.importorskip("pyarrow") frame = fw.read_polars(cotahist, layout(), options) assert frame.shape == (3, 6) diff --git a/python/tests/test_write.py b/python/tests/test_write.py index 7b61c2b..62b6487 100644 --- a/python/tests/test_write.py +++ b/python/tests/test_write.py @@ -2,14 +2,16 @@ import datetime as dt -import pandas as pd -import polars as pl -import pyarrow as pa import pytest import fixedwidth as fw from conftest import CONTENT, layout +# Writing goes through pyarrow, which has no musllinux wheels; the musl wheel job tests without it. +pd = pytest.importorskip("pandas") +pl = pytest.importorskip("polars") +pa = pytest.importorskip("pyarrow") + QUOTES = "".join(line + "\n" for line in CONTENT.splitlines()[1:4]) From 7f47469dbd4207fe574bffb41bd9fc4643f44386 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 22:26:42 -0300 Subject: [PATCH 73/85] ci: run each workflow only when its paths change Follows ExcelReader's workflows. - benchmark-pr is opt-in through the run-benchmarks label, reads its groups from tests/Benchmarks/benchmark-groups.json (shared with benchmark-pages), guards that every benchmark belongs to a group, and comments a delta against the published master series. benchmark-pages runs only when its paths change. The stale IntHeavyParserBenchmarks pattern is gone. - ci filters by path and skips drafts. The required Build & Test jobs always start and skip their steps. Linux builds the solution with the analyzers, checks whitespace, packs and runs every test with coverage; Windows and macOS build and run the library tests without analyzers. The dependency audit moves to audit.yml, weekly and on dependency changes. - codeql filters by path the same way, adds Python, and builds only the shipped projects without analyzers. - native-bindings splits into Python, C++ and musl jobs with their own filters, tests every platform a release ships, and builds the wheel. - codecov.yml flags C# and Python coverage with carryforward; coverage.runsettings leaves out generated code and the C ABI exports. - release promotes PublicAPI.Unshipped to Shipped in a PR after a stable tag. - global.json pins SDK 10.0.402, which every workflow now installs and caches. - Dependabot groups follow the test and benchmark packages in use. --- .github/dependabot.yml | 11 +- .github/scripts/promote_public_api.py | 83 ++++++++++ .github/scripts/test_promote_public_api.py | 77 +++++++++ .github/workflows/audit.yml | 48 ++++++ .github/workflows/benchmark-pages.yml | 71 ++++---- .github/workflows/benchmark-pr.yml | 154 +++++++++++++----- .github/workflows/ci.yml | 140 +++++++++------- .github/workflows/codeql.yml | 66 +++++++- .github/workflows/native-bindings.yml | 105 ++++++++++-- .github/workflows/release.yml | 69 ++++++-- CONTRIBUTING.md | 9 +- codecov.yml | 22 +++ coverage.runsettings | 30 ++++ global.json | 7 +- tests/Benchmarks/Perf/ComparisonBenchmarks.cs | 2 +- tests/Benchmarks/bench_delta.py | 130 +++++++++++++++ tests/Benchmarks/benchmark-groups.json | 32 ++++ tests/Benchmarks/check-benchmark-groups.sh | 44 +++++ tests/FixedWidthParser.Tests/TestModels.cs | 2 +- 19 files changed, 923 insertions(+), 179 deletions(-) create mode 100644 .github/scripts/promote_public_api.py create mode 100644 .github/scripts/test_promote_public_api.py create mode 100644 .github/workflows/audit.yml create mode 100644 codecov.yml create mode 100644 coverage.runsettings create mode 100644 tests/Benchmarks/bench_delta.py create mode 100644 tests/Benchmarks/benchmark-groups.json create mode 100644 tests/Benchmarks/check-benchmark-groups.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 08ad484..c80d8d1 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -21,8 +21,15 @@ updates: test: patterns: - "xunit*" - - "Microsoft.NET.Test.Sdk" - - "coverlet.*" + - "Microsoft.Testing.*" + - "CsCheck" + # The libraries the comparison benchmarks measure against. + benchmarks: + patterns: + - "BenchmarkDotNet" + - "FileHelpers" + - "FlatFiles" + - "RecordParser" labels: - dependencies diff --git a/.github/scripts/promote_public_api.py b/.github/scripts/promote_public_api.py new file mode 100644 index 0000000..5744ffd --- /dev/null +++ b/.github/scripts/promote_public_api.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Moves every entry out of each PublicAPI.Unshipped.txt into its PublicAPI.Shipped.txt. + +Run after a release ships: everything that was "unshipped" at release time is now part of +the public API contract. Operates on every src/*/PublicAPI.Unshipped.txt. + +Exit codes: 0 = promoted something, 2 = nothing needed promoting (not an error — the caller +skips the rest of the job), 1 = an actual failure (an uncaught exception, same as Python's +default for any unhandled error). 2 is deliberately not 1 - an uncaught exception used +to exit 1 exactly like the intentional "nothing to promote" case, so release.yml's `if +python3 ...; then changed=true; else changed=false; fi` silently treated a crash (missing +file, permission error) the same as a no-op success, disarming the next release's +breaking-change detection instead of failing the job. +""" + +import glob +import os +import sys + +HEADER = "#nullable enable" + + +def read_entries(path): + with open(path, encoding="utf-8") as f: + return [line.strip() for line in f if line.strip() and line.strip() != HEADER] + + +def write_entries(path, entries): + with open(path, "w", encoding="utf-8", newline="\n") as f: + f.write(HEADER + "\n") + f.writelines(entry + "\n" for entry in sorted(entries)) + + +REMOVED_PREFIX = "*REMOVED*" + + +def promote_one(unshipped_path, shipped_path): + """Applies unshipped_path's entries onto shipped_path and clears unshipped_path. + + A plain entry is added to Shipped. A `*REMOVED*Foo` entry deletes the plain `Foo` entry + from Shipped (an API removal is meaningful only for one release cycle) and is itself + discarded rather than copied - RS0024 forbids a *REMOVED* line inside Shipped.txt, since + Shipped is the current API surface, not a removal history. + + Returns the number of entries read from unshipped_path (0 if there was nothing to do). + """ + new_entries = read_entries(unshipped_path) + if not new_entries: + return 0 + + shipped_entries = set(read_entries(shipped_path)) if os.path.exists(shipped_path) else set() + for entry in new_entries: + if entry.startswith(REMOVED_PREFIX): + shipped_entries.discard(entry[len(REMOVED_PREFIX):]) + else: + shipped_entries.add(entry) + + write_entries(shipped_path, shipped_entries) + write_entries(unshipped_path, []) + return len(new_entries) + + +def main(): + unshipped_files = sorted(glob.glob("src/*/PublicAPI.Unshipped.txt")) + if not unshipped_files: + print("No src/*/PublicAPI.Unshipped.txt files found.") + return 2 + + promoted_any = False + for unshipped_path in unshipped_files: + shipped_path = unshipped_path.replace("Unshipped.txt", "Shipped.txt") + count = promote_one(unshipped_path, shipped_path) + if count == 0: + print(f"{unshipped_path}: nothing to promote.") + continue + print(f"{unshipped_path}: promoted {count} entries to {shipped_path}.") + promoted_any = True + + return 0 if promoted_any else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/test_promote_public_api.py b/.github/scripts/test_promote_public_api.py new file mode 100644 index 0000000..8cd0ae0 --- /dev/null +++ b/.github/scripts/test_promote_public_api.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Unit tests for promote_public_api.py. + +Run: python3 .github/scripts/test_promote_public_api.py +""" + +import os +import tempfile +import unittest + +import promote_public_api as promote + + +class PromoteTests(unittest.TestCase): + def _write(self, path, lines): + with open(path, "w", encoding="utf-8", newline="\n") as f: + f.write(promote.HEADER + "\n") + f.writelines(line + "\n" for line in lines) + + def _read(self, path): + with open(path, encoding="utf-8") as f: + return {line.strip() for line in f if line.strip() and line.strip() != promote.HEADER} + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.shipped = os.path.join(self._tmp.name, "PublicAPI.Shipped.txt") + self.unshipped = os.path.join(self._tmp.name, "PublicAPI.Unshipped.txt") + + def tearDown(self): + self._tmp.cleanup() + + def test_plain_entry_is_added_to_shipped(self): + self._write(self.shipped, ["Foo.Bar() -> void"]) + self._write(self.unshipped, ["Foo.Baz() -> void"]) + + promote.promote_one(self.unshipped, self.shipped) + + self.assertEqual(self._read(self.shipped), {"Foo.Bar() -> void", "Foo.Baz() -> void"}) + self.assertEqual(self._read(self.unshipped), set()) + + def test_removed_entry_deletes_the_plain_entry_from_shipped(self): + self._write(self.shipped, ["Foo.Bar() -> void", "Foo.Baz() -> void"]) + self._write(self.unshipped, ["*REMOVED*Foo.Bar() -> void"]) + + promote.promote_one(self.unshipped, self.shipped) + + self.assertEqual(self._read(self.shipped), {"Foo.Baz() -> void"}) + self.assertEqual(self._read(self.unshipped), set()) + + def test_removed_entry_is_never_written_to_shipped(self): + self._write(self.shipped, ["Foo.Bar() -> void"]) + self._write(self.unshipped, ["*REMOVED*Foo.Bar() -> void"]) + + promote.promote_one(self.unshipped, self.shipped) + + for entry in self._read(self.shipped): + self.assertFalse(entry.startswith("*REMOVED*"), entry) + + def test_removed_entry_with_no_matching_shipped_entry_is_a_noop_delete(self): + self._write(self.shipped, ["Foo.Bar() -> void"]) + self._write(self.unshipped, ["*REMOVED*Foo.NeverShipped() -> void"]) + + promote.promote_one(self.unshipped, self.shipped) + + self.assertEqual(self._read(self.shipped), {"Foo.Bar() -> void"}) + + def test_mixed_add_and_remove_in_the_same_promotion(self): + self._write(self.shipped, ["Foo.Old() -> void"]) + self._write(self.unshipped, ["*REMOVED*Foo.Old() -> void", "Foo.New() -> void"]) + + promote.promote_one(self.unshipped, self.shipped) + + self.assertEqual(self._read(self.shipped), {"Foo.New() -> void"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml new file mode 100644 index 0000000..a63aa24 --- /dev/null +++ b/.github/workflows/audit.yml @@ -0,0 +1,48 @@ +name: Dependency audit + +# A newly published advisory flags a package without any change here, so the audit runs weekly as well as when +# the dependencies change. +on: + pull_request: + branches: [ master, develop ] + paths: + - '**/*.csproj' + - 'Directory.Build.props' + - 'global.json' + - '.github/workflows/audit.yml' + push: + branches: [ master ] + paths: + - '**/*.csproj' + - 'Directory.Build.props' + - 'global.json' + - '.github/workflows/audit.yml' + schedule: + - cron: '0 6 * * 1' + workflow_dispatch: + +permissions: + contents: read + +jobs: + audit: + name: Dependency audit + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + + - name: Restore + run: dotnet restore FixedWidthParser.slnx + + - name: Check for vulnerable packages + run: | + output=$(dotnet list FixedWidthParser.slnx package --vulnerable --include-transitive) + echo "$output" + if echo "$output" | grep -q -E '\b(High|Critical)\b'; then + echo "::error::High or Critical severity vulnerability found in dependencies." + exit 1 + fi diff --git a/.github/workflows/benchmark-pages.yml b/.github/workflows/benchmark-pages.yml index 9e34c53..451f6bf 100644 --- a/.github/workflows/benchmark-pages.yml +++ b/.github/workflows/benchmark-pages.yml @@ -1,7 +1,8 @@ name: Benchmarks → GitHub Pages -# Runs on every push to master. Benchmark classes are split across parallel -# runners and the results are merged and published to GitHub Pages. +# Runs on each push to master that can move a benchmark. The groups in tests/Benchmarks/benchmark-groups.json +# run on parallel runners, and the results are merged and published to GitHub Pages, where benchmark-pr.yml +# reads them as its baseline. # # Prerequisites (one-time setup in GitHub repo settings): # Settings → Pages → Source: "Deploy from a branch" → Branch: gh-pages / root @@ -11,74 +12,64 @@ name: Benchmarks → GitHub Pages on: push: branches: [master] + paths: + - 'src/FixedWidthParser/**' + - 'src/FixedWidthParser.Generator/**' + - 'tests/Benchmarks/**' + - 'Directory.Build.props' + - 'global.json' + - '.github/workflows/benchmark-pages.yml' concurrency: group: benchmark-pages cancel-in-progress: true permissions: - contents: write # needed to push data to the gh-pages branch + contents: read jobs: + groups: + name: Load benchmark groups + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.load.outputs.matrix }} + steps: + - uses: actions/checkout@v7 + - id: load + run: echo "matrix=$(jq -c . tests/Benchmarks/benchmark-groups.json)" >> "$GITHUB_OUTPUT" + benchmark: name: Run (${{ matrix.group.name }}) + needs: groups runs-on: ubuntu-latest strategy: fail-fast: false matrix: - group: - # Space-separated fully-qualified class patterns. - # Each pattern becomes a separate --filter flag (BenchmarkDotNet ORs them). - - name: parser - filter: >- - Benchmarks.Perf.ParserBenchmarks.* - Benchmarks.Perf.SourceGenParserBenchmarks.* - Benchmarks.Perf.IntHeavyParserBenchmarks.* - - name: reader - filter: >- - Benchmarks.Perf.ReaderBenchmarks.* - Benchmarks.Perf.ReaderStartupBenchmarks.* - Benchmarks.Perf.AsyncReaderBenchmarks.* - - name: byte - filter: >- - Benchmarks.Perf.ReaderPoolingBenchmarks.* - - name: writer - filter: Benchmarks.Perf.WriterBenchmarks.* - - name: comparison - filter: >- - Benchmarks.Perf.ComparisonWriterBenchmarks.* - Benchmarks.Perf.ComparisonBenchmarks.* + group: ${{ fromJSON(needs.groups.outputs.matrix) }} steps: - uses: actions/checkout@v7 - uses: actions/setup-dotnet@v6 with: - dotnet-version: '10.0.x' - - - name: Cache NuGet packages - uses: actions/cache@v6 - with: - path: ~/.nuget/packages - key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj', '**/Directory.Build.props') }} - restore-keys: | - ${{ runner.os }}-nuget- + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' - name: Restore run: dotnet restore tests/Benchmarks/Benchmarks.csproj - name: Build (Release) - run: dotnet build tests/Benchmarks/Benchmarks.csproj --configuration Release --no-restore + run: dotnet build tests/Benchmarks/Benchmarks.csproj --configuration Release --no-restore -p:RunAnalyzers=false - name: Run benchmarks run: | - # --filter accepts multiple space-separated patterns as values of a - # single flag: `--filter Pat1 Pat2 Pat3`. - # Single-quoting the substitution prevents bash glob-expanding the '*'. + # --filter takes several space-separated patterns as values of one flag; single-quoting the + # substitution keeps bash from glob-expanding the '*'. IFS=' ' read -ra patterns <<< '${{ matrix.group.filter }}' dotnet run --project tests/Benchmarks/Benchmarks.csproj \ --configuration Release --no-build \ - -- --filter "${patterns[@]}" --warmupCount 3 --iterationCount 10 + -- --filter "${patterns[@]}" ${{ matrix.group.pagesArgs }} - name: Upload JSON results uses: actions/upload-artifact@v7 @@ -93,6 +84,8 @@ jobs: # Run even when some matrix shards fail so partial results are still published. if: always() && needs.benchmark.result != 'cancelled' runs-on: ubuntu-latest + permissions: + contents: write # pushes to the gh-pages branch steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index 84b0e50..236429f 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -1,16 +1,17 @@ name: Benchmarks → PR Comment -# Runs on every PR push targeting master and develop. -# Results are posted (and updated) as a sticky comment on the PR only — -# no data is written to the gh-pages branch or anywhere else shared. +# Opt-in: runs only on a pull request labelled run-benchmarks whose changes can move a benchmark. The suite +# is several runner-minutes per group, too much to repeat on each push. Results go to a sticky comment on the +# PR, as a delta against the master series benchmark-pages.yml publishes; nothing shared is written. on: pull_request: branches: [master, develop] - types: [opened, synchronize, reopened] + types: [opened, synchronize, reopened, ready_for_review, labeled] concurrency: - group: benchmark-pr-${{ github.event.pull_request.number }} + # Adding any other label must not cancel a run in progress. + group: benchmark-pr-${{ github.event.pull_request.number }}${{ github.event.action == 'labeled' && github.event.label.name != 'run-benchmarks' && format('-{0}', github.run_id) || '' }} cancel-in-progress: true permissions: @@ -18,65 +19,98 @@ permissions: pull-requests: write # needed to post/update the PR comment jobs: + changes: + name: Detect changed paths + if: >- + github.event.pull_request.draft != true && + (github.event.action != 'labeled' || github.event.label.name == 'run-benchmarks') + runs-on: ubuntu-latest + outputs: + benchmarks: ${{ steps.filter.outputs.benchmarks }} + run: ${{ steps.filter.outputs.benchmarks == 'true' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks') }} + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Filter + id: filter + uses: dorny/paths-filter@v4 + with: + filters: | + benchmarks: + - 'src/FixedWidthParser/**' + - 'src/FixedWidthParser.Generator/**' + - 'tests/Benchmarks/**' + - 'Directory.Build.props' + - 'global.json' + - '.github/workflows/benchmark-pr.yml' + + groups: + name: Load benchmark groups + needs: changes + if: needs.changes.outputs.run == 'true' + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.load.outputs.matrix }} + steps: + - uses: actions/checkout@v7 + - id: load + run: echo "matrix=$(jq -c . tests/Benchmarks/benchmark-groups.json)" >> "$GITHUB_OUTPUT" + + # Cheap, so it runs on every PR that touches the benchmarks, labelled or not. + guard: + name: Every benchmark belongs to a group + needs: changes + if: needs.changes.outputs.benchmarks == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' + + - name: Build (Release) + run: dotnet build tests/Benchmarks/Benchmarks.csproj --configuration Release -p:RunAnalyzers=false + + - name: Check group coverage + run: bash tests/Benchmarks/check-benchmark-groups.sh + benchmark: name: Run (${{ matrix.group.name }}) + needs: [changes, groups] + if: needs.changes.outputs.run == 'true' runs-on: ubuntu-latest strategy: fail-fast: false matrix: - group: - # Space-separated fully-qualified class patterns. - # Each pattern becomes a separate --filter flag (BenchmarkDotNet ORs them). - - name: parser - filter: >- - Benchmarks.Perf.ParserBenchmarks.* - Benchmarks.Perf.SourceGenParserBenchmarks.* - Benchmarks.Perf.IntHeavyParserBenchmarks.* - - name: reader - filter: >- - Benchmarks.Perf.ReaderBenchmarks.* - Benchmarks.Perf.ReaderStartupBenchmarks.* - Benchmarks.Perf.AsyncReaderBenchmarks.* - - name: byte - filter: >- - Benchmarks.Perf.ReaderPoolingBenchmarks.* - - name: writer - filter: Benchmarks.Perf.WriterBenchmarks.* - - name: comparison - filter: >- - Benchmarks.Perf.ComparisonWriterBenchmarks.* - Benchmarks.Perf.ComparisonBenchmarks.* + group: ${{ fromJSON(needs.groups.outputs.matrix) }} steps: - uses: actions/checkout@v7 - uses: actions/setup-dotnet@v6 with: - dotnet-version: '10.0.x' - - - name: Cache NuGet packages - uses: actions/cache@v6 - with: - path: ~/.nuget/packages - key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj', '**/Directory.Build.props') }} - restore-keys: | - ${{ runner.os }}-nuget- + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' - name: Restore run: dotnet restore tests/Benchmarks/Benchmarks.csproj - name: Build (Release) - run: dotnet build tests/Benchmarks/Benchmarks.csproj --configuration Release --no-restore + run: dotnet build tests/Benchmarks/Benchmarks.csproj --configuration Release --no-restore -p:RunAnalyzers=false - name: Run benchmarks run: | - # --filter accepts multiple space-separated patterns as values of a - # single flag: `--filter Pat1 Pat2 Pat3`. - # Single-quoting the substitution prevents bash glob-expanding the '*'. + # --filter takes several space-separated patterns as values of one flag; single-quoting the + # substitution keeps bash from glob-expanding the '*'. IFS=' ' read -ra patterns <<< '${{ matrix.group.filter }}' dotnet run --project tests/Benchmarks/Benchmarks.csproj \ --configuration Release --no-build \ - -- --filter "${patterns[@]}" --warmupCount 1 --iterationCount 5 + -- --filter "${patterns[@]}" ${{ matrix.group.prArgs }} - name: Upload markdown results uses: actions/upload-artifact@v7 @@ -85,14 +119,32 @@ jobs: path: BenchmarkDotNet.Artifacts/results/*-report-github.md retention-days: 1 + - name: Upload JSON results + uses: actions/upload-artifact@v7 + with: + name: bench-json-${{ matrix.group.name }} + path: BenchmarkDotNet.Artifacts/results/*-report-full.json + retention-days: 1 + comment: name: Post PR Comment - needs: benchmark + needs: [changes, benchmark] # Post partial results even when some shards fail. - if: always() && needs.benchmark.result != 'cancelled' + if: always() && needs.changes.outputs.run == 'true' && needs.benchmark.result != 'cancelled' runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Checkout the published benchmark series + continue-on-error: true + uses: actions/checkout@v7 + with: + ref: gh-pages + path: gh-pages + fetch-depth: 1 + - name: Download all markdown results uses: actions/download-artifact@v8 with: @@ -100,14 +152,25 @@ jobs: merge-multiple: true path: results/ + - name: Download all JSON results + uses: actions/download-artifact@v8 + with: + pattern: bench-json-* + merge-multiple: true + path: results-json/ + - name: Build comment body id: build run: | + delta=(--results results-json) + if [ -f gh-pages/dev/bench/data.js ]; then + delta+=(--baseline gh-pages/dev/bench/data.js) + fi { echo 'body<Full results per group' echo '' for f in results/*-report-github.md; do [ -f "$f" ] || continue @@ -117,6 +180,7 @@ jobs: cat "$f" echo '' done + echo '' echo 'BENCHMARK_EOF' } >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a4f7b0..20d1149 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,10 @@ name: CI on: + # develop is covered by its pull request into master; a push trigger there would run everything twice. pull_request: branches: [ master, develop ] + types: [ opened, synchronize, reopened, ready_for_review ] push: branches: [ master ] @@ -14,55 +16,95 @@ permissions: contents: read jobs: + changes: + name: Detect changed paths + if: github.event.pull_request.draft != true + runs-on: ubuntu-latest + outputs: + dotnet: ${{ github.event_name != 'pull_request' || steps.filter.outputs.dotnet == 'true' }} + aot: ${{ github.event_name != 'pull_request' || steps.filter.outputs.aot == 'true' }} + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Filter + id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@v4 + with: + filters: | + dotnet: + - 'src/**' + - 'tests/**' + - 'Directory.Build.props' + - 'global.json' + - 'FixedWidthParser.slnx' + - '.editorconfig' + - 'coverage.runsettings' + - '.github/workflows/ci.yml' + aot: + - 'src/FixedWidthParser/**' + - 'src/FixedWidthParser.Generator/**' + - 'tests/AotSmoke/**' + - 'Directory.Build.props' + - 'global.json' + - '.github/workflows/ci.yml' + + # "Build & Test ()" are required checks on master, and a check whose job never starts stays pending, + # so the job always runs and skips its steps when no .NET source changed. test: name: Build & Test (${{ matrix.os }}) + needs: changes runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] + env: + RUN: ${{ needs.changes.outputs.dotnet }} + LINUX: ${{ matrix.os == 'ubuntu-latest' }} steps: + - name: Nothing to build + if: env.RUN != 'true' + run: echo "No .NET sources changed; skipping build and tests." + - name: Checkout + if: env.RUN == 'true' uses: actions/checkout@v7 - name: Setup .NET + if: env.RUN == 'true' uses: actions/setup-dotnet@v6 with: - dotnet-version: '10.0.x' - - - name: Cache NuGet packages - uses: actions/cache@v6 - with: - path: ~/.nuget/packages - key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj', '**/Directory.Build.props') }} - restore-keys: | - ${{ runner.os }}-nuget- - - - name: Restore - run: dotnet restore FixedWidthParser.slnx - - - name: Build (Release) - run: dotnet build FixedWidthParser.slnx --configuration Release --no-restore -p:DeterministicSourcePaths=false - - - name: Pack (validates against the baseline when package validation is on) - if: matrix.os == 'ubuntu-latest' - run: dotnet pack src/FixedWidthParser/FixedWidthParser.csproj --configuration Release --no-restore --output ./package-validation - - - name: Test (Release) + collect coverage - run: >- - dotnet test --solution FixedWidthParser.slnx - --results-directory ./TestResults - --configuration Release - --verbosity normal - --no-build - -- - --report-xunit-trx - --coverage - --coverage-output-format cobertura + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' + + # Linux builds the whole solution with the analyzers, which are about two thirds of the compile and report + # the same on every OS, packs it and runs the generator tests, which are pure Roslyn. Windows and macOS + # build and run only the library tests, without the analyzers; source generators still run. + - name: Build and test the solution (Linux) + if: env.RUN == 'true' && env.LINUX == 'true' + run: | + dotnet restore FixedWidthParser.slnx + dotnet build FixedWidthParser.slnx --configuration Release --no-restore -p:DeterministicSourcePaths=false + dotnet format whitespace FixedWidthParser.slnx --verify-no-changes + # Packing validates the package against the last release when package validation is on. + dotnet pack src/FixedWidthParser/FixedWidthParser.csproj --configuration Release --no-restore --output ./package-validation + dotnet test --solution FixedWidthParser.slnx --results-directory ./TestResults --configuration Release --no-build \ + -- --report-xunit-trx --coverage --coverage-settings coverage.runsettings --coverage-output-format cobertura + + - name: Build and test the library (Windows, macOS) + if: env.RUN == 'true' && env.LINUX != 'true' + shell: bash + run: | + dotnet build tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj --configuration Release -p:RunAnalyzers=false + dotnet test --project tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj --results-directory ./TestResults \ + --configuration Release --no-build -- --report-xunit-trx - name: Upload test results - if: always() + if: always() && env.RUN == 'true' uses: actions/upload-artifact@v7 with: name: test-results-${{ matrix.os }} @@ -70,15 +112,18 @@ jobs: if-no-files-found: ignore - name: Upload coverage to Codecov - if: matrix.os == 'ubuntu-latest' + if: env.RUN == 'true' && env.LINUX == 'true' uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./TestResults/*.cobertura.xml + flags: csharp fail_ci_if_error: false aot-smoke: name: NativeAOT smoke (win-x64) + needs: changes + if: needs.changes.outputs.aot == 'true' runs-on: windows-latest steps: - name: Checkout @@ -87,33 +132,12 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v6 with: - dotnet-version: '10.0.x' + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' - name: Publish AotSmoke (PublishAot) - run: dotnet publish tests/AotSmoke/AotSmoke.csproj --configuration Release --runtime win-x64 -p:IlcTreatWarningsAsErrors=true + run: dotnet publish tests/AotSmoke/AotSmoke.csproj --configuration Release --runtime win-x64 -p:IlcTreatWarningsAsErrors=true -p:RunAnalyzers=false - name: Run the published binary run: ./tests/AotSmoke/bin/Release/net10.0/win-x64/publish/AotSmoke.exe - - audit: - name: Dependency audit - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - - uses: actions/setup-dotnet@v6 - with: - dotnet-version: '10.0.x' - - - name: Restore - run: dotnet restore FixedWidthParser.slnx - - - name: Check for vulnerable packages - run: | - echo "Scanning for known-vulnerable packages…" - output=$(dotnet list FixedWidthParser.slnx package --vulnerable --include-transitive) - echo "$output" - if echo "$output" | grep -q -E '\b(High|Critical)\b'; then - echo "::error::High or Critical severity vulnerability found in dependencies." - exit 1 - fi diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index af31a41..d108ca3 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,10 +1,12 @@ name: CodeQL on: + # develop is covered by its pull request into master; a push trigger there would run everything twice. push: branches: [ master ] pull_request: branches: [ master, develop ] + types: [ opened, synchronize, reopened, ready_for_review ] schedule: # Weekly scan (Mondays 06:00 UTC) to catch newly published query updates. - cron: '0 6 * * 1' @@ -18,30 +20,82 @@ permissions: security-events: write # needed to upload results to the Security tab jobs: + changes: + name: Detect changed languages + if: github.event.pull_request.draft != true + runs-on: ubuntu-latest + outputs: + csharp: ${{ github.event_name != 'pull_request' || steps.filter.outputs.csharp == 'true' }} + python: ${{ github.event_name != 'pull_request' || steps.filter.outputs.python == 'true' }} + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Filter + id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@v4 + with: + filters: | + csharp: + - 'src/**' + - 'Directory.Build.props' + - 'global.json' + - '.github/workflows/codeql.yml' + python: + - 'python/**' + - '.github/workflows/codeql.yml' + + # Every language always gets a job: "Analyze (C#)" is a required check, and a check whose job never starts + # stays pending. An unchanged language's job skips its steps instead. analyze: - name: Analyze (C#) + name: Analyze (${{ matrix.name }}) + needs: changes runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - { language: csharp, name: 'C#' } + - { language: python, name: Python } + env: + RUN: ${{ needs.changes.outputs[matrix.language] }} steps: + - name: Skip (language unchanged) + if: env.RUN != 'true' + run: echo "No ${{ matrix.name }} changes; skipping the analysis." + - name: Checkout + if: env.RUN == 'true' uses: actions/checkout@v7 - name: Setup .NET + if: env.RUN == 'true' && matrix.language == 'csharp' uses: actions/setup-dotnet@v6 with: - dotnet-version: '10.0.x' + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' - name: Initialize CodeQL + if: env.RUN == 'true' uses: github/codeql-action/init@v4 with: - languages: csharp + languages: ${{ matrix.language }} queries: security-extended - # Manual build so CodeQL sees the source generator + analyzers correctly. + # Only the shipped projects: tests and benchmarks double the build for code no one runs against untrusted + # input. Building them by hand lets CodeQL see the source generator's output. - name: Build (Release) - run: dotnet build FixedWidthParser.slnx --configuration Release + if: env.RUN == 'true' && matrix.language == 'csharp' + run: | + for project in src/*/*.csproj; do + dotnet build "$project" --configuration Release -p:RunAnalyzers=false + done - name: Perform CodeQL Analysis + if: env.RUN == 'true' uses: github/codeql-action/analyze@v4 with: - category: "/language:csharp" + category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/native-bindings.yml b/.github/workflows/native-bindings.yml index 92f49af..66867ef 100644 --- a/.github/workflows/native-bindings.yml +++ b/.github/workflows/native-bindings.yml @@ -1,6 +1,7 @@ name: Native bindings on: + # develop is covered by its pull request into master; a push trigger there would run everything twice. push: branches: [master] pull_request: @@ -20,33 +21,45 @@ jobs: if: github.event.pull_request.draft != true runs-on: ubuntu-latest outputs: - bindings: ${{ steps.filter.outputs.bindings }} + native: ${{ steps.filter.outputs.native }} + python: ${{ steps.filter.outputs.python }} + cpp: ${{ steps.filter.outputs.cpp }} steps: - name: Checkout uses: actions/checkout@v7 + # A change to the native library tests every binding; a change to one binding tests only that one. - name: Filter id: filter uses: dorny/paths-filter@v4 with: filters: | - bindings: + native: - 'src/FixedWidthParser/**' - 'src/FixedWidthParser.Native/**' + - 'Directory.Build.props' + - 'global.json' + - 'python/scripts/build_native.py' + - '.github/workflows/native-bindings.yml' + python: - 'python/**' - - 'cpp/**' - 'tests/FixedWidthParser.NativeSmoke/**' - - '.github/workflows/native-bindings.yml' + cpp: + - 'cpp/**' + # Every platform a release ships, so a release never builds one for the first time. build-native: name: Build native library (${{ matrix.os }}) needs: changes - if: needs.changes.outputs.bindings == 'true' + if: >- + needs.changes.outputs.native == 'true' || + needs.changes.outputs.python == 'true' || + needs.changes.outputs.cpp == 'true' runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] steps: - name: Checkout uses: actions/checkout@v7 @@ -54,7 +67,9 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v6 with: - dotnet-version: '10.0.x' + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' - name: Publish the native library into the Python package shell: bash @@ -69,15 +84,15 @@ jobs: path: python/src/fixedwidth/_lib/* if-no-files-found: error - test-bindings: - name: Python + C + C++ (${{ matrix.os }}) + test-python: + name: Python + C smoke (${{ matrix.os }}) needs: [changes, build-native] - if: needs.changes.outputs.bindings == 'true' + if: needs.changes.outputs.native == 'true' || needs.changes.outputs.python == 'true' runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] steps: - name: Checkout uses: actions/checkout@v7 @@ -101,10 +116,27 @@ jobs: run: chmod +x python/src/fixedwidth/_lib/* - name: Install the Python package - run: pip install -e "python[dev,pandas,polars]" + run: pip install -e "python[dev,pandas,polars]" ${{ matrix.os == 'ubuntu-latest' && 'pytest-cov' || '' }} - name: Run the Python tests - run: pytest python/tests -v + run: >- + pytest python/tests -v + ${{ matrix.os == 'ubuntu-latest' && '--cov=fixedwidth --cov-report=xml:coverage-python.xml' || '' }} + + - name: Upload Python coverage to Codecov + if: matrix.os == 'ubuntu-latest' + uses: codecov/codecov-action@v7 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage-python.xml + flags: python + fail_ci_if_error: false + + - name: Build the wheel and check it installs here + run: | + pip install build + python -m build --wheel python/ --outdir dist/ + python python/scripts/check_wheel_tag.py dist - name: Run the Python benchmark (smoke) run: python python/benchmarks/bench_read.py --lines 2000 --runs 1 @@ -120,6 +152,53 @@ jobs: cmake --build build/nativesmoke --config Release ctest --test-dir build/nativesmoke --output-on-failure -C Release + test-musl: + name: musl library + Python (${{ matrix.runner }}) + needs: changes + if: needs.changes.outputs.native == 'true' || needs.changes.outputs.python == 'true' + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + runner: [ubuntu-latest, ubuntu-24.04-arm] + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Build, test and wheel in Alpine + env: + RunAnalyzers: 'false' + run: sh python/scripts/build_musl_wheel.sh + + test-cpp: + name: C++ wrapper (${{ matrix.os }}) + needs: [changes, build-native] + if: needs.changes.outputs.native == 'true' || needs.changes.outputs.cpp == 'true' + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Download native library + uses: actions/download-artifact@v8 + with: + name: native-lib-${{ matrix.os }} + path: python/src/fixedwidth/_lib + + - name: Restore executable bit (Linux/macOS) + if: runner.os != 'Windows' + shell: bash + run: chmod +x python/src/fixedwidth/_lib/* + + # lib.exe, which turns fixedwidth.def into an import library, is only on PATH in a developer prompt. + - name: MSVC developer environment (Windows) + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + - name: Build and run the C++ tests run: | cmake -S cpp -B build/cpp -DCMAKE_BUILD_TYPE=Release diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b819e64..8b62ab6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,15 +30,9 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v6 with: - dotnet-version: '10.0.x' - - - name: Cache NuGet packages - uses: actions/cache@v6 - with: - path: ~/.nuget/packages - key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj', '**/Directory.Build.props') }} - restore-keys: | - ${{ runner.os }}-nuget- + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' - name: Derive version from tag id: version @@ -92,7 +86,9 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v6 with: - dotnet-version: '10.0.x' + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' - name: Build the native library shell: bash @@ -287,3 +283,56 @@ jobs: with: packages-dir: dist/ skip-existing: true + + promote-public-api: + name: Promote PublicAPI.Unshipped -> Shipped + needs: publish + if: ${{ !contains(github.ref_name, '-') }} + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + + steps: + - name: Checkout develop + uses: actions/checkout@v7 + with: + ref: develop + + - name: Test the promotion script + working-directory: .github/scripts + run: python3 test_promote_public_api.py -v + + - name: Promote unshipped API entries + id: promote + run: | + set +e + python3 .github/scripts/promote_public_api.py + code=$? + set -e + # 2 means nothing to promote, a real no-op. Any other nonzero code is a crash (a missing file, a + # permission error) and must fail the job rather than pass as changed=false. + if [ "$code" -eq 0 ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + elif [ "$code" -eq 2 ]; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + exit "$code" + fi + + - name: Open PR + if: steps.promote.outputs.changed == 'true' + uses: peter-evans/create-pull-request@v8 + with: + commit-message: "chore: promote public API for ${{ github.ref_name }}" + title: "chore: promote public API for ${{ github.ref_name }}" + body: >- + Everything that was in `PublicAPI.Unshipped.txt` when `${{ github.ref_name }}` was + published is now part of the shipped contract. Moves it into `PublicAPI.Shipped.txt` + so future removals and signature changes get flagged as breaking. + branch: chore/promote-public-api-${{ github.ref_name }} + base: develop + delete-branch: true + add-paths: | + src/*/PublicAPI.Shipped.txt + src/*/PublicAPI.Unshipped.txt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5b04957..97c2380 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -88,9 +88,12 @@ mood, lowercase summary, no trailing period. Common types in this repo: `feat`, - If a change is user-visible (new API, behavior change, performance claim), add it to [CHANGELOG.md](CHANGELOG.md) and mention it in the PR description. - CI builds and tests on Linux, Windows, and macOS: the .NET suite, and the native library with its - Python, C and C++ bindings. It also validates the package against the last release, audits - dependencies, publishes and runs the NativeAOT smoke app, and runs CodeQL and the benchmark - comparison. A change that only builds on one OS isn't ready to merge. + Python, C and C++ bindings on every platform a release ships. It also checks whitespace formatting, + validates the package against the last release, publishes and runs the NativeAOT smoke app, and runs + CodeQL. Each job runs only when the paths it covers change, and draft PRs run nothing. A change that only + builds on one OS isn't ready to merge. +- Benchmarks are opt-in: label the PR `run-benchmarks` to get a comment comparing it with the latest master + run. A new benchmark class must be added to a group in `tests/Benchmarks/benchmark-groups.json`. ## Releasing diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..1d2eab8 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,22 @@ +coverage: + status: + project: + csharp: + flags: + - csharp + python: + flags: + - python + +# Each language uploads under its own flag from a path-filtered job, so a flag a commit did not run keeps its +# last report instead of dropping to nothing. +flag_management: + default_rules: + carryforward: true + individual_flags: + - name: csharp + paths: + - src/ + - name: python + paths: + - python/ diff --git a/coverage.runsettings b/coverage.runsettings new file mode 100644 index 0000000..291aac2 --- /dev/null +++ b/coverage.runsettings @@ -0,0 +1,30 @@ + + + + + + + + + + + .*[\/]obj[\/].* + + .*[\/]FixedWidthParser\.Native[\/]Exports(\.\w+)?\.cs + + + + + *<.*>.* + + + + + + + + diff --git a/global.json b/global.json index 8f73781..ce2af11 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,10 @@ { + "sdk": { + "version": "10.0.402", + "rollForward": "latestMinor", + "allowPrerelease": false + }, "test": { "runner": "Microsoft.Testing.Platform" } -} \ No newline at end of file +} diff --git a/tests/Benchmarks/Perf/ComparisonBenchmarks.cs b/tests/Benchmarks/Perf/ComparisonBenchmarks.cs index 38d5dad..39b10e7 100644 --- a/tests/Benchmarks/Perf/ComparisonBenchmarks.cs +++ b/tests/Benchmarks/Perf/ComparisonBenchmarks.cs @@ -58,7 +58,7 @@ private static FixedLengthReaderOptions CreateReader() .Map(m => m.Age, 10, 5) .Map(m => m.Salary, 15, 10) .Build(); - return new (){ Parser = parser.Parse, ParallelismOptions = new() { Enabled = false } }; + return new() { Parser = parser.Parse, ParallelismOptions = new() { Enabled = false } }; } [GlobalSetup] diff --git a/tests/Benchmarks/bench_delta.py b/tests/Benchmarks/bench_delta.py new file mode 100644 index 0000000..8bd2301 --- /dev/null +++ b/tests/Benchmarks/bench_delta.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Renders the PR benchmark comment as a delta against the series published to GitHub Pages. + +The PR job measures only the pull request's own build, so its absolute numbers carry no verdict on +their own: runner variance on a shared machine swamps most real changes. benchmark-pages.yml already +publishes a per-commit series for master, so a baseline costs one file read rather than a second +benchmark run. + +Usage: bench_delta.py --results [--baseline ] [--threshold 10] +""" + +from __future__ import annotations + +import argparse +import glob +import json +import os + + +def load_baseline(path: str | None) -> tuple[str | None, dict[str, float]]: + """The newest run's measurements, as (commit sha, {benchmark name: mean ns}).""" + if not path or not os.path.exists(path): + return None, {} + with open(path, encoding="utf-8") as handle: + raw = handle.read() + data = json.loads(raw[raw.index("{"):]) + runs = data.get("entries", {}).get("Benchmark", []) + if not runs: + return None, {} + newest = runs[-1] + measurements = {bench["name"]: bench["value"] for bench in newest.get("benches", [])} + return newest.get("commit", {}).get("id"), measurements + + +def load_results(directory: str) -> tuple[dict[str, float], list[str]]: + """(measured means, benchmarks that ran but produced no measurement). FullName matches the + baseline's `name` verbatim. BenchmarkDotNet writes null Statistics when the benchmark process + crashed or threw.""" + results: dict[str, float] = {} + failed: list[str] = [] + for path in sorted(glob.glob(os.path.join(directory, "*-report-full.json"))): + with open(path, encoding="utf-8") as handle: + for bench in json.load(handle).get("Benchmarks", []): + statistics = bench.get("Statistics") + if statistics is None: + failed.append(bench["FullName"]) + else: + results[bench["FullName"]] = statistics["Mean"] + return results, failed + + +def humanize(nanoseconds: float) -> str: + for unit, scale in (("s", 1e9), ("ms", 1e6), ("µs", 1e3)): + if nanoseconds >= scale: + return f"{nanoseconds / scale:.2f} {unit}" + return f"{nanoseconds:.0f} ns" + + +def render(baseline_sha, baseline, results, threshold, failed=()) -> str: + lines = ["## Benchmark Results", ""] + if failed: + lines += ["### Failed to run", "", "These produced no measurement; check the job log for the crash or exception.", ""] + lines += [f"- `{name}`" for name in failed] + lines.append("") + if not baseline: + lines += [ + ("_No published baseline was found, so these are absolute numbers only. They come from a " + "shared `ubuntu-latest` runner and cannot be compared against another machine's._"), + "", + ] + return "\n".join(lines) + + lines += [ + (f"_Baseline: master [`{baseline_sha[:8]}`](../commit/{baseline_sha}), from the series " + "published by `benchmark-pages.yml`. Both sides ran on a shared `ubuntu-latest` runner, so " + f"treat anything under ±{threshold:g}% as noise._"), + "", + ] + + moved, steady, unmatched = [], 0, 0 + for name, current in results.items(): + before = baseline.get(name) + if before is None: + unmatched += 1 + continue + if before == 0: + continue + percent = (current - before) / before * 100 + if abs(percent) >= threshold: + moved.append((percent, name, before, current)) + else: + steady += 1 + + if moved: + moved.sort(reverse=True) + lines += [ + f"### Moved by at least {threshold:g}%", + "", + "| Benchmark | master | This PR | Delta |", + "|---|---:|---:|---:|", + ] + lines += [ + f"| `{name}` | {humanize(before)} | {humanize(current)} | {percent:+.1f}% |" + for percent, name, before, current in moved + ] + lines.append("") + else: + lines += [f"No benchmark moved by {threshold:g}% or more.", ""] + + summary = [f"{steady} within ±{threshold:g}%"] + if unmatched: + summary.append(f"{unmatched} with no baseline entry (new, renamed, or reparameterized)") + lines += ["_" + ", ".join(summary) + "._", ""] + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--results", required=True, help="Directory holding *-report-full.json") + parser.add_argument("--baseline", help="gh-pages dev/bench/data.js; absent means no comparison") + parser.add_argument("--threshold", type=float, default=10.0, help="Percent change worth reporting") + args = parser.parse_args() + + baseline_sha, baseline = load_baseline(args.baseline) + results, failed = load_results(args.results) + print(render(baseline_sha, baseline, results, args.threshold, failed)) + + +if __name__ == "__main__": + main() diff --git a/tests/Benchmarks/benchmark-groups.json b/tests/Benchmarks/benchmark-groups.json new file mode 100644 index 0000000..46ebe46 --- /dev/null +++ b/tests/Benchmarks/benchmark-groups.json @@ -0,0 +1,32 @@ +[ + { + "name": "parser", + "filter": "Benchmarks.Perf.ParserBenchmarks.* Benchmarks.Perf.SourceGenParserBenchmarks.*", + "pagesArgs": "--warmupCount 3 --iterationCount 10", + "prArgs": "--warmupCount 1 --iterationCount 5" + }, + { + "name": "reader", + "filter": "Benchmarks.Perf.ReaderBenchmarks.* Benchmarks.Perf.ReaderStartupBenchmarks.* Benchmarks.Perf.AsyncReaderBenchmarks.*", + "pagesArgs": "--warmupCount 3 --iterationCount 10", + "prArgs": "--warmupCount 1 --iterationCount 5" + }, + { + "name": "byte", + "filter": "Benchmarks.Perf.ReaderPoolingBenchmarks.*", + "pagesArgs": "--warmupCount 3 --iterationCount 10", + "prArgs": "--warmupCount 1 --iterationCount 5" + }, + { + "name": "writer", + "filter": "Benchmarks.Perf.WriterBenchmarks.*", + "pagesArgs": "--warmupCount 3 --iterationCount 10", + "prArgs": "--warmupCount 1 --iterationCount 5" + }, + { + "name": "comparison", + "filter": "Benchmarks.Perf.ComparisonWriterBenchmarks.* Benchmarks.Perf.ComparisonBenchmarks.*", + "pagesArgs": "--warmupCount 3 --iterationCount 10", + "prArgs": "--warmupCount 1 --iterationCount 5" + } +] diff --git a/tests/Benchmarks/check-benchmark-groups.sh b/tests/Benchmarks/check-benchmark-groups.sh new file mode 100644 index 0000000..f77d769 --- /dev/null +++ b/tests/Benchmarks/check-benchmark-groups.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +groups_file="$here/benchmark-groups.json" +project="$here/Benchmarks.csproj" + +filters="$(python3 -c ' +import json, sys +with open(sys.argv[1], encoding="utf-8") as f: + for group in json.load(f): + for pattern in group["filter"].split(): + print(pattern) +' "$groups_file" | tr -d '\r')" + +ids="$(dotnet run --project "$project" -c Release --no-build -- --list flat | + tr -d '\r' | grep '^Benchmarks\.')" + +regexes=() +while IFS= read -r filter; do + escaped=${filter//./\\.} + regexes+=("^${escaped//\*/.*}$") +done <<<"$filters" + +missing=() +while IFS= read -r id; do + matched=0 + for regex in "${regexes[@]}"; do + if [[ $id =~ $regex ]]; then + matched=1 + break + fi + done + ((matched)) || missing+=("$id") +done <<<"$ids" + +if ((${#missing[@]})); then + echo "Benchmarks matched by no group in benchmark-groups.json:" >&2 + printf ' %s\n' "${missing[@]}" >&2 + echo "Add them to a group, or add a new group." >&2 + exit 1 +fi + +echo "Every benchmark is covered by a group." diff --git a/tests/FixedWidthParser.Tests/TestModels.cs b/tests/FixedWidthParser.Tests/TestModels.cs index f44428c..9f97fe3 100644 --- a/tests/FixedWidthParser.Tests/TestModels.cs +++ b/tests/FixedWidthParser.Tests/TestModels.cs @@ -461,4 +461,4 @@ public sealed record class MutableOptionsModel [FixedColumn(53, 8, Format = "yyyyMMdd")] public DateOnly? NullableDate { get; set; } [FixedColumn(61, 8, Converter = typeof(CentsConverter))] public CentsValue? NullableCents { get; set; } } -} \ No newline at end of file +} From 0d1845234c9f6b777259c67d36ca1b6a0c88b3ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:33:45 +0000 Subject: [PATCH 74/85] Bump Microsoft.Testing.Extensions.TrxReport from 2.3.3 to 2.5.1 --- updated-dependencies: - dependency-name: Microsoft.Testing.Extensions.TrxReport dependency-version: 2.5.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .../FixedWidthParser.Generator.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj b/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj index 68d9094..29760e0 100644 --- a/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj +++ b/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj @@ -14,7 +14,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all From e632b5aedba9758dad484e7fb32995511102c7fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:34:14 +0000 Subject: [PATCH 75/85] Bump Microsoft.Testing.Extensions.TrxReport from 2.3.3 to 2.5.1 --- updated-dependencies: - dependency-name: Microsoft.Testing.Extensions.TrxReport dependency-version: 2.5.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj index b7aed9a..160ccbd 100644 --- a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj +++ b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj @@ -23,7 +23,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all From a8a2c9ac6111f4a7c3985207c8e4d547ee8dec45 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 22:34:32 -0300 Subject: [PATCH 76/85] ci: add result job to enforce binding checks in workflow --- .github/workflows/native-bindings.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/native-bindings.yml b/.github/workflows/native-bindings.yml index 66867ef..8938e8d 100644 --- a/.github/workflows/native-bindings.yml +++ b/.github/workflows/native-bindings.yml @@ -204,3 +204,17 @@ jobs: cmake -S cpp -B build/cpp -DCMAKE_BUILD_TYPE=Release cmake --build build/cpp --config Release ctest --test-dir build/cpp --output-on-failure -C Release + + # The one check branch protection requires from this workflow. A matrix job skipped by its `if` never reports + # its expanded name, so requiring those names would leave the check pending whenever no binding changed. + result: + name: Bindings result + needs: [changes, build-native, test-python, test-musl, test-cpp] + if: always() && github.event.pull_request.draft != true + runs-on: ubuntu-latest + steps: + - name: Fail if any job failed or was cancelled + if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: | + echo "A bindings job failed or was cancelled." + exit 1 From 3e3dfe3264e5921938aed35094453b83d00590ca Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sat, 24 Oct 2026 23:25:58 -0300 Subject: [PATCH 77/85] feat(cpp): install as a CMake package and download the release's native library Follows ExcelReader's C++ packaging. The wrapper resolves the native library from FIXEDWIDTH_NATIVE_LIB, then the repo's published copy, then the GitHub Release asset for FIXEDWIDTH_VERSION (the checked-out tag by default), saved under the library's own name so the Windows import library and the Linux SONAME match it. Consumers need no .NET SDK. FIXEDWIDTH_INSTALL installs the headers, the library, the Windows import library and a fixedwidth config package; find_package(fixedwidth) never downloads. cpp/tests/package builds against an installed prefix, and the C++ CI job now installs and runs it on Linux, Windows and macOS. The README covers FetchContent, the installed package and the settings. --- .github/workflows/native-bindings.yml | 9 +++ CHANGELOG.md | 4 +- cpp/CMakeLists.txt | 95 ++++++++++++++++++++++----- cpp/README.md | 58 ++++++++++++++-- cpp/cmake/CopyNativeLibrary.cmake | 15 +++++ cpp/cmake/NativeLib.cmake | 68 +++++++++++++++---- cpp/cmake/fixedwidthConfig.cmake.in | 26 ++++++++ cpp/tests/package/CMakeLists.txt | 16 +++++ cpp/tests/package/main.cpp | 32 +++++++++ 9 files changed, 289 insertions(+), 34 deletions(-) create mode 100644 cpp/cmake/CopyNativeLibrary.cmake create mode 100644 cpp/cmake/fixedwidthConfig.cmake.in create mode 100644 cpp/tests/package/CMakeLists.txt create mode 100644 cpp/tests/package/main.cpp diff --git a/.github/workflows/native-bindings.yml b/.github/workflows/native-bindings.yml index 8938e8d..2ee8c7e 100644 --- a/.github/workflows/native-bindings.yml +++ b/.github/workflows/native-bindings.yml @@ -205,6 +205,15 @@ jobs: cmake --build build/cpp --config Release ctest --test-dir build/cpp --output-on-failure -C Release + - name: Install the package and build a find_package() consumer against it + shell: bash + run: | + cmake --install build/cpp --config Release --prefix "${{ github.workspace }}/build/prefix" + cmake -S cpp/tests/package -B build/consumer -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${{ github.workspace }}/build/prefix" + cmake --build build/consumer --config Release + ctest --test-dir build/consumer --output-on-failure -C Release + # The one check branch protection requires from this workflow. A matrix job skipped by its `if` never reports # its expanded name, so requiring those names would leave the check pending whenever no binding changed. result: diff --git a/CHANGELOG.md b/CHANGELOG.md index 40c05f0..6788c55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,7 +69,9 @@ Windows-1252 after registering `CodePagesEncodingProvider`). thread count. Not on NuGet. - The `fixedwidth` Python package over it (NumPy views, pyarrow, pandas, polars), 52 times faster than `pandas.read_fwf` on a 1,000,000-line COTAHIST-shaped file, and a header-only C++23 wrapper - (`fixedwidth.hpp`) that also maps structs to columns through `FixedWidthMapper`. + (`fixedwidth.hpp`) that also maps structs to columns through `FixedWidthMapper`. It installs as a CMake + package (`find_package(fixedwidth)`) or through FetchContent, and downloads the release's native library for + the platform, so it needs no .NET SDK. - Releases publish the Python package to PyPI as `fixedwidthparser`, with wheels for Windows x64, macOS arm64 and x64, and Linux glibc and musl on x64 and arm64, and attach each platform's native library to the GitHub Release. diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 6c883df..10d9ab8 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,31 +1,39 @@ # The C++ wrapper: a header-only INTERFACE target, fw::fixedwidth, linked against the published -# FixedWidthParser.Native library. +# FixedWidthParser.Native library. See cpp/README.md for using it from another project. # -# python python/scripts/build_native.py -# cmake -S cpp -B build/cpp -DFIXEDWIDTH_NATIVE_LIB= +# cmake -S cpp -B build/cpp # cmake --build build/cpp && ctest --test-dir build/cpp --output-on-failure +# cmake --install build/cpp --prefix # then find_package(fixedwidth) cmake_minimum_required(VERSION 3.16) project(FixedWidthParser LANGUAGES CXX) set(CMAKE_CXX_STANDARD 23) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(_include_dir "${CMAKE_CURRENT_SOURCE_DIR}/../src/FixedWidthParser.Native/include") -if(WIN32) - set(_lib_name "FixedWidthParser.Native.dll") -elseif(APPLE) - set(_lib_name "FixedWidthParser.Native.dylib") -else() - set(_lib_name "FixedWidthParser.Native.so") -endif() -set(FIXEDWIDTH_NATIVE_LIB "${CMAKE_CURRENT_SOURCE_DIR}/../python/src/fixedwidth/_lib/${_lib_name}" - CACHE FILEPATH "Path to the published FixedWidthParser.Native shared library") -if(NOT EXISTS "${FIXEDWIDTH_NATIVE_LIB}") - message(FATAL_ERROR "FixedWidthParser.Native not found at ${FIXEDWIDTH_NATIVE_LIB}; run python/scripts/build_native.py or set FIXEDWIDTH_NATIVE_LIB") +set(FIXEDWIDTH_NATIVE_LIB "" CACHE FILEPATH + "FixedWidthParser.Native shared library to link; empty uses the repo's published copy or downloads the release's") +set(FIXEDWIDTH_VERSION "" CACHE STRING + "Release tag whose native library is downloaded (v2.0.0); empty takes the tag checked out, from git describe") +if(NOT FIXEDWIDTH_VERSION) + find_package(Git QUIET) + if(Git_FOUND) + execute_process(COMMAND ${GIT_EXECUTABLE} describe --tags --exact-match + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE _git_tag OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET RESULT_VARIABLE _git_result) + endif() + if(_git_result EQUAL 0 AND _git_tag) + set(FIXEDWIDTH_VERSION "${_git_tag}") + else() + set(FIXEDWIDTH_VERSION "v0.0.0") + endif() endif() +set(_include_dir "${CMAKE_CURRENT_SOURCE_DIR}/../src/FixedWidthParser.Native/include") + include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/NativeLib.cmake") -fixedwidth_import_native_lib("${FIXEDWIDTH_NATIVE_LIB}" "${_include_dir}/fixedwidth.def") +include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/CopyNativeLibrary.cmake") +fixedwidth_resolve_native_lib(_native_lib) +fixedwidth_import_native_lib("${_native_lib}" "${_include_dir}/fixedwidth.def") add_library(fw_fixedwidth INTERFACE) target_include_directories(fw_fixedwidth INTERFACE "${_include_dir}") @@ -34,6 +42,61 @@ target_link_libraries(fw_fixedwidth INTERFACE fw::native) add_library(fw::fixedwidth ALIAS fw_fixedwidth) if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) + set(_top_level ON) +else() + set(_top_level OFF) +endif() +option(FIXEDWIDTH_INSTALL "Generate install rules and a find_package(fixedwidth) config package" ${_top_level}) +option(FIXEDWIDTH_BUILD_TESTS "Build the C++ wrapper's tests" ${_top_level}) + +if(FIXEDWIDTH_INSTALL) + include(GNUInstallDirs) + include(CMakePackageConfigHelpers) + + get_filename_component(FIXEDWIDTH_INSTALLED_NATIVE_NAME "${_native_lib}" NAME) + if(WIN32) + set(FIXEDWIDTH_INSTALLED_NATIVE_DIR "${CMAKE_INSTALL_BINDIR}") + else() + set(FIXEDWIDTH_INSTALLED_NATIVE_DIR "${CMAKE_INSTALL_LIBDIR}") + endif() + set(FIXEDWIDTH_INSTALLED_IMPLIB_DIR "${CMAKE_INSTALL_LIBDIR}") + set(FIXEDWIDTH_INSTALLED_INCLUDE_DIR "${CMAKE_INSTALL_INCLUDEDIR}") + + install(DIRECTORY "${_include_dir}/" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + FILES_MATCHING PATTERN "*.h" PATTERN "*.hpp") + install(FILES "${_native_lib}" DESTINATION "${FIXEDWIDTH_INSTALLED_NATIVE_DIR}") + if(WIN32) + get_target_property(_native_implib fixedwidth_native IMPORTED_IMPLIB) + get_filename_component(FIXEDWIDTH_INSTALLED_IMPLIB_NAME "${_native_implib}" NAME) + install(FILES "${_native_implib}" DESTINATION "${FIXEDWIDTH_INSTALLED_IMPLIB_DIR}") + endif() + + set(_config_dir "${CMAKE_INSTALL_LIBDIR}/cmake/fixedwidth") + configure_package_config_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/fixedwidthConfig.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/fixedwidthConfig.cmake" + INSTALL_DESTINATION "${_config_dir}" + PATH_VARS FIXEDWIDTH_INSTALLED_NATIVE_DIR FIXEDWIDTH_INSTALLED_IMPLIB_DIR FIXEDWIDTH_INSTALLED_INCLUDE_DIR) + + # The package version takes only the tag's digits: v2.0.0-rc.1 installs as 2.0.0. + if(FIXEDWIDTH_VERSION MATCHES "^v?([0-9]+(\\.[0-9]+)*)") + set(_pkg_version "${CMAKE_MATCH_1}") + else() + set(_pkg_version "0.0.0") + endif() + write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/fixedwidthConfigVersion.cmake" + VERSION "${_pkg_version}" + COMPATIBILITY SameMajorVersion) + + install(FILES + "${CMAKE_CURRENT_BINARY_DIR}/fixedwidthConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/fixedwidthConfigVersion.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/CopyNativeLibrary.cmake" + DESTINATION "${_config_dir}") +endif() + +if(FIXEDWIDTH_BUILD_TESTS) enable_testing() add_subdirectory(tests) endif() diff --git a/cpp/README.md b/cpp/README.md index e6ec580..9687bda 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -64,14 +64,60 @@ the `fw::Table` underneath, with `lines_read()` and `lines_skipped()`. `fw::Type An integer field narrower than `int64_t` that a value does not fit fails the read with `FW_INVALID_ARGUMENT` instead of wrapping. -## CMake +## Install + +Needs a C++23 compiler and CMake 3.16 or later; on Windows, `lib.exe` (run CMake from a Visual Studio developer +prompt) or MinGW's `dlltool`. No .NET SDK: CMake downloads the prebuilt native library for your platform from +the GitHub Release of the tag you build (Windows x64, macOS arm64/x64, Linux glibc x64/arm64). + +**FetchContent**, the simplest: + +```cmake +include(FetchContent) +FetchContent_Declare(fixedwidth + GIT_REPOSITORY https://github.com/GabrielMarquezMatte/FixedWidthParser.git + GIT_TAG v2.0.0 + SOURCE_SUBDIR cpp) +FetchContent_MakeAvailable(fixedwidth) + +target_link_libraries(app PRIVATE fw::fixedwidth) +fixedwidth_copy_native_library(app) # Windows: puts the DLL next to app.exe; a no-op elsewhere +``` + +**Installed package**, for `find_package`: + + git clone --branch v2.0.0 https://github.com/GabrielMarquezMatte/FixedWidthParser.git + cmake -S FixedWidthParser/cpp -B build -DCMAKE_BUILD_TYPE=Release + cmake --install build --prefix + +```cmake +find_package(fixedwidth 2 REQUIRED) # with in CMAKE_PREFIX_PATH +target_link_libraries(app PRIVATE fw::fixedwidth) +fixedwidth_copy_native_library(app) +``` + +The install puts the headers in `include/`, the library in `bin/` (Windows) or `lib/`, the Windows import +library in `lib/`, and the package in `lib/cmake/fixedwidth`. `find_package` never downloads anything. + +Settings: + +- `FIXEDWIDTH_VERSION`: the release tag whose library is downloaded. Empty takes the tag checked out, which + `GIT_TAG` and `--branch` above set. +- `FIXEDWIDTH_NATIVE_LIB` (CMake or environment variable): a library you built or downloaded yourself, such as + musl's `fixedwidth-native-linux-musl-x64.so` from the release, saved as `FixedWidthParser.Native.so`. Nothing + is downloaded then. + +On Windows an import library is generated from `include/fixedwidth.def`. GCC's `-Wextra` warns about designated +initializers that leave defaulted options out; that is the intended use, so build with +`-Wno-missing-field-initializers`. + +## Developing python python/scripts/build_native.py cmake -S cpp -B build/cpp cmake --build build/cpp && ctest --test-dir build/cpp --output-on-failure -`add_subdirectory(cpp)` gives the `fw::fixedwidth` target; `FIXEDWIDTH_NATIVE_LIB` points at the published -library (by default the copy in `python/src/fixedwidth/_lib`). On Windows an import library is generated from -`include/fixedwidth.def` with `lib.exe` or `dlltool`, and `fixedwidth_copy_native_library()` copies -the DLL next to an executable. GCC's `-Wextra` warns about designated initializers that leave defaulted -options out; that is the intended use, so build with `-Wno-missing-field-initializers`. +In a checkout, the library `build_native.py` publishes into `python/src/fixedwidth/_lib` takes precedence over +the download. `FIXEDWIDTH_BUILD_TESTS` and `FIXEDWIDTH_INSTALL` default to on when `cpp/` is the top-level +project and off under FetchContent or `add_subdirectory`. `cpp/tests/package` builds against an installed prefix, +as CI does. diff --git a/cpp/cmake/CopyNativeLibrary.cmake b/cpp/cmake/CopyNativeLibrary.cmake new file mode 100644 index 0000000..1bb7e51 --- /dev/null +++ b/cpp/cmake/CopyNativeLibrary.cmake @@ -0,0 +1,15 @@ +# fixedwidth_copy_native_library() copies the native library next to an executable after it links. +# +# On Windows a DLL beside neither the executable nor PATH is not found, and the process exits with 0xc0000135 +# (STATUS_DLL_NOT_FOUND) before main runs. A no-op on Linux and macOS, where the rpath resolves it. +function(fixedwidth_copy_native_library target) + if(NOT WIN32) + return() + endif() + if(NOT TARGET ${target}) + message(FATAL_ERROR "fixedwidth_copy_native_library: '${target}' is not a target") + endif() + add_custom_command(TARGET ${target} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different "$" "$" + COMMENT "Copying FixedWidthParser.Native next to ${target}") +endfunction() diff --git a/cpp/cmake/NativeLib.cmake b/cpp/cmake/NativeLib.cmake index 76f8008..14282d9 100644 --- a/cpp/cmake/NativeLib.cmake +++ b/cpp/cmake/NativeLib.cmake @@ -1,3 +1,60 @@ +# fixedwidth_resolve_native_lib() finds the FixedWidthParser.Native shared library for this platform: +# +# 1. FIXEDWIDTH_NATIVE_LIB, as a CMake or environment variable: a library you built or downloaded. +# 2. The copy python/scripts/build_native.py publishes into python/src/fixedwidth/_lib, in a checkout of the repo. +# 3. Otherwise the release asset for FIXEDWIDTH_VERSION, downloaded once into the build tree. +set(_fixedwidth_repo_root "${CMAKE_CURRENT_LIST_DIR}/../..") + +function(fixedwidth_resolve_native_lib out_var) + if(WIN32) + set(_os "win") + set(_ext "dll") + elseif(APPLE) + set(_os "osx") + set(_ext "dylib") + else() + set(_os "linux") + set(_ext "so") + endif() + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64|ARM64)$") + set(_arch "arm64") + else() + set(_arch "x64") + endif() + set(_lib_name "FixedWidthParser.Native.${_ext}") + + if(FIXEDWIDTH_NATIVE_LIB) + set(_lib "${FIXEDWIDTH_NATIVE_LIB}") + elseif(DEFINED ENV{FIXEDWIDTH_NATIVE_LIB}) + set(_lib "$ENV{FIXEDWIDTH_NATIVE_LIB}") + elseif(EXISTS "${_fixedwidth_repo_root}/python/src/fixedwidth/_lib/${_lib_name}") + set(_lib "${_fixedwidth_repo_root}/python/src/fixedwidth/_lib/${_lib_name}") + else() + # Saved under the library's own name, not the asset's: the import library on Windows and the SONAME on + # Linux both record the name the loader looks for. + set(_lib "${CMAKE_BINARY_DIR}/fixedwidth-native/${FIXEDWIDTH_VERSION}/${_os}-${_arch}/${_lib_name}") + if(NOT EXISTS "${_lib}") + set(_url "https://github.com/GabrielMarquezMatte/FixedWidthParser/releases/download/${FIXEDWIDTH_VERSION}/fixedwidth-native-${_os}-${_arch}.${_ext}") + message(STATUS "Downloading FixedWidthParser.Native: ${_url}") + file(DOWNLOAD "${_url}" "${_lib}" STATUS _status) + list(GET _status 0 _code) + if(NOT _code EQUAL 0) + file(REMOVE "${_lib}") + list(GET _status 1 _message) + message(FATAL_ERROR "Downloading ${_url} failed: ${_message}. Set FIXEDWIDTH_VERSION to a release " + "tag, or FIXEDWIDTH_NATIVE_LIB to a library you have.") + endif() + endif() + endif() + + # A relative path would be re-resolved against the build directory once baked into the imported target. + get_filename_component(_lib "${_lib}" ABSOLUTE BASE_DIR "${CMAKE_BINARY_DIR}") + if(NOT EXISTS "${_lib}") + message(FATAL_ERROR "FixedWidthParser.Native not found at ${_lib}") + endif() + set(${out_var} "${_lib}" PARENT_SCOPE) +endfunction() + # fixedwidth_import_native_lib( ) creates the IMPORTED target fw::native. # # NativeAOT ships no import library on Windows, so one is generated from the checked-in .def (lib.exe @@ -34,14 +91,3 @@ function(fixedwidth_import_native_lib library def_file) add_library(fw::native ALIAS fixedwidth_native) endfunction() - -# Copies the native library next to an executable: on Windows a DLL beside neither the executable nor -# PATH is not found, and the process exits before main. A no-op elsewhere, where the rpath resolves it. -function(fixedwidth_copy_native_library target) - if(NOT WIN32) - return() - endif() - add_custom_command(TARGET ${target} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different "$" "$" - COMMENT "Copying FixedWidthParser.Native next to ${target}") -endfunction() diff --git a/cpp/cmake/fixedwidthConfig.cmake.in b/cpp/cmake/fixedwidthConfig.cmake.in new file mode 100644 index 0000000..deddbcf --- /dev/null +++ b/cpp/cmake/fixedwidthConfig.cmake.in @@ -0,0 +1,26 @@ +@PACKAGE_INIT@ + +# The same two targets cpp/CMakeLists.txt defines for an in-tree build, pointed at the installed copies. The +# native library was resolved once, at install time, so find_package() never downloads anything. + +if(NOT TARGET fw::native) + add_library(fw::native SHARED IMPORTED) + set_target_properties(fw::native PROPERTIES + IMPORTED_LOCATION "@PACKAGE_FIXEDWIDTH_INSTALLED_NATIVE_DIR@/@FIXEDWIDTH_INSTALLED_NATIVE_NAME@") + if(WIN32) + set_target_properties(fw::native PROPERTIES + IMPORTED_IMPLIB "@PACKAGE_FIXEDWIDTH_INSTALLED_IMPLIB_DIR@/@FIXEDWIDTH_INSTALLED_IMPLIB_NAME@") + endif() +endif() + +if(NOT TARGET fw::fixedwidth) + add_library(fw::fixedwidth INTERFACE IMPORTED) + set_target_properties(fw::fixedwidth PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "@PACKAGE_FIXEDWIDTH_INSTALLED_INCLUDE_DIR@" + INTERFACE_COMPILE_FEATURES cxx_std_23 + INTERFACE_LINK_LIBRARIES fw::native) +endif() + +include("${CMAKE_CURRENT_LIST_DIR}/CopyNativeLibrary.cmake") + +check_required_components(fixedwidth) diff --git a/cpp/tests/package/CMakeLists.txt b/cpp/tests/package/CMakeLists.txt new file mode 100644 index 0000000..786be19 --- /dev/null +++ b/cpp/tests/package/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.16) + +# Consumes the installed package the way a downstream project does - find_package() against an install prefix, +# not add_subdirectory() against this repo. Its own project, so a config file that only works inside the build +# tree fails here. +project(fixedwidth_package_consumer LANGUAGES CXX) + +find_package(fixedwidth REQUIRED) + +add_executable(package_consumer main.cpp) +target_link_libraries(package_consumer PRIVATE fw::fixedwidth) +fixedwidth_copy_native_library(package_consumer) + +enable_testing() +add_test(NAME package_consumer COMMAND package_consumer) +set_tests_properties(package_consumer PROPERTIES PASS_REGULAR_EXPRESSION "package ok") diff --git a/cpp/tests/package/main.cpp b/cpp/tests/package/main.cpp new file mode 100644 index 0000000..09514e0 --- /dev/null +++ b/cpp/tests/package/main.cpp @@ -0,0 +1,32 @@ +#include + +#include +#include +#include + +int main() +{ + std::vector columns{ + {.name = "code", .start = 0, .length = 5, .type = fw::Type::String}, + {.name = "price", .start = 5, .length = 8, .type = fw::Type::Float64, .scale = 2}, + }; + auto layout = fw::Layout::create(columns); + if (!layout) + { + std::puts(layout.error().message.c_str()); + return 1; + } + auto table = fw::read_memory(std::string_view{"PETR400001234\nVALE300005678\n"}, *layout); + if (!table) + { + std::puts(table.error().message.c_str()); + return 1; + } + if (table->row_count() != 2 || table->string(0, 1) != "VALE3" || table->values(1)[0] != 12.34) + { + std::puts("unexpected values"); + return 1; + } + std::puts("package ok"); + return 0; +} From 3ac88c1effa5fbb987dd740497342f1bee5ba224 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sun, 25 Oct 2026 00:04:24 -0300 Subject: [PATCH 78/85] feat: parse and write enum columns As ExcelReader does: an enum column reads a member's name in any case or its number, zero-padded codes included (02), and rejects a number the enum does not define and a combination of [Flags] members. It writes the name, or the number with Format = "D". FixedWidthRuntime.TryEnum ports ExcelReader's EnumCache, one frozen case-insensitive map of names and numbers per enum, and widens ASCII text straight to chars, so a column costs what an int column does (~17 ns, against ~29 ns for Enum.TryParse with Enum.IsDefined). Attribute layouts and generated models take an enum property as it is, nullable included; fluent layouts add EnumColumn and NullableEnumColumn. --- CHANGELOG.md | 3 + README.md | 2 +- docs/guide/models.md | 11 +- .../FixedWidthModelGenerator.cs | 14 +- .../Infrastructure/FixedWidthRuntime.cs | 56 +++++++ .../Layouts/FixedWidthLayoutBuilder.cs | 49 ++++++ .../Processors/ColumnParserFactory.cs | 17 +- src/FixedWidthParser/PublicAPI.Unshipped.txt | 3 + .../FixedWidthParser.Tests/EnumColumnTests.cs | 146 ++++++++++++++++++ 9 files changed, 293 insertions(+), 8 deletions(-) create mode 100644 tests/FixedWidthParser.Tests/EnumColumnTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6788c55..ea1e79c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,6 +89,9 @@ Windows-1252 after registering `CodePagesEncodingProvider`). - `FixedWidth.TryFormat(in model, layout, Span, in FixedWidthContext, out int)`. - `FixedColumnAttribute.Converter`: a per-property `IFixedWidthConverter` that takes priority over the built-in parsing and formatting, with diagnostic `FWP008` when it targets the wrong type. +- Enum columns, as ExcelReader reads them: a member's name in any case or its number, and nothing the enum + does not define. They write the name, or the number with `Format = "D"`. Attribute layouts and generated + models take an enum property as it is; fluent layouts add `EnumColumn` and `NullableEnumColumn`. - Nullable value-type columns (`int?`, `decimal?`, `DateTime?`, …): a blank column parses to `null` without invoking the parser or converter, and `null` writes as a blank column. - The source-generated writer: `IFixedWidthModel.TryFormat`, with the same alignment, padding, diff --git a/README.md b/README.md index b9d6748..836b334 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Upgrading from 1.x? See [the 2.0 entry in the changelog](CHANGELOG.md#200). - **Bounded memory on untrusted input**: `MaxLineLength` stops a source without line breaks from growing the buffer. - **Writing** of single records and batches, synchronous and asynchronous, batched into one pooled buffer. - **Configurable formatting** per column: alignment, padding character, format string and explicit overflow policy. -- **Culture-aware** numbers and dates, with a fast path for `double`, `float` and `decimal`, and any `IUtf8SpanParsable`/`ISpanParsable` type. +- **Culture-aware** numbers and dates, with a fast path for `double`, `float` and `decimal`, enums by name or number, and any `IUtf8SpanParsable`/`ISpanParsable` type. - **Layout validation** when a layout is built or generated: negative `Start`, non-positive `Length`, overlapping columns and non-ASCII padding fail clearly. - **`ref struct` model support** on single-line parsing and formatting, on every reader and on parallel aggregation. - **Python and C/C++** through a NativeAOT library with a C ABI: columnar and Arrow reads, parallel reads and writes. See [Python and C/C++](#python-and-cc). diff --git a/docs/guide/models.md b/docs/guide/models.md index bd46ff8..ca322f3 100644 --- a/docs/guide/models.md +++ b/docs/guide/models.md @@ -91,7 +91,16 @@ Each column can be tuned through named attribute arguments: ## Column Types -`string`, `double` and `float`, any `IUtf8SpanParsable` (parsed straight from bytes), any other `ISpanParsable` (decoded first), `T?` of any of them (a blank column is `null`), and anything with a `Converter`. Writing needs `string`, `ISpanFormattable` or a converter. +`string`, `double` and `float`, enums, any `IUtf8SpanParsable` (parsed straight from bytes), any other `ISpanParsable` (decoded first), `T?` of any of them (a blank column is `null`), and anything with a `Converter`. Writing needs `string`, `ISpanFormattable` (enums included) or a converter. + +An enum column reads a member's name in any case or its number (`2`, `02`), and rejects a number no member has and a combination of `[Flags]` members. It writes the name; `Format = "D"` writes the number, which `Padding = '0'` and `Alignment.Right` zero-pad. Fluent layouts add one with `EnumColumn` or `NullableEnumColumn`. + +```csharp +public enum Side { Buy = 1, Sell = 2 } + +[FixedColumn(0, 2, Format = "D", Padding = '0', Alignment = Alignment.Right)] +public Side Side { get; set; } // "02" reads and writes as Side.Sell +``` ## Encodings diff --git a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs index e172848..5876018 100644 --- a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs +++ b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs @@ -36,7 +36,7 @@ public sealed class FixedWidthModelGenerator : IIncrementalGenerator private static readonly DiagnosticDescriptor UnsupportedColumnType = new( "FWP003", "Unsupported fixed-width column type", - "Column '{0}' has type '{1}', which is not string, double, float, IUtf8SpanParsable or ISpanParsable; set a Converter for it", + "Column '{0}' has type '{1}', which is not string, double, float, an enum, IUtf8SpanParsable or ISpanParsable; set a Converter for it", "FixedWidthParser", DiagnosticSeverity.Error, isEnabledByDefault: true); private static readonly DiagnosticDescriptor NegativeStart = new( @@ -240,7 +240,7 @@ private static bool ImplementsMarker(INamedTypeSymbol symbol, INamedTypeSymbol m && SymbolEqualityComparer.Default.Equals(i.TypeArguments[0], symbol)); } - // Mirrors ColumnParserFactory: converter → date/time with Format → string → double/float → + // Mirrors ColumnParserFactory: converter → date/time with Format → string → double/float → enum → // IUtf8SpanParsable → ISpanParsable over the decoded text. private static ColumnKind Classify(ITypeSymbol type, ITypeSymbol? converterType, string? format, Compilation compilation) { @@ -267,6 +267,10 @@ private static ColumnKind Classify(ITypeSymbol type, ITypeSymbol? converterType, break; } + if (type.TypeKind == TypeKind.Enum) + { + return ColumnKind.Enum; + } if (ImplementsGeneric(type, type, compilation, "System.IUtf8SpanParsable`1")) { return ColumnKind.Utf8Parsable; @@ -610,6 +614,10 @@ private static void AppendColumnParse(StringBuilder sb, string stmt, ColumnInfo sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append(c.Kind == ColumnKind.Utf8Parsable ? ".TryParse<" : ".TryParseDecoded<") .Append(c.TypeFqn).Append(">(").Append(col).Append(", context").Append(parse).Append(", out ").Append(c.TypeFqn).Append(' ').Append(localName).AppendLine(fail); break; + case ColumnKind.Enum: + sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append(".TryEnum<") + .Append(c.TypeFqn).Append(">(").Append(col).Append(", context").Append(parse).Append(", out ").Append(c.TypeFqn).Append(' ').Append(localName).AppendLine(fail); + break; case ColumnKind.Converter: sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append(".TryConvert<").Append(c.TypeFqn).Append(", ").Append(c.ConverterFqn) .Append(">(").Append(col).Append(", context, __converter").Append(i).Append(parse).Append(", out ").Append(c.TypeFqn) @@ -724,7 +732,7 @@ private static string TrimModeLiteral(int value) }; } - private enum ColumnKind { String, Double, Float, Utf8Parsable, DecodedParsable, Exact, Converter, BadConverter, Unsupported } + private enum ColumnKind { String, Double, Float, Enum, Utf8Parsable, DecodedParsable, Exact, Converter, BadConverter, Unsupported } private enum WriteKind { String, Formattable, Utf8Formattable, Converter, Unsupported } diff --git a/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs index 23bf281..5150050 100644 --- a/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs +++ b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs @@ -1,5 +1,6 @@ using System.Buffers.Text; using System.Buffers; +using System.Collections.Frozen; using System.ComponentModel; using System.Diagnostics.CodeAnalysis; using System.Globalization; @@ -116,6 +117,61 @@ public static bool TryParseDecoded(ReadOnlySpan column, in FixedWi return ParseDecoded(TrimColumn(column, in options), in context, null, static (text, _, provider, [MaybeNullWhen(false)] out result) => TValue.TryParse(text, provider, out result), out value); } + /// + /// Parses an enum column, as ExcelReader does: a member's name in any case, or its number (2, 02). + /// A number no member has, or a combination of flags, does not parse. + /// + [SkipLocalsInit] + public static bool TryEnum(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out TEnum value) + where TEnum : struct, Enum + { + var trimmed = TrimColumn(column, in options); + // ASCII reads the same in every supported encoding, and widening it skips the decoder. + Span chars = stackalloc char[64]; + if (trimmed.Length <= chars.Length && Ascii.ToUtf16(trimmed, chars, out int written) == OperationStatus.Done) + { + return EnumNames.TryLookup(chars[..written], out value); + } + return ParseDecoded(trimmed, in context, null, static (text, _, _, out result) => EnumNames.TryLookup(text, out result), out value); + } + + // ExcelReader's EnumCache: every member under its name and its number, in one case-insensitive frozen map. + // With the ASCII widening in TryEnum, a column costs what an int column does (~17 ns), where Enum.TryParse + // with Enum.IsDefined took ~29 ns. + private static class EnumNames + where TEnum : struct, Enum + { + private static readonly FrozenDictionary.AlternateLookup> Lookup = Build(); + + private static FrozenDictionary.AlternateLookup> Build() + { + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (TEnum member in Enum.GetValues()) + { + map[member.ToString()] = member; + map[member.ToString("D")] = member; + } + return map.ToFrozenDictionary(StringComparer.OrdinalIgnoreCase).GetAlternateLookup>(); + } + + public static bool TryLookup(ReadOnlySpan text, out TEnum value) + { + text = text.Trim(); + if (Lookup.TryGetValue(text, out value)) + { + return true; + } + // A zero-padded code is its number: 02 is 2, and 00 is 0. + int digits = text.IndexOfAnyExcept('0'); + if (digits == 0 || text.IsEmpty) + { + return false; + } + var number = digits < 0 ? "0" : text[digits..]; + return !number.ContainsAnyExceptInRange('0', '9') && Lookup.TryGetValue(number, out value); + } + } + /// Parses a column through an instance. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool TryConvert( diff --git a/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs b/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs index ca6bb95..a41f837 100644 --- a/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs +++ b/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs @@ -102,6 +102,55 @@ public FixedWidthLayoutBuilder DecodedColumn( return Add(column, parser, formatter); } + /// + /// Adds an enum column: it reads a member's name in any case or its number, and writes the name, or the + /// number with "D". A number no member has does not read. + /// + /// + public FixedWidthLayoutBuilder EnumColumn( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, FixedWidthColumnOptions? options = null) + where TEnum : struct, Enum + { + var column = Begin(name, start, length, typeof(TEnum), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForEnum(set, column.Parse); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + formatter = new SpanFormattableColumnFormatter(start, length, column.FormatOptions, name, get); + } + return Add(column, parser, formatter); + } + + /// + /// Adds a enum column: blank reads as , and + /// writes the column filled with its trim character. + /// + /// + public FixedWidthLayoutBuilder NullableEnumColumn( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, FixedWidthColumnOptions? options = null) + where TEnum : struct, Enum + { + var column = Begin(name, start, length, typeof(TEnum?), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForNullable(set, column.Parse, + adapted => ColumnParserFactory.ForEnum(adapted, column.Parse)); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + var inner = new SpanFormattableColumnFormatter( + start, length, column.FormatOptions, name, (in TModel model) => get(in model).GetValueOrDefault()); + formatter = new NullableColumnFormatter(start, length, column.Parse.TrimByte, get, inner); + } + return Add(column, parser, formatter); + } + /// /// Adds a column: a column holding only its trim character reads as /// , and writes the column filled with it. diff --git a/src/FixedWidthParser/Processors/ColumnParserFactory.cs b/src/FixedWidthParser/Processors/ColumnParserFactory.cs index 411e3b9..617c948 100644 --- a/src/FixedWidthParser/Processors/ColumnParserFactory.cs +++ b/src/FixedWidthParser/Processors/ColumnParserFactory.cs @@ -8,8 +8,8 @@ namespace FixedWidthParser.Processors /// Combines a column's setter and a value parser into a single /// closure that calls the same helpers as generated code. Resolution /// order: a converter, a date or time with a Format, string, double/float, - /// then , then over the decoded - /// text. Attribute layouts reach it by reflection (), fluent layouts through + /// an enum, then , then over the + /// decoded text. Attribute layouts reach it by reflection (), fluent layouts through /// the typed For* entry points. /// internal static class ColumnParserFactory @@ -56,6 +56,10 @@ private static ColumnParser CreateValue(Type valueType, Delegate { return ForString((ColumnSetter)setter, options); } + if (valueType.IsEnum) + { + return (ColumnParser)Method(nameof(ForEnum), typeof(TModel), valueType).Invoke(null, [setter, options])!; + } if (Implements(valueType, typeof(IUtf8SpanParsable<>))) { return (ColumnParser)Method(nameof(ForUtf8Parsable), typeof(TModel), valueType).Invoke(null, [setter, options])!; @@ -66,7 +70,14 @@ private static ColumnParser CreateValue(Type valueType, Delegate } throw new InvalidOperationException( - $"Column '{memberName}' has type '{valueType}', which is not string, double, float, IUtf8SpanParsable or ISpanParsable; set a Converter for it."); + $"Column '{memberName}' has type '{valueType}', which is not string, double, float, an enum, IUtf8SpanParsable or ISpanParsable; set a Converter for it."); + } + + internal static ColumnParser ForEnum(ColumnSetter setter, ColumnParseOptions options) + where TModel : allows ref struct + where TEnum : struct, Enum + { + return Bind(setter, options, FixedWidthRuntime.TryEnum); } internal static ColumnParser ForString(ColumnSetter setter, ColumnParseOptions options) diff --git a/src/FixedWidthParser/PublicAPI.Unshipped.txt b/src/FixedWidthParser/PublicAPI.Unshipped.txt index f0ac24c..c66e75a 100644 --- a/src/FixedWidthParser/PublicAPI.Unshipped.txt +++ b/src/FixedWidthParser/PublicAPI.Unshipped.txt @@ -195,3 +195,6 @@ static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryFloat(System.ReadOnl static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryParse(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out TValue value) -> bool static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryParseDecoded(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out TValue value) -> bool static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryTimeOnlyExact(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out System.TimeOnly value) -> bool +FixedWidthParser.FixedWidthLayoutBuilder.EnumColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.FixedWidthLayoutBuilder.NullableEnumColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryEnum(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out TEnum value) -> bool diff --git a/tests/FixedWidthParser.Tests/EnumColumnTests.cs b/tests/FixedWidthParser.Tests/EnumColumnTests.cs new file mode 100644 index 0000000..9b3b984 --- /dev/null +++ b/tests/FixedWidthParser.Tests/EnumColumnTests.cs @@ -0,0 +1,146 @@ +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + public enum Side { Buy = 1, Sell = 2 } + + [Flags] + public enum Access { None = 0, Read = 1, Write = 2 } + + public sealed class EnumModel + { + [FixedColumn(0, 10)] public Side Side { get; set; } + [FixedColumn(10, 5)] public Side? MaybeSide { get; set; } + [FixedColumn(15, 10)] public Access Access { get; set; } + } + + public partial class GenEnumModel : IFixedWidthModel + { + [FixedColumn(0, 10)] public Side Side { get; set; } + [FixedColumn(10, 5)] public Side? MaybeSide { get; set; } + [FixedColumn(15, 10)] public Access Access { get; set; } + } + + public enum Kind { Ação = 1, Fundo = 2 } + + public sealed class KindModel + { + [FixedColumn(0, 6)] public Kind Kind { get; set; } + } + + public sealed class EnumCodeModel + { + [FixedColumn(0, 2, Format = "D", Padding = '0', Alignment = Alignment.Right)] public Side Side { get; set; } + } + + public partial class GenEnumCodeModel : IFixedWidthModel + { + [FixedColumn(0, 2, Format = "D", Padding = '0', Alignment = Alignment.Right)] public Side Side { get; set; } + } + + /// + /// Enum columns, as ExcelReader reads them: a member's name in any case or its number, nothing the enum does not + /// define. Attribute, generated and fluent layouts agree. + /// + public class EnumColumnTests + { + private static byte[] Line(string side, string maybeSide = "", string access = "Read") + { + return Bytes(side.PadRight(10) + maybeSide.PadRight(5) + access.PadRight(10)); + } + + private static FixedWidthLayout Fluent() + { + return FixedWidthLayout.Build(b => b + .Factory(() => new EnumModel()) + .EnumColumn("Side", 0, 10, (in EnumModel m) => m.Side, (ref EnumModel m, Side v) => m.Side = v) + .NullableEnumColumn("MaybeSide", 10, 5, (in EnumModel m) => m.MaybeSide, (ref EnumModel m, Side? v) => m.MaybeSide = v) + .EnumColumn("Access", 15, 10, (in EnumModel m) => m.Access, (ref EnumModel m, Access v) => m.Access = v)); + } + + [Theory] + [InlineData("Buy", Side.Buy)] + [InlineData("sell", Side.Sell)] + [InlineData("SELL", Side.Sell)] + [InlineData("1", Side.Buy)] + [InlineData("02", Side.Sell)] + [InlineData(" Sell", Side.Sell)] + public void NameOrNumber_Parses(string text, Side expected) + { + Assert.True(FixedWidth.TryParse(Line(text), FixedWidthLayout.FromAttributes(), Ctx, out var attributes)); + Assert.True(FixedWidth.TryParse(Line(text), Ctx, out var generated)); + Assert.True(FixedWidth.TryParse(Line(text), Fluent(), Ctx, out var fluent)); + Assert.Equal(expected, attributes.Side); + Assert.Equal(expected, generated.Side); + Assert.Equal(expected, fluent.Side); + } + + [Theory] + [InlineData("Hold")] + [InlineData("3")] + [InlineData("0")] + [InlineData("")] + [InlineData("Buy,Sell")] + [InlineData("0Buy")] + [InlineData("00")] + [InlineData("-1")] + public void UndefinedValue_DoesNotParse(string text) + { + Assert.False(FixedWidth.TryParse(Line(text), FixedWidthLayout.FromAttributes(), Ctx, out _)); + Assert.False(FixedWidth.TryParse(Line(text), Ctx, out _)); + Assert.False(FixedWidth.TryParse(Line(text), Fluent(), Ctx, out _)); + } + + [Fact] + public void FlagsCombination_DoesNotParse() + { + Assert.True(FixedWidth.TryParse(Line("Buy", access: "Write"), FixedWidthLayout.FromAttributes(), Ctx, out var single)); + Assert.Equal(Access.Write, single.Access); + Assert.True(FixedWidth.TryParse(Line("Buy", access: "00"), Ctx, out var zero)); + Assert.Equal(Access.None, zero.Access); + Assert.False(FixedWidth.TryParse(Line("Buy", access: "Read,Write"), FixedWidthLayout.FromAttributes(), Ctx, out _)); + Assert.False(FixedWidth.TryParse(Line("Buy", access: "3"), Ctx, out _)); + } + + [Fact] + public void Nullable_BlankIsNull() + { + Assert.True(FixedWidth.TryParse(Line("Buy"), FixedWidthLayout.FromAttributes(), Ctx, out var blank)); + Assert.Null(blank.MaybeSide); + Assert.True(FixedWidth.TryParse(Line("Buy", "sell"), Ctx, out var present)); + Assert.Equal(Side.Sell, present.MaybeSide); + Assert.True(FixedWidth.TryParse(Line("Buy", "2"), Fluent(), Ctx, out var fluent)); + Assert.Equal(Side.Sell, fluent.MaybeSide); + } + + [Fact] + public void Write_WritesTheName() + { + const string expected = "Sell None "; + var model = new EnumModel { Side = Side.Sell, MaybeSide = null, Access = Access.None }; + + Assert.Equal(expected, WriteOne(FixedWidthLayout.FromAttributes(), model)); + Assert.Equal(expected, WriteOne(Fluent(), model)); + Assert.Equal(expected, Format(new GenEnumModel { Side = Side.Sell, MaybeSide = null, Access = Access.None })); + Assert.Equal("Buy Sell Read ", Format(new GenEnumModel { Side = Side.Buy, MaybeSide = Side.Sell, Access = Access.Read })); + } + + [Fact] + public void NonAsciiName_IsDecodedWithTheEncoding() + { + var latin1 = CtxWith(encoding: System.Text.Encoding.Latin1); + + Assert.True(FixedWidth.TryParse(System.Text.Encoding.Latin1.GetBytes("AÇÃO "), FixedWidthLayout.FromAttributes(), latin1, out var model)); + Assert.Equal(Kind.Ação, model.Kind); + } + + [Fact] + public void FormatD_WritesTheNumber_AndReadsItBack() + { + Assert.Equal("02", WriteOne(FixedWidthLayout.FromAttributes(), new EnumCodeModel { Side = Side.Sell })); + Assert.Equal("01", Format(new GenEnumCodeModel { Side = Side.Buy })); + Assert.True(FixedWidth.TryParse("02"u8, Ctx, out var parsed)); + Assert.Equal(Side.Sell, parsed.Side); + } + } +} From e06bc7129c10e75ded4ba95a26ddc75b9155ddc1 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sun, 25 Oct 2026 13:48:32 -0300 Subject: [PATCH 79/85] feat(native): read batches and Arrow streams from a forward-only stream Follows ExcelReader's xl_stream. An fw_stream is a read callback, which returns the bytes it copied, 0 at the end or -1 after fw_set_source_error, and an optional release the library calls once it is done, including when opening fails. fw_reader_open_stream and fw_read_arrow_stream_stream read it forward in constant memory, skipping the header lines through a buffer instead of seeking. Python's readers take any object with readinto() or read(): read_batches and read_arrow_stream pull from it as batches are taken, the whole-table reads read it into memory first, and an io.BytesIO is read in place from its position. A Python exception in read() reaches the caller with its message. The C++ wrapper takes a std::istream& in Reader::open_stream, TypedReader::open_stream and read_arrow_stream. --- CHANGELOG.md | 6 + cpp/README.md | 16 +- cpp/tests/arrow.cpp | 12 + cpp/tests/read.cpp | 19 ++ python/README.md | 15 +- python/src/fixedwidth/_native.py | 16 + python/src/fixedwidth/reader.py | 89 +++++- python/tests/test_stream.py | 104 ++++++ .../Arrow/ArrowApi.Stream.cs | 8 + src/FixedWidthParser.Native/Exports.Arrow.cs | 23 ++ src/FixedWidthParser.Native/Exports.Reader.cs | 34 ++ src/FixedWidthParser.Native/README.md | 11 +- .../Reading/CallbackReadStream.cs | 141 +++++++++ .../Reading/NativeTableReader.cs | 7 +- .../Reading/ReaderApi.cs | 68 +++- .../include/fixedwidth.def | 3 + .../include/fixedwidth.h | 19 ++ .../include/fixedwidth.hpp | 49 +++ .../include/fixedwidth_arrow.h | 3 + .../include/fixedwidth_arrow.hpp | 13 + .../Native/NativeStreamReaderTests.cs | 298 ++++++++++++++++++ 21 files changed, 932 insertions(+), 22 deletions(-) create mode 100644 python/tests/test_stream.py create mode 100644 src/FixedWidthParser.Native/Reading/CallbackReadStream.cs create mode 100644 tests/FixedWidthParser.Tests/Native/NativeStreamReaderTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index ea1e79c..2d8a9d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,12 @@ Windows-1252 after registering `CodePagesEncodingProvider`). (`fixedwidth.hpp`) that also maps structs to columns through `FixedWidthMapper`. It installs as a CMake package (`find_package(fixedwidth)`) or through FetchContent, and downloads the release's native library for the platform, so it needs no .NET SDK. +- Batched and Arrow-stream reads from a forward-only source, as ExcelReader's `xl_stream`: an `fw_stream` of + `read` and `release` callbacks (`fw_reader_open_stream`, `fw_read_arrow_stream_stream`, and + `fw_set_source_error` for a callback's failure). Header lines are skipped without seeking, and the batches + stay in constant memory. Python's readers take any object with `readinto()` or `read()` (an open file, + `gzip.open`, `zipfile.ZipFile.open`), and an `io.BytesIO` is read in place; the C++ wrapper takes a + `std::istream&` (`Reader::open_stream`, `TypedReader::open_stream`, `read_arrow_stream`). - Releases publish the Python package to PyPI as `fixedwidthparser`, with wheels for Windows x64, macOS arm64 and x64, and Linux glibc and musl on x64 and arm64, and attach each platform's native library to the GitHub Release. diff --git a/cpp/README.md b/cpp/README.md index 9687bda..bf20fe1 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -57,9 +57,19 @@ rest of `ColumnSpec`: `scale`, `format`, `trim_char`, `trim_mode`, `alignment`, `TableView` keeps the library's columns and converts a row on access; a `std::string_view` field points into it. `view[i]` is unchecked, `view.at(i)` returns an empty optional out of range, and `view.table()` is -the `fw::Table` underneath, with `lines_read()` and `lines_skipped()`. `fw::TypedReader::open_file` or -`open_memory` reads in batches: `next()` returns the next `TableView`, or an empty optional at the end. -`fw::layout_for()` returns the `fw::Layout` the bindings describe, for the untyped API. +the `fw::Table` underneath, with `lines_read()` and `lines_skipped()`. `fw::TypedReader::open_file`, +`open_memory` or `open_stream` reads in batches: `next()` returns the next `TableView`, or an empty optional at +the end. `fw::layout_for()` returns the `fw::Layout` the bindings describe, for the untyped API. + +`fw::Reader::open_stream`, `TypedReader::open_stream` and `fw::read_arrow_stream` take a `std::istream&` and +read it forward a batch at a time, in constant memory, so a decompressing or network stream works. The stream +must outlive the reader; a stream that turns `bad()` fails the read. + +```cpp +std::ifstream file("COTAHIST_A2023.TXT", std::ios::binary); +auto reader = fw::TypedReader::open_stream(file, {.read = {.header_lines = 1, .skip_invalid = true}}); +while (auto batch = reader->next(); batch && *batch) { /* (*batch)->size() rows */ } +``` An integer field narrower than `int64_t` that a value does not fit fails the read with `FW_INVALID_ARGUMENT` instead of wrapping. diff --git a/cpp/tests/arrow.cpp b/cpp/tests/arrow.cpp index b2b5726..4c41c9d 100644 --- a/cpp/tests/arrow.cpp +++ b/cpp/tests/arrow.cpp @@ -3,6 +3,8 @@ #include #include +#include +#include int main() { @@ -30,6 +32,16 @@ int main() } CHECK(rows == 3); + std::istringstream in{std::string(content)}; + auto from_stream = fw::read_arrow_stream(in, layout, {.read = skip_header_and_invalid(), .batch_rows = 2}); + CHECK(from_stream.has_value()); + int64_t streamed_rows = 0; + while (from_stream->get()->get_next(from_stream->get(), &batch) == 0 && batch.release != nullptr) { + streamed_rows += batch.length; + batch.release(&batch); + } + CHECK(streamed_rows == 3); + std::puts("arrow ok"); return 0; } diff --git a/cpp/tests/read.cpp b/cpp/tests/read.cpp index 8f1d340..e0848a6 100644 --- a/cpp/tests/read.cpp +++ b/cpp/tests/read.cpp @@ -1,7 +1,9 @@ #include "common.hpp" #include +#include #include +#include static void check_table(const fw::Table& table) { @@ -59,6 +61,23 @@ int main() } CHECK(rows == 3); + // The same batches from a std::istream, read forward without seeking. + std::istringstream in{std::string(content)}; + auto streamed = fw::Reader::open_stream(in, layout, {.read = skip_header_and_invalid(), .batch_rows = 2}); + CHECK(streamed.has_value()); + std::vector codes; + while (true) { + auto batch = streamed->next(); + CHECK(batch.has_value()); + if (!batch->has_value()) { + break; + } + for (int64_t row = 0; row < (*batch)->row_count(); ++row) { + codes.emplace_back((*batch)->string(1, row)); + } + } + CHECK((codes == std::vector{"PETR4", "VALE3", "ITUB4"})); + std::vector overlap{ {.name = "a", .start = 0, .length = 4, .type = fw::Type::String}, {.name = "b", .start = 2, .length = 4, .type = fw::Type::String}, diff --git a/python/README.md b/python/README.md index 445e72b..c98233f 100644 --- a/python/README.md +++ b/python/README.md @@ -39,7 +39,20 @@ df = fw.read_pandas("COTAHIST_A2023.TXT", layout, options) | `fw.read_batches(source, layout, fw.ReaderOptions(batch_rows=...))` | an iterator of `fw.Table` | | `fw.read_arrow_stream(source, layout, fw.ReaderOptions(...))` | an Arrow PyCapsule stream for `polars.DataFrame(...)` or `pyarrow.RecordBatchReader.from_stream(...)` | -`source` is a path, or `bytes`/`bytearray`/`memoryview` to read from memory. Every optional setting lives in +`source` is a path, `bytes`/`bytearray`/`memoryview`/`io.BytesIO` to read from memory (a `BytesIO` in place, +from its position), or any stream with `readinto()` or `read()`: an open file, a pipe, `gzip.open(...)`, +`zipfile.ZipFile(...).open(...)`. `read_batches` and `read_arrow_stream` pull from a stream as the batches are +taken, in constant memory; the whole-table reads read it into memory first, since they split it into byte ranges. + +```python +import zipfile + +with zipfile.ZipFile("COTAHIST_A2023.ZIP") as archive, archive.open("COTAHIST_A2023.TXT") as stream: + for batch in fw.read_batches(stream, layout, fw.ReaderOptions(read=options, batch_rows=100_000)): + ... +``` + +Every optional setting lives in `fw.ReadOptions`: `header_lines`, `skip_invalid`, `degree_of_parallelism` (0 = one thread per processor, 1 = sequential), `encoding` (a code page or `"latin-1"`, `"cp1252"`, ...), `max_line_length` and `skip_byte_order_mark`. A parallel read gives the same rows, in the same order, as a sequential one. diff --git a/python/src/fixedwidth/_native.py b/python/src/fixedwidth/_native.py index b3369f1..947865a 100644 --- a/python/src/fixedwidth/_native.py +++ b/python/src/fixedwidth/_native.py @@ -68,6 +68,19 @@ class ReaderOptions(ctypes.Structure): _fields_ = [("struct_size", ctypes.c_int32), ("read", ReadOptions), ("batch_rows", ctypes.c_int64)] +StreamRead = ctypes.CFUNCTYPE(ctypes.c_int64, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64) +StreamRelease = ctypes.CFUNCTYPE(None, ctypes.c_void_p) + + +class NativeStream(ctypes.Structure): + _fields_ = [ + ("struct_size", ctypes.c_int32), + ("user_data", ctypes.c_void_p), + ("read", StreamRead), + ("release", StreamRelease), + ] + + class Column(ctypes.Structure): _fields_ = [ ("type", ctypes.c_int32), @@ -172,12 +185,15 @@ class ArrowArrayStream(ctypes.Structure): "fw_free_table": (None, [_P(Table)]), "fw_reader_open_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, ctypes.c_void_p, _P(ReaderOptions), _P(ctypes.c_void_p)]), "fw_reader_open_memory": (ctypes.c_int32, [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p, _P(ReaderOptions), _P(ctypes.c_void_p)]), + "fw_reader_open_stream": (ctypes.c_int32, [_P(NativeStream), ctypes.c_void_p, _P(ReaderOptions), _P(ctypes.c_void_p)]), + "fw_set_source_error": (None, [ctypes.c_char_p, ctypes.c_int32]), "fw_reader_next": (ctypes.c_int32, [ctypes.c_void_p, _P(Table)]), "fw_reader_close": (None, [ctypes.c_void_p]), "fw_read_arrow_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, ctypes.c_void_p, _P(ReadOptions), _P(ArrowArray), _P(ArrowSchema), _P(ReadResult)]), "fw_read_arrow_memory": (ctypes.c_int32, [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p, _P(ReadOptions), _P(ArrowArray), _P(ArrowSchema), _P(ReadResult)]), "fw_read_arrow_stream_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, ctypes.c_void_p, _P(ReaderOptions), _P(ArrowArrayStream)]), "fw_read_arrow_stream_memory": (ctypes.c_int32, [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p, _P(ReaderOptions), _P(ArrowArrayStream)]), + "fw_read_arrow_stream_stream": (ctypes.c_int32, [_P(NativeStream), ctypes.c_void_p, _P(ReaderOptions), _P(ArrowArrayStream)]), "fw_write_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, _P(Table), ctypes.c_void_p, _P(WriteOptions)]), "fw_write_memory": (ctypes.c_int32, [_P(Table), ctypes.c_void_p, _P(WriteOptions), _P(Buffer)]), "fw_free_buffer": (None, [_P(Buffer)]), diff --git a/python/src/fixedwidth/reader.py b/python/src/fixedwidth/reader.py index 9c7721d..71e7f9f 100644 --- a/python/src/fixedwidth/reader.py +++ b/python/src/fixedwidth/reader.py @@ -3,9 +3,12 @@ from __future__ import annotations import ctypes +import io +import itertools +import operator import os from collections.abc import Iterator -from typing import Any, Union +from typing import IO, Any, Union from fixedwidth import _native from fixedwidth.arrow_stream import ArrowStream @@ -13,11 +16,18 @@ from fixedwidth.table import Table, _Owner from fixedwidth.types import ReaderOptions, ReadOptions, code_page -Source = Union[str, "os.PathLike[str]", bytes, bytearray, memoryview] +Source = Union[str, "os.PathLike[str]", bytes, bytearray, memoryview, IO[bytes]] + + +def _is_stream(source: object) -> bool: + """An object with readinto() or read() that is not an in-memory buffer: an open file, a pipe, a decompressor.""" + return not isinstance(source, (str, os.PathLike, bytes, bytearray, memoryview, io.BytesIO)) and ( + hasattr(source, "readinto") or hasattr(source, "read") + ) class _Input: - """A file path or a buffer, as the pointer + length arguments the native calls take.""" + """A file path or a buffer, as the pointer + length arguments the native calls take. A stream is read whole.""" def __init__(self, source: Source) -> None: self.keep: object = None @@ -25,6 +35,13 @@ def __init__(self, source: Source) -> None: self.path, self.path_len = _native.utf8(os.fspath(source)) return self.path = None + if isinstance(source, io.BytesIO): + # Read in place from the current position; the exported buffer stops the BytesIO resizing meanwhile. + source = source.getbuffer()[source.tell():] + elif _is_stream(source): + source = source.read() + if source is None: + raise OSError("the stream has no data ready; reading it needs a blocking stream") view = memoryview(source) if view.readonly or not view.contiguous: data = bytes(view) @@ -70,8 +87,54 @@ def _ref(value: object) -> object: return ctypes.byref(value) if value is not None else None +# Streams the native side is reading, by the key it passes back to the callbacks; release drops them. +_keys = itertools.count(1) +_live: dict[int, object] = {} + + +def _fail(error: BaseException) -> int: + message = (str(error) or type(error).__name__).encode("utf-8", errors="replace") + _native.load_library().fw_set_source_error(message, len(message)) + return -1 + + +@_native.StreamRead +def _read(key: int, buffer: int, length: int) -> int: + try: + stream = _live[key] + readinto = getattr(stream, "readinto", None) + if readinto is None: + data = bytes(stream.read(length)) + if len(data) > length: + raise OSError("the stream returned more bytes than requested") + ctypes.memmove(buffer, data, len(data)) + return len(data) + count = readinto(memoryview((ctypes.c_ubyte * length).from_address(buffer)).cast("B")) + if count is None: + raise OSError("the stream has no data ready; reading it needs a blocking stream") + count = operator.index(count) + if count > length: + raise OSError("the stream returned more bytes than requested") + return count + except BaseException as error: # noqa: BLE001 - every failure, KeyboardInterrupt included, must reach the native side + return _fail(error) + + +@_native.StreamRelease +def _release(key: int) -> None: + _live.pop(key, None) + + +def _stream(source: object) -> _native.NativeStream: + """An fw_stream over `source`, kept alive until the native side releases it.""" + key = next(_keys) + _live[key] = source + return _native.NativeStream(ctypes.sizeof(_native.NativeStream), key, _read, _release) + + def read(source: Source, layout: Layout, options: ReadOptions | None = None) -> Table: - """Reads a whole file or buffer into a Table (see `ReadOptions` for header lines, skipping and threads).""" + """Reads a whole file, buffer or stream into a Table (see `ReadOptions` for header lines, skipping and threads). + A stream is read into memory first, as the parallel read needs all of it; `read_batches()` streams it instead.""" lib = _native.load_library() data = _Input(source) raw = _read_options(options) @@ -119,12 +182,15 @@ def read_polars(source: Source, layout: Layout, options: ReadOptions | None = No def read_batches(source: Source, layout: Layout, options: ReaderOptions | None = None) -> Iterator[Table]: - """Reads sequentially, yielding a Table of at most `options.batch_rows` rows at a time.""" + """Reads sequentially, yielding a Table of at most `options.batch_rows` rows at a time. A stream (an open file, + a pipe, `gzip.open(...)`, `zipfile.ZipFile(...).open(...)`) is read as the batches are taken, in constant memory.""" lib = _native.load_library() - data = _Input(source) raw = _reader_options(options) reader = ctypes.c_void_p() - if data.is_file: + if _is_stream(source): + data = None + status = lib.fw_reader_open_stream(ctypes.byref(_stream(source)), layout.handle, _ref(raw), ctypes.byref(reader)) + elif (data := _Input(source)).is_file: status = lib.fw_reader_open_file(data.path, data.path_len, layout.handle, _ref(raw), ctypes.byref(reader)) else: status = lib.fw_reader_open_memory(data.address, data.length, layout.handle, _ref(raw), ctypes.byref(reader)) @@ -144,12 +210,15 @@ def read_batches(source: Source, layout: Layout, options: ReaderOptions | None = def read_arrow_stream(source: Source, layout: Layout, options: ReaderOptions | None = None) -> ArrowStream: """The batches of `read_batches()` as a single-use Arrow PyCapsule stream, for `polars.DataFrame(stream)` - or `pyarrow.RecordBatchReader.from_stream(stream)`. Needs no third-party package to create.""" + or `pyarrow.RecordBatchReader.from_stream(stream)`. Needs no third-party package to create. A stream source is + read lazily, as in `read_batches()`.""" lib = _native.load_library() - data = _Input(source) raw = _reader_options(options) stream = _native.ArrowArrayStream() - if data.is_file: + if _is_stream(source): + data = None + status = lib.fw_read_arrow_stream_stream(ctypes.byref(_stream(source)), layout.handle, _ref(raw), ctypes.byref(stream)) + elif (data := _Input(source)).is_file: status = lib.fw_read_arrow_stream_file(data.path, data.path_len, layout.handle, _ref(raw), ctypes.byref(stream)) else: status = lib.fw_read_arrow_stream_memory(data.address, data.length, layout.handle, _ref(raw), ctypes.byref(stream)) diff --git a/python/tests/test_stream.py b/python/tests/test_stream.py new file mode 100644 index 0000000..64605bf --- /dev/null +++ b/python/tests/test_stream.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import gc +import gzip +import io + +import numpy as np +import pytest + +import fixedwidth as fw +from conftest import CONTENT, layout + +DATA = CONTENT.encode("utf-8") + + +class Trickle: + """A stream with only read(), handing out a few bytes per call, that can fail partway.""" + + def __init__(self, data: bytes, chunk: int = 3, fail_at: int = -1) -> None: + self.data, self.chunk, self.fail_at, self.position = data, chunk, fail_at, 0 + + def read(self, size: int = -1) -> bytes: + if 0 <= self.fail_at <= self.position: + raise ConnectionError("the network went away") + size = len(self.data) if size < 0 else min(size, self.chunk) + out = self.data[self.position : self.position + size] + self.position += len(out) + return out + + +def codes(tables) -> list[str]: + return [code for table in tables for code in table["codneg"]] + + +@pytest.mark.parametrize("make", [lambda: io.BufferedReader(io.BytesIO(DATA)), lambda: Trickle(DATA)]) +def test_read_batches_from_a_stream_equals_memory(options, make): + reader = fw.ReaderOptions(read=options, batch_rows=2) + expected = list(fw.read_batches(DATA, layout(), reader)) + + batches = list(fw.read_batches(make(), layout(), reader)) + + assert [b.num_rows for b in batches] == [b.num_rows for b in expected] + assert codes(batches) == codes(expected) == ["PETR4", "VALE3", "ITUB4"] + assert np.array_equal(np.concatenate([b["quatot"] for b in batches]), np.concatenate([b["quatot"] for b in expected])) + + +def test_read_batches_from_gzip(tmp_path, options): + path = tmp_path / "cotahist.txt.gz" + with gzip.open(path, "wb") as out: + out.write(DATA) + + with gzip.open(path, "rb") as stream: + batches = list(fw.read_batches(stream, layout(), fw.ReaderOptions(read=options))) + + assert codes(batches) == ["PETR4", "VALE3", "ITUB4"] + + +def test_whole_read_from_a_stream_equals_memory(options): + table = fw.read(Trickle(DATA, chunk=1000), layout(), options) + + assert table["codneg"] == fw.read(DATA, layout(), options)["codneg"] + assert table.lines_skipped == 3 # the header, the invalid line and the trailer + + +def test_bytesio_is_read_from_its_position(options): + buffer = io.BytesIO(b"junk" + DATA) + buffer.seek(4) + + assert fw.read(buffer, layout(), options)["codneg"] == ["PETR4", "VALE3", "ITUB4"] + assert codes(fw.read_batches(buffer, layout(), fw.ReaderOptions(read=options))) == ["PETR4", "VALE3", "ITUB4"] + + +def test_a_failing_stream_reports_its_error(options): + with pytest.raises(fw.FixedWidthError, match="the network went away"): + list(fw.read_batches(Trickle(DATA, fail_at=80), layout(), fw.ReaderOptions(read=options, batch_rows=1))) + + +def test_parse_error_counts_the_header_lines(): + strict = fw.ReaderOptions(read=fw.ReadOptions(header_lines=1, degree_of_parallelism=1)) + + with pytest.raises(fw.ParseError) as error: + list(fw.read_batches(Trickle(DATA), layout(), strict)) + + assert error.value.line_number == 5 + + +def test_the_stream_is_released_when_the_reader_closes(options): + from fixedwidth import reader as module + + batches = fw.read_batches(Trickle(DATA), layout(), fw.ReaderOptions(read=options, batch_rows=1)) + next(batches) + assert len(module._live) == 1 + batches.close() + gc.collect() + + assert module._live == {} + + +def test_arrow_stream_from_a_stream(options): + pl = pytest.importorskip("polars") + + frame = pl.DataFrame(fw.read_arrow_stream(Trickle(DATA), layout(), fw.ReaderOptions(read=options, batch_rows=2))) + + assert frame["codneg"].to_list() == ["PETR4", "VALE3", "ITUB4"] diff --git a/src/FixedWidthParser.Native/Arrow/ArrowApi.Stream.cs b/src/FixedWidthParser.Native/Arrow/ArrowApi.Stream.cs index 9d15ed2..63dc8c8 100644 --- a/src/FixedWidthParser.Native/Arrow/ArrowApi.Stream.cs +++ b/src/FixedWidthParser.Native/Arrow/ArrowApi.Stream.cs @@ -46,6 +46,14 @@ internal static int OpenStreamMemory(byte* data, long length, NativeLayout layou return status == NativeStatus.Ok ? Publish(reader!, outStream) : status; } + /// Reads lazily and owns it: it is disposed when this fails or the stream is released. + internal static int OpenStreamStream(Stream source, NativeLayout layout, NativeReaderOptionsRaw* options, ArrowArrayStream* outStream) + { + *outStream = default; + int status = ReaderApi.CreateStream(source, layout, options, out var reader); + return status == NativeStatus.Ok ? Publish(reader!, outStream) : status; + } + private static int Publish(NativeTableReader reader, ArrowArrayStream* outStream) { *outStream = new ArrowArrayStream diff --git a/src/FixedWidthParser.Native/Exports.Arrow.cs b/src/FixedWidthParser.Native/Exports.Arrow.cs index 1fbbf1f..3a93d0b 100644 --- a/src/FixedWidthParser.Native/Exports.Arrow.cs +++ b/src/FixedWidthParser.Native/Exports.Arrow.cs @@ -52,6 +52,29 @@ public static int ReadArrowStreamMemory(byte* data, long dataLength, nint layout return WithLayout(layout, l => ArrowApi.OpenStreamMemory(data, dataLength, l, options, outStream)); } + [UnmanagedCallersOnly(EntryPoint = "fw_read_arrow_stream_stream")] + public static int ReadArrowStreamStream(NativeStreamRaw* stream, nint layout, NativeReaderOptionsRaw* options, ArrowArrayStream* outStream) + { + if (outStream is null) + { + return InvalidArgument("fw_read_arrow_stream_stream needs out_stream."); + } + *outStream = default; +#pragma warning disable CA2000 // The reader takes the stream over, or it is disposed below. + if (!CallbackReadStream.TryCreate(stream, "fw_read_arrow_stream_stream", out var source)) + { + return NativeStatus.InvalidArgument; + } + int status = WithLayout(layout, l => ArrowApi.OpenStreamStream(source, l, options, outStream)); + if (status != NativeStatus.Ok) + { + // A layout that is not live never reached OpenStreamStream, which releases the stream on its own failures. + source.Dispose(); + } +#pragma warning restore CA2000 + return status; + } + [UnmanagedCallersOnly] internal static void ReleaseArrowSchemaCallback(ArrowSchema* schema) { diff --git a/src/FixedWidthParser.Native/Exports.Reader.cs b/src/FixedWidthParser.Native/Exports.Reader.cs index 92fdfd3..6a12dc3 100644 --- a/src/FixedWidthParser.Native/Exports.Reader.cs +++ b/src/FixedWidthParser.Native/Exports.Reader.cs @@ -37,6 +37,40 @@ public static int ReaderOpenMemory(byte* data, long dataLength, nint layout, Nat }); } + [UnmanagedCallersOnly(EntryPoint = "fw_reader_open_stream")] + public static int ReaderOpenStream(NativeStreamRaw* stream, nint layout, NativeReaderOptionsRaw* options, nint* outReader) + { + if (outReader is null) + { + return InvalidArgument("fw_reader_open_stream needs out_reader."); + } + *outReader = 0; +#pragma warning disable CA2000 // The reader takes the stream over, or it is disposed below. + if (!CallbackReadStream.TryCreate(stream, "fw_reader_open_stream", out var source)) + { + return NativeStatus.InvalidArgument; + } + int status = WithLayout(layout, l => + { + int opened = ReaderApi.OpenStream(source, l, options, out nint reader); + *outReader = reader; + return opened; + }); + if (status != NativeStatus.Ok) + { + // A layout that is not live never reached OpenStream, which releases the stream on its own failures. + source.Dispose(); + } +#pragma warning restore CA2000 + return status; + } + + [UnmanagedCallersOnly(EntryPoint = "fw_set_source_error")] + public static void SetSourceError(byte* message, int length) + { + SourceErrors.Set(message is null || length <= 0 ? null : System.Text.Encoding.UTF8.GetString(message, length)); + } + [UnmanagedCallersOnly(EntryPoint = "fw_reader_next")] public static int ReaderNext(nint reader, NativeTableRaw* outTable) { diff --git a/src/FixedWidthParser.Native/README.md b/src/FixedWidthParser.Native/README.md index 977d2f6..6860f58 100644 --- a/src/FixedWidthParser.Native/README.md +++ b/src/FixedWidthParser.Native/README.md @@ -34,10 +34,17 @@ struct starts with `struct_size`; `NULL` means all defaults. | Errors | `fw_abi_version`, `fw_last_error`, `fw_last_parse_error` | | Layouts | `fw_layout_create`, `fw_layout_free` | | Columnar reads | `fw_read_file`, `fw_read_memory`, `fw_free_table` | -| Batches | `fw_reader_open_file`, `fw_reader_open_memory`, `fw_reader_next`, `fw_reader_close` | -| Arrow | `fw_read_arrow_file`, `fw_read_arrow_memory`, `fw_read_arrow_stream_file`, `fw_read_arrow_stream_memory` | +| Batches | `fw_reader_open_file`, `fw_reader_open_memory`, `fw_reader_open_stream`, `fw_reader_next`, `fw_reader_close` | +| Arrow | `fw_read_arrow_file`, `fw_read_arrow_memory`, `fw_read_arrow_stream_file`, `fw_read_arrow_stream_memory`, `fw_read_arrow_stream_stream` | +| Streams | `fw_set_source_error` | | Writes | `fw_write_file`, `fw_write_memory`, `fw_free_buffer` | +A batched read or an Arrow stream can pull from an `fw_stream` instead of a file or memory: a `read` callback +that fills a buffer and returns the count, 0 at the end or -1 after `fw_set_source_error`, and an optional +`release` the library calls once it is done. It reads forward only, skipping the header lines as it goes, so a +pipe, a socket or a decompressor works; the batches stay in constant memory. Whole-table reads take a file or +memory, since they split the source into byte ranges. + `include/fixedwidth.def` lists them for import-library generation on Windows; a test fails when it drifts from the `[UnmanagedCallersOnly]` entry points. diff --git a/src/FixedWidthParser.Native/Reading/CallbackReadStream.cs b/src/FixedWidthParser.Native/Reading/CallbackReadStream.cs new file mode 100644 index 0000000..9560381 --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/CallbackReadStream.cs @@ -0,0 +1,141 @@ +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +namespace FixedWidthParser.Native.Reading +{ + /// fw_stream. + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct NativeStreamRaw + { + public int StructSize; + public void* UserData; + public delegate* unmanaged Read; + public delegate* unmanaged Release; + } + + /// + /// The caller's fw_stream as a forward-only : ExcelReader's xl_stream. Disposing it + /// calls release once. + /// + internal sealed unsafe class CallbackReadStream : Stream + { + private readonly delegate* unmanaged _read; + private readonly delegate* unmanaged _release; + private readonly void* _userData; + private int _released; + + private CallbackReadStream(NativeStreamRaw raw) + { + _read = raw.Read; + _release = raw.Release; + _userData = raw.UserData; + } + + /// Wraps , or explains why it cannot; release is not called then. + internal static bool TryCreate(NativeStreamRaw* raw, string function, [NotNullWhen(true)] out CallbackReadStream? stream) + { + stream = null; + if (raw is null || raw->StructSize != sizeof(NativeStreamRaw) || raw->Read is null) + { + NativeApi.SetLastError($"{function} needs a non-NULL stream whose struct_size is sizeof(fw_stream) and whose read is set."); + return false; + } + stream = new CallbackReadStream(*raw); + return true; + } + + public override bool CanRead => true; + + public override bool CanSeek => false; + + public override bool CanWrite => false; + + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override int Read(Span buffer) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _released) != 0, this); + if (buffer.IsEmpty) + { + return 0; + } + SourceErrors.Clear(); + long got; + fixed (byte* destination = buffer) + { + got = _read(_userData, destination, buffer.Length); + } + if (got < 0 || got > buffer.Length) + { + throw SourceErrors.Failed("fw_stream.read", got, buffer.Length); + } + return (int)got; + } + + public override int Read(byte[] buffer, int offset, int count) + { + return Read(buffer.AsSpan(offset, count)); + } + + public override void Flush() + { + // Nothing to flush: the stream only reads. + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + protected override void Dispose(bool disposing) + { + if (Interlocked.Exchange(ref _released, 1) == 0 && _release is not null) + { + _release(_userData); + } + base.Dispose(disposing); + } + } + + /// The message a stream callback leaves through fw_set_source_error before returning -1. + internal static class SourceErrors + { + [ThreadStatic] + private static string? _pending; + + internal static void Set(string? message) + { + _pending = string.IsNullOrEmpty(message) ? null : message; + } + + internal static void Clear() + { + _pending = null; + } + + internal static IOException Failed(string where, long got, long requested) + { + string? message = _pending; + _pending = null; + return got == -1 + ? new IOException(message ?? $"{where} failed.") + : new IOException($"{where} returned {got} for a request of {requested} bytes, which is not a valid result."); + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/NativeTableReader.cs b/src/FixedWidthParser.Native/Reading/NativeTableReader.cs index 9e4ebbe..f1a274c 100644 --- a/src/FixedWidthParser.Native/Reading/NativeTableReader.cs +++ b/src/FixedWidthParser.Native/Reading/NativeTableReader.cs @@ -31,16 +31,17 @@ internal sealed unsafe class NativeTableReader : IDisposable private LineBufferState _lines; private bool _done; - internal NativeTableReader(Stream stream, long dataStart, long headerLines, NativeLayout layout, ReadSettings settings, long batchRows) + /// Positioned at the first data line, past header lines. + /// Whether nothing of the source was skipped, so a byte order mark may begin it. + internal NativeTableReader(Stream stream, bool atStart, long headerLines, NativeLayout layout, ReadSettings settings, long batchRows) { _stream = stream; - _stream.Position = dataStart; _layout = layout; _options = settings.ReaderOptions(new StrongBox(), degreeOfParallelism: 1, chunkSize: 0); _context = _options.Context; _batchRows = batchRows == 0 ? DefaultBatchRows : batchRows; _lines = default; - _lines.Rent(_options, dataStart == 0 && _options.SkipsByteOrderMark); + _lines.Rent(_options, atStart && _options.SkipsByteOrderMark); _lines.LineNumber = headerLines; } diff --git a/src/FixedWidthParser.Native/Reading/ReaderApi.cs b/src/FixedWidthParser.Native/Reading/ReaderApi.cs index ee29b26..c954ede 100644 --- a/src/FixedWidthParser.Native/Reading/ReaderApi.cs +++ b/src/FixedWidthParser.Native/Reading/ReaderApi.cs @@ -23,6 +23,14 @@ internal static int OpenMemory(byte* data, long length, NativeLayout layout, Nat return status; } + /// + internal static int OpenStream(Stream source, NativeLayout layout, NativeReaderOptionsRaw* options, out nint reader) + { + int status = CreateStream(source, layout, options, out var created); + reader = status == NativeStatus.Ok ? NativeHandleTable.Register(created!) : 0; + return status; + } + #pragma warning restore CA2000, IDISP001 internal static int CreateFile(ReadOnlySpan path, NativeLayout layout, NativeReaderOptionsRaw* options, out NativeTableReader? reader) @@ -38,7 +46,8 @@ internal static int CreateFile(ReadOnlySpan path, NativeLayout layout, Nat { stream = new FileStream(Encoding.UTF8.GetString(path), FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1, FileOptions.SequentialScan); long dataStart = ReadApi.HeaderEnd(new FileSource(stream.SafeFileHandle), settings!.HeaderLines, out long headerLines); - reader = new NativeTableReader(stream, dataStart, headerLines, layout, settings, batchRows); + stream.Position = dataStart; + reader = new NativeTableReader(stream, dataStart == 0, headerLines, layout, settings, batchRows); return NativeStatus.Ok; } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException) @@ -59,12 +68,65 @@ internal static int CreateMemory(byte* data, long length, NativeLayout layout, N } long dataStart = ReadApi.HeaderEnd(new PointerSource(data, length), settings!.HeaderLines, out long headerLines); #pragma warning disable CA2000 // The reader owns and disposes the stream. - var stream = new UnmanagedMemoryStream(data, length); + var stream = new UnmanagedMemoryStream(data, length) { Position = dataStart }; #pragma warning restore CA2000 - reader = new NativeTableReader(stream, dataStart, headerLines, layout, settings, batchRows); + reader = new NativeTableReader(stream, dataStart == 0, headerLines, layout, settings, batchRows); return NativeStatus.Ok; } + /// + /// Reads forward from where it is, skipping the header lines as it goes. Takes ownership: + /// is disposed, so its release runs, if this fails, or with the reader. + /// + internal static int CreateStream(Stream source, NativeLayout layout, NativeReaderOptionsRaw* options, out NativeTableReader? reader) + { + reader = null; + int status = TrySettings(options, out var settings, out long batchRows); + if (status != NativeStatus.Ok) + { +#pragma warning disable IDISP007 // CreateStream owns the source it is handed. + source.Dispose(); +#pragma warning restore IDISP007 + return status; + } + // Buffered so the header skip reads a byte at a time from memory, not from the caller's callback. +#pragma warning disable CA2000 // The reader owns and disposes the stream, and with it the source. + var stream = new BufferedStream(source, StreamBufferSize); +#pragma warning restore CA2000 + try + { + long headerLines = SkipLines(stream, settings!.HeaderLines); + reader = new NativeTableReader(stream, settings.HeaderLines == 0, headerLines, layout, settings, batchRows); + return NativeStatus.Ok; + } + catch (IOException ex) + { + stream.Dispose(); + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + + private const int StreamBufferSize = 64 * 1024; + + /// Skips physical lines, as counts them, and returns how many there were. + private static long SkipLines(Stream stream, long count) + { + long skipped = 0; + bool partial = false; + while (skipped < count) + { + int b = stream.ReadByte(); + if (b < 0) + { + return skipped + (partial ? 1 : 0); + } + partial = b != '\n'; + skipped += partial ? 0 : 1; + } + return skipped; + } + internal static int Next(nint handle, NativeTableRaw* table) { *table = default; diff --git a/src/FixedWidthParser.Native/include/fixedwidth.def b/src/FixedWidthParser.Native/include/fixedwidth.def index 89679f2..2f09346 100644 --- a/src/FixedWidthParser.Native/include/fixedwidth.def +++ b/src/FixedWidthParser.Native/include/fixedwidth.def @@ -10,11 +10,14 @@ EXPORTS fw_read_arrow_memory fw_read_arrow_stream_file fw_read_arrow_stream_memory + fw_read_arrow_stream_stream fw_read_file fw_read_memory fw_reader_close fw_reader_next fw_reader_open_file fw_reader_open_memory + fw_reader_open_stream + fw_set_source_error fw_write_file fw_write_memory diff --git a/src/FixedWidthParser.Native/include/fixedwidth.h b/src/FixedWidthParser.Native/include/fixedwidth.h index bddba5f..fd7165b 100644 --- a/src/FixedWidthParser.Native/include/fixedwidth.h +++ b/src/FixedWidthParser.Native/include/fixedwidth.h @@ -172,6 +172,25 @@ int32_t fw_reader_open_file(const uint8_t* path, int32_t path_len, const fw_layo int32_t fw_reader_open_memory(const uint8_t* data, int64_t data_len, const fw_layout* layout, const fw_reader_options* options, fw_reader** out_reader); +/* A forward-only source the library pulls from, for a pipe, a socket, a decompressor or a language's own stream + * objects. read copies up to len bytes into buf and returns how many, 0 at the end, or -1 on failure, after + * fw_set_source_error to say why; it is called from the thread reading. release, which may be NULL, is called + * once the library is done with the stream: when the reader is closed or the Arrow stream released, or right + * away if opening fails after the stream was accepted. A stream rejected as an argument is not released. */ +typedef struct fw_stream { + int32_t struct_size; + void* user_data; + int64_t (*read)(void* user_data, uint8_t* buf, int64_t len); + void (*release)(void* user_data); +} fw_stream; + +/* Reads the stream from where it stands, skipping read.header_lines first; it never seeks. */ +int32_t fw_reader_open_stream(const fw_stream* stream, const fw_layout* layout, + const fw_reader_options* options, fw_reader** out_reader); + +/* Called by a read callback before it returns -1: the message becomes the failure's fw_last_error. */ +void fw_set_source_error(const uint8_t* message, int32_t len); + /* FW_OK with a table the caller frees with fw_free_table, FW_EOF at the end. After a failure the * reader is finished: later calls return FW_EOF. */ int32_t fw_reader_next(fw_reader* reader, fw_table* out_table); diff --git a/src/FixedWidthParser.Native/include/fixedwidth.hpp b/src/FixedWidthParser.Native/include/fixedwidth.hpp index 2f77a80..b8a2c02 100644 --- a/src/FixedWidthParser.Native/include/fixedwidth.hpp +++ b/src/FixedWidthParser.Native/include/fixedwidth.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -91,6 +92,33 @@ namespace fw { return reinterpret_cast(text.data()); } + + // fw_stream.read over a std::istream: a short count at the end, -1 once the stream is bad. An exception + // must not unwind into the library, so it fails the read too. + inline int64_t read_istream(void* user_data, uint8_t* buffer, int64_t length) noexcept + { + constexpr std::string_view failed = "the std::istream failed"; + try + { + auto& in = *static_cast(user_data); + in.read(reinterpret_cast(buffer), static_cast(length)); + if (!in.bad()) + { + return static_cast(in.gcount()); + } + } + catch (...) + { + } + fw_set_source_error(bytes(failed), static_cast(failed.size())); + return -1; + } + + /// An fw_stream reading `in`, which the caller keeps alive; nothing to release. + inline fw_stream istream_source(std::istream& in) + { + return {.struct_size = sizeof(fw_stream), .user_data = &in, .read = &read_istream, .release = nullptr}; + } } /// One column: name, zero-based byte start, byte length and type, plus everything optional. @@ -320,6 +348,19 @@ namespace fw return Reader(reader); } + /// Reads `in` forward, a batch at a time, without seeking: it must outlive the reader. + static Result open_stream(std::istream& in, const Layout& layout, const ReaderOptions& options = {}) + { + const fw_reader_options raw = options.to_c(); + const fw_stream stream = detail::istream_source(in); + fw_reader* reader = nullptr; + if (auto ok = detail::check(fw_reader_open_stream(&stream, layout.get(), &raw, &reader)); !ok) + { + return std::unexpected(ok.error()); + } + return Reader(reader); + } + Reader(Reader&& other) noexcept : reader_(std::exchange(other.reader_, nullptr)) {} Reader& operator=(Reader&& other) noexcept { @@ -740,6 +781,14 @@ namespace fw .transform([](Reader reader) { return TypedReader(std::move(reader)); }); } + /// Reads `in` forward without seeking: it must outlive the reader. + static Result open_stream(std::istream& in, const ReaderOptions& options = {}) + { + return layout_for() + .and_then([&](const Layout& layout) { return Reader::open_stream(in, layout, options); }) + .transform([](Reader reader) { return TypedReader(std::move(reader)); }); + } + /// The next batch, or an empty optional at the end. Result>> next() { diff --git a/src/FixedWidthParser.Native/include/fixedwidth_arrow.h b/src/FixedWidthParser.Native/include/fixedwidth_arrow.h index aa8c46a..0818192 100644 --- a/src/FixedWidthParser.Native/include/fixedwidth_arrow.h +++ b/src/FixedWidthParser.Native/include/fixedwidth_arrow.h @@ -80,6 +80,9 @@ int32_t fw_read_arrow_stream_file(const uint8_t* path, int32_t path_len, const f const fw_reader_options* options, struct ArrowArrayStream* out_stream); int32_t fw_read_arrow_stream_memory(const uint8_t* data, int64_t data_len, const fw_layout* layout, const fw_reader_options* options, struct ArrowArrayStream* out_stream); +/* The same over an fw_stream, which is released with the Arrow stream. */ +int32_t fw_read_arrow_stream_stream(const fw_stream* stream, const fw_layout* layout, + const fw_reader_options* options, struct ArrowArrayStream* out_stream); #ifdef __cplusplus } diff --git a/src/FixedWidthParser.Native/include/fixedwidth_arrow.hpp b/src/FixedWidthParser.Native/include/fixedwidth_arrow.hpp index b1c5a8c..c3df05e 100644 --- a/src/FixedWidthParser.Native/include/fixedwidth_arrow.hpp +++ b/src/FixedWidthParser.Native/include/fixedwidth_arrow.hpp @@ -120,4 +120,17 @@ namespace fw } return stream; } + + /// Reads `in` lazily, without seeking: it must outlive the stream. + inline Result read_arrow_stream(std::istream& in, const Layout& layout, const ReaderOptions& options = {}) + { + ArrowStream stream; + const fw_reader_options raw = options.to_c(); + const fw_stream source = detail::istream_source(in); + if (auto ok = detail::check(fw_read_arrow_stream_stream(&source, layout.get(), &raw, stream.get())); !ok) + { + return std::unexpected(ok.error()); + } + return stream; + } } diff --git a/tests/FixedWidthParser.Tests/Native/NativeStreamReaderTests.cs b/tests/FixedWidthParser.Tests/Native/NativeStreamReaderTests.cs new file mode 100644 index 0000000..871b8c4 --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeStreamReaderTests.cs @@ -0,0 +1,298 @@ +using System.Runtime.InteropServices; +using System.Text; +using Apache.Arrow; +using Apache.Arrow.C; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Arrow; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; +using static FixedWidthParser.Tests.Native.NativeTestReads; + +namespace FixedWidthParser.Tests.Native +{ + /// + /// fw_reader_open_stream and fw_read_arrow_stream_stream over a real fw_stream whose callbacks + /// hand out a few bytes at a time: the batches equal a read of the same bytes from memory, header lines are skipped + /// without seeking, a failing callback's message reaches the caller, and release runs exactly once. + /// + public sealed unsafe class NativeStreamReaderTests + { + private static readonly byte[] Content = Encoding.UTF8.GetBytes( + "HEADER\n" + string.Concat(Enumerable.Range(1, 10).Select(i => i == 5 ? "ZZXX\r\n" : $"AB{i:D2}\n")) + "AB11"); + + /// What the callbacks read: at most bytes per call, failing at . + private sealed class Source(byte[] data, int chunk, int failAt = -1, long overReport = 0) + { + public int Position; + public int Released; + + public long Read(byte* buffer, long length) + { + if (failAt >= 0 && Position >= failAt) + { + byte[] message = "the disk went away"u8.ToArray(); + fixed (byte* m = message) + { + SourceErrors.Set(Encoding.UTF8.GetString(m, message.Length)); + } + return -1; + } + int count = (int)Math.Min(Math.Min(length, Chunk), data.Length - Position); + data.AsSpan(Position, count).CopyTo(new Span(buffer, count)); + Position += count; + return overReport > 0 ? length + overReport : count; + } + + public int Chunk => chunk; + } + + [UnmanagedCallersOnly] + private static long ReadCallback(void* userData, byte* buffer, long length) + { + return ((Source)GCHandle.FromIntPtr((IntPtr)userData).Target!).Read(buffer, length); + } + + [UnmanagedCallersOnly] + private static void ReleaseCallback(void* userData) + { + ((Source)GCHandle.FromIntPtr((IntPtr)userData).Target!).Released++; + } + + /// An fw_stream over ; the handle keeps it alive for the test. + private static CallbackReadStream Wrap(Source source, GCHandle handle) + { + var raw = new NativeStreamRaw + { + StructSize = sizeof(NativeStreamRaw), + UserData = (void*)GCHandle.ToIntPtr(handle), + Read = &ReadCallback, + Release = &ReleaseCallback, + }; + Assert.True(CallbackReadStream.TryCreate(&raw, "test", out var stream)); + return stream; + } + + private static NativeLayout Layout() + { + return Build(new Col("code", 0, 2, TString), new Col("n", 2, 2, TInt64)); + } + + private static NativeReaderOptionsRaw ReaderOptions(long batchRows, long headerLines = 1, bool skipInvalid = true) + { + return new NativeReaderOptionsRaw + { + StructSize = sizeof(NativeReaderOptionsRaw), + Read = Options(headerLines: (int)headerLines, skipInvalid: skipInvalid), + BatchRows = batchRows, + }; + } + + private static List Drain(nint reader, out int lastStatus) + { + var batches = new List(); + while (true) + { + NativeTableRaw table = default; + lastStatus = ReaderApi.Next(reader, &table); + if (lastStatus != NativeStatus.Ok) + { + return batches; + } + batches.Add(Collect(lastStatus, &table, default)); + } + } + + [Theory] + [InlineData(1, 3)] + [InlineData(7, 4)] + [InlineData(4096, 1000)] + public void Stream_EqualsMemory_AndReleasesOnce(int chunk, long batchRows) + { + var layout = Layout(); + var whole = FromMemory(Content, layout, Options(headerLines: 1, skipInvalid: true)); + var source = new Source(Content, chunk); + var handle = GCHandle.Alloc(source); + try + { + var options = ReaderOptions(batchRows); + Assert.Equal(NativeStatus.Ok, ReaderApi.OpenStream(Wrap(source, handle), layout, &options, out nint reader)); + var batches = Drain(reader, out int last); + Assert.Equal(0, source.Released); + ReaderApi.Close(reader); + + Assert.Equal(NativeStatus.Eof, last); + Assert.Equal(whole.Column(0), batches.SelectMany(b => b.Column(0))); + Assert.Equal(whole.Column(1), batches.SelectMany(b => b.Column(1))); + Assert.Equal(1, source.Released); + } + finally + { + handle.Free(); + } + } + + [Theory] + [InlineData("AB01\nAB02\n", 0)] + [InlineData("HEAD\nAB01\nAB02", 1)] + [InlineData("H1\r\n\nAB01\n", 2)] + [InlineData("ONLY A HEADER", 3)] + public void HeaderLinesAndByteOrderMark_MatchMemory(string text, long headerLines) + { + byte[] bytes = Encoding.UTF8.GetBytes(text); + var layout = Layout(); + var expected = FromMemory(bytes, layout, Options(headerLines: (int)headerLines)); + var source = new Source(bytes, 1); + var handle = GCHandle.Alloc(source); + try + { + var options = ReaderOptions(100, headerLines, skipInvalid: false); + Assert.Equal(NativeStatus.Ok, ReaderApi.OpenStream(Wrap(source, handle), layout, &options, out nint reader)); + var batches = Drain(reader, out int last); + ReaderApi.Close(reader); + + Assert.True(last == NativeStatus.Eof, LastError()); + Assert.Equal(expected.Column(1), batches.SelectMany(b => b.Column(1))); + } + finally + { + handle.Free(); + } + } + + [Fact] + public void ParseError_CountsTheSkippedHeaderLines() + { + var source = new Source(Content, 5); + var handle = GCHandle.Alloc(source); + try + { + var options = ReaderOptions(100, skipInvalid: false); + Assert.Equal(NativeStatus.Ok, ReaderApi.OpenStream(Wrap(source, handle), Layout(), &options, out nint reader)); + Drain(reader, out int last); + NativeApi.LastParseError(out long line, out _); + ReaderApi.Close(reader); + + Assert.Equal(NativeStatus.ParseError, last); + Assert.Equal(6, line); + } + finally + { + handle.Free(); + } + } + + [Fact] + public void FailingRead_ReportsTheCallbacksMessage() + { + var source = new Source(Content, 8, failAt: 30); + var handle = GCHandle.Alloc(source); + try + { + var options = ReaderOptions(2); + Assert.Equal(NativeStatus.Ok, ReaderApi.OpenStream(Wrap(source, handle), Layout(), &options, out nint reader)); + Drain(reader, out int last); + + Assert.Equal(NativeStatus.Error, last); + Assert.Equal("the disk went away", LastError()); + ReaderApi.Close(reader); + Assert.Equal(1, source.Released); + } + finally + { + handle.Free(); + } + } + + [Fact] + public void ReadReturningMoreThanAsked_IsAnError() + { + var source = new Source(Content, 4096, overReport: 1); + var handle = GCHandle.Alloc(source); + try + { + var options = ReaderOptions(100, headerLines: 0); + Assert.Equal(NativeStatus.Ok, ReaderApi.OpenStream(Wrap(source, handle), Layout(), &options, out nint reader)); + Drain(reader, out int last); + ReaderApi.Close(reader); + + Assert.Equal(NativeStatus.Error, last); + Assert.Contains("fw_stream.read returned", LastError(), StringComparison.Ordinal); + } + finally + { + handle.Free(); + } + } + + [Fact] + public void BadOptions_ReleaseTheStream() + { + var source = new Source(Content, 16); + var handle = GCHandle.Alloc(source); + try + { + var options = new NativeReaderOptionsRaw { StructSize = 3 }; + Assert.Equal(NativeStatus.InvalidArgument, ReaderApi.OpenStream(Wrap(source, handle), Layout(), &options, out nint reader)); + + Assert.Equal(0, reader); + Assert.Equal(1, source.Released); + } + finally + { + handle.Free(); + } + } + + [Fact] + public void RejectedStruct_IsNotReleased() + { + var raw = new NativeStreamRaw { StructSize = 0, Read = &ReadCallback, Release = &ReleaseCallback }; + Assert.False(CallbackReadStream.TryCreate(&raw, "fw_reader_open_stream", out _)); + Assert.Contains("struct_size is sizeof(fw_stream)", LastError(), StringComparison.Ordinal); + + raw = new NativeStreamRaw { StructSize = sizeof(NativeStreamRaw), Release = &ReleaseCallback }; + Assert.False(CallbackReadStream.TryCreate(&raw, "fw_reader_open_stream", out _)); + } + + [Fact] + public void ArrowStream_OverAStream_EqualsMemory_AndReleasesWithIt() + { + var layout = Layout(); + var whole = FromMemory(Content, layout, Options(headerLines: 1, skipInvalid: true)); + var source = new Source(Content, 3); + var handle = GCHandle.Alloc(source); + try + { + var stream = (CArrowArrayStream*)NativeMemory.AllocZeroed((nuint)sizeof(CArrowArrayStream)); + var options = ReaderOptions(4); + Assert.Equal(NativeStatus.Ok, ArrowApi.OpenStreamStream(Wrap(source, handle), layout, &options, (ArrowArrayStream*)stream)); + var imported = CArrowArrayStreamImporter.ImportArrayStream(stream); + var quantities = new List(); + try + { + // An unsafe context cannot await; the native stream completes every read synchronously. + while (imported.ReadNextRecordBatchAsync(TestContext.Current.CancellationToken).AsTask().GetAwaiter().GetResult() is { } batch) + { + using (batch) + { + var column = (Int64Array)batch.Column(1); + quantities.AddRange(Enumerable.Range(0, column.Length).Select(i => column.GetValue(i))); + } + } + } + finally + { + imported.Dispose(); + } + + Assert.Equal(whole.Column(1), quantities.Cast()); + Assert.Equal(1, source.Released); + } + finally + { + handle.Free(); + } + } + } +} From 3b873ab13b0e37b403235da30135b5ebddc4b216 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sun, 25 Oct 2026 15:08:57 -0300 Subject: [PATCH 80/85] docs: refresh the Python benchmark numbers and document thread safety A fresh 1,000,000-line run against the current build: fw.read takes 0.125 s, 123x pandas.read_fwf, read_pandas 96x and read_polars 114x, and fw.write 0.148 s. The README and CHANGELOG no longer claim 52x, and the benchmark page lists the versions it ran with. The Python and native READMEs now say what holds across threads: every call releases the GIL for its native work, a layout is immutable and shared, a failure's message and line number belong to the thread whose call failed, a reader is used by one thread at a time, and many calls at once are better off with degree_of_parallelism 1 each. --- CHANGELOG.md | 4 +-- README.md | 3 +- docs/python-benchmarks.md | 43 ++++++++++++++------------- python/README.md | 17 +++++++++++ src/FixedWidthParser.Native/README.md | 9 ++++++ 5 files changed, 52 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d8a9d2..6d6cabc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,8 +67,8 @@ Windows-1252 after registering `CodePagesEncodingProvider`). Reads fill a columnar `fw_table` or Arrow C Data Interface arrays and streams, in parallel by byte range, with `header_lines` and `skip_invalid`; writes take a columnar table and are byte-identical whatever the thread count. Not on NuGet. -- The `fixedwidth` Python package over it (NumPy views, pyarrow, pandas, polars), 52 times faster than - `pandas.read_fwf` on a 1,000,000-line COTAHIST-shaped file, and a header-only C++23 wrapper +- The `fixedwidth` Python package over it (NumPy views, pyarrow, pandas, polars): `read_pandas` is 96 times + faster than `pandas.read_fwf` on a 1,000,000-line COTAHIST-shaped file. And a header-only C++23 wrapper (`fixedwidth.hpp`) that also maps structs to columns through `FixedWidthMapper`. It installs as a CMake package (`find_package(fixedwidth)`) or through FetchContent, and downloads the release's native library for the platform, so it needs no .NET SDK. diff --git a/README.md b/README.md index 836b334..3fe3eaf 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,8 @@ layout = fw.Layout([ df = fw.read_pandas("COTAHIST_A2023.TXT", layout, fw.ReadOptions(header_lines=1, skip_invalid=True)) ``` -On a 1,000,000-line COTAHIST-shaped file, `fw.read_pandas` is 52 times faster than `pandas.read_fwf` +On a 1,000,000-line COTAHIST-shaped file, `fw.read_pandas` is 96 times faster than `pandas.read_fwf`, and +`fw.read_polars` 114 times ([measurements](docs/python-benchmarks.md)). The [Python README](python/README.md), the [C++ README](cpp/README.md) and the [native library README](src/FixedWidthParser.Native/README.md) cover the rest. diff --git a/docs/python-benchmarks.md b/docs/python-benchmarks.md index b8a019f..2d59320 100644 --- a/docs/python-benchmarks.md +++ b/docs/python-benchmarks.md @@ -15,22 +15,23 @@ could not reproduce a 16-digit one from a float64 either. ## Reading -1,000,000 lines (235 MiB). AMD Ryzen 7 5700X (8 cores, 16 threads), Windows 10, Python 3.14.7, pandas 3.1.0, -pyarrow 26.0.0, polars 2.0.0. `read_fwf` best of 1 run, the others best of 5. +1,000,000 lines (235 MiB). AMD Ryzen 7 5700X (8 cores, 16 threads), Windows 10, Python 3.14.7, pandas 3.0.5, +pyarrow 26.0.0, polars 1.44.1. `read_fwf` best of 1 run, the others best of 5. | Reader | Seconds | vs read_fwf | |---|---:|---:| -| pandas.read_fwf | 14.751 | 1.0x | -| polars str.slice (reference) | 0.349 | 42x | -| fw.read_pandas | 0.212 | 70x | -| fw.read_pandas, 1 thread | 0.655 | 23x | -| fw.read_polars | 0.198 | 74x | -| fw.read_arrow | 0.195 | 76x | -| fw.read (NumPy views) | 0.166 | 89x | -| fw.read, 1 thread | 0.600 | 25x | - -A single thread is already 25x faster than `read_fwf`; the default, one thread per processor, adds another 3.6x. -`fw.read_pandas` costs about 40 ms over `fw.read_arrow`, pyarrow's conversion to pandas. +| pandas.read_fwf | 15.353 | 1.0x | +| polars str.slice (reference) | 0.217 | 71x | +| fw.read_pandas | 0.160 | 96x | +| fw.read_pandas, 1 thread | 0.625 | 25x | +| fw.read_polars | 0.135 | 114x | +| fw.read_arrow | 0.123 | 125x | +| fw.read (NumPy views) | 0.125 | 123x | +| fw.read, 1 thread | 0.581 | 26x | + +A single thread is already 26x faster than `read_fwf`; the default, one thread per processor, adds another 4.6x. +`fw.read_pandas` costs about 40 ms over `fw.read_arrow`, pyarrow's conversion to pandas. The sections below trace +how the read got here; their before and after columns come from runs of their own. ## Writing @@ -48,17 +49,17 @@ Same machine and versions; `numpy.savetxt` best of 1 run, the others best of 5. | Writer | Seconds | vs savetxt | |---|---:|---:| -| numpy.savetxt (pandas) | 4.302 | 1.0x | -| polars pad + concat_str | 0.707 | 6.1x | -| fw.write (pandas) | 0.219 | 20x | +| numpy.savetxt (pandas) | 4.288 | 1.0x | +| polars pad + concat_str | 0.710 | 6.0x | +| fw.write (pandas) | 0.221 | 19x | | fw.write (polars) | 0.190 | 23x | -| fw.write (pyarrow) | 0.162 | 27x | -| fw.write (pyarrow), 1 thread | 0.550 | 7.8x | -| fw.write_bytes (pyarrow) | 0.342 | 13x | +| fw.write (pyarrow) | 0.148 | 29x | +| fw.write (pyarrow), 1 thread | 0.559 | 7.7x | +| fw.write_bytes (pyarrow) | 0.337 | 13x | -`fw.write` is 4.4x faster than the polars pipeline. A pandas frame costs ~60 ms more than a pyarrow table, its +`fw.write` is 4.8x faster than the polars pipeline. A pandas frame costs ~70 ms more than a pyarrow table, its conversion to Arrow; `write_bytes` timing includes copying the 235 MB into a `bytes` and writing it to the file. -Of `fw.write`'s 0.16 s, ~20 ms is the Python side handing the columns over and ~140 ms the native write. +Of `fw.write`'s 0.15 s, ~20 ms is the Python side handing the columns over and the rest the native write. ### Making the native write faster diff --git a/python/README.md b/python/README.md index c98233f..5c67833 100644 --- a/python/README.md +++ b/python/README.md @@ -64,6 +64,23 @@ A line that fails to parse raises `fw.ParseError` with `line_number` and `column when the column has no nulls) and `to_arrow()`, which shares the native buffers. Without pyarrow, `read_polars` reads sequentially through the Arrow stream. +## Threads and asyncio + +Every function releases the GIL for its native work, so reads and writes on several threads, or through +`asyncio.to_thread`, run in parallel: + +```python +frames = await asyncio.gather(*(asyncio.to_thread(fw.read_polars, path, layout, options) for path in paths)) +``` + +- A `fw.Layout` is immutable and can be shared by every thread at once. +- A failure is raised on the thread whose call failed, with that call's message and line number. +- A `read_batches` iterator, and the stream it reads, belong to one thread at a time. +- A stream's `read()` is called on the thread doing the read and takes the GIL while it runs; Python's files, + sockets and `zlib` release it again while they wait or decompress, so other threads keep going. +- `degree_of_parallelism=0` gives each call one thread per processor. When many calls already run at once, pass + `1`: the calls are the parallelism, and splitting each one too only makes them queue for the same cores. + ## Columns `fw.Column(name, start, length, type, *, nullable, scale, format, trim_char, trim_mode, alignment, padding, overflow)`. diff --git a/src/FixedWidthParser.Native/README.md b/src/FixedWidthParser.Native/README.md index 6860f58..994ef94 100644 --- a/src/FixedWidthParser.Native/README.md +++ b/src/FixedWidthParser.Native/README.md @@ -48,6 +48,15 @@ memory, since they split the source into byte ranges. `include/fixedwidth.def` lists them for import-library generation on Windows; a test fails when it drifts from the `[UnmanagedCallersOnly]` entry points. +## Threads + +Any thread may call any function, and calls on different tables, readers and streams run concurrently. A layout +is immutable and may be shared by every thread; handles live in a concurrent table. `fw_last_error` and +`fw_last_parse_error` belong to the calling thread, so read them on the thread whose call failed. A reader or an +Arrow stream is used by one thread at a time, and an `fw_stream`'s `read` runs on whichever thread is reading. +A read or write with `degree_of_parallelism` 0 splits across every processor; several such calls at once share +the same thread pool, so give each `1` when the calls themselves are the parallelism. + ## Consuming from C Check `fw_abi_version()` against `FW_ABI_VERSION` first. NativeAOT ships no import library on Windows: load From 48b567c561134ad6c6a401d962488d059f6b268e Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sun, 25 Oct 2026 15:08:57 -0300 Subject: [PATCH 81/85] test(native): compare random partitioned reads with the sequential read The partitioned read parses into slices of the final columns sized from the first line's length, then closes the gaps in place or copies a column out when a slice overflows. A CsCheck property drives every path through it with 2,000 random files: invalid, empty and short lines, trailing bytes and CRLF that change the line length, a byte order mark, header lines, multi-byte text, and Latin-1 text longer as UTF-8 than its column. Whatever the chunk size and thread count, the table, the skipped count and a failure's line and column must be the sequential read's. Breaking the in-place compaction or the copy out of an outgrown slice on purpose fails it. --- .../Native/NativeParallelPropertyTests.cs | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 tests/FixedWidthParser.Tests/Native/NativeParallelPropertyTests.cs diff --git a/tests/FixedWidthParser.Tests/Native/NativeParallelPropertyTests.cs b/tests/FixedWidthParser.Tests/Native/NativeParallelPropertyTests.cs new file mode 100644 index 0000000..fa981c7 --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeParallelPropertyTests.cs @@ -0,0 +1,109 @@ +using System.Text; +using CsCheck; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Layouts; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; +using static FixedWidthParser.Tests.Native.NativeTestReads; + +namespace FixedWidthParser.Tests.Native +{ + /// + /// A partitioned read parses into slices of the final columns sized from the first line's length + /// (TableArena), then closes the gaps in place or copies a column out when a slice overflows. Random files + /// drive every path through it: invalid, empty and short lines, trailing bytes and CRLF that change the line + /// length, a byte order mark and header lines, multi-byte text, and Latin-1 text whose UTF-8 is longer than the + /// column. Whatever the chunk size and thread count, the result must be the sequential read's. + /// + public sealed class NativeParallelPropertyTests + { + private const int Latin1 = 28591; + + private sealed record Case(string[] Lines, int HeaderLines, bool Bom, bool FinalNewline, bool Latin, bool SkipInvalid, int ChunkSize, int Degree) + { + public byte[] Bytes() + { + var text = new StringBuilder(); + for (int i = 0; i < HeaderLines; i++) + { + text.Append("HEADER ").Append(i).Append('\n'); + } + text.AppendJoin("", Lines); + if (!FinalNewline && text.Length > 0 && text[^1] == '\n') + { + text.Length--; + if (text.Length > 0 && text[^1] == '\r') + { + text.Length--; + } + } + byte[] body = (Latin ? Encoding.Latin1 : Encoding.UTF8).GetBytes(text.ToString()); + return Bom && !Latin ? [0xEF, 0xBB, 0xBF, .. body] : body; + } + } + + // code[0,4) text, qty[4,8) nullable integer, price[8,14) two implied decimals. + private static NativeLayout Layout() + { + return Build( + new Col("code", 0, 4, TString), + new Col("qty", 4, 4, TInt64) { Nullable = true }, + new Col("price", 8, 6, TFloat64) { Scale = 2 }); + } + + private static Gen GenLine(bool latin) + { + // Four bytes in either encoding; as UTF-8, the Latin-1 one is eight. + var code = Gen.OneOf(Gen.Char["ABCXYZ019"].Array[4].Select(c => new string(c)), Gen.Const(latin ? "ÉÉÉÉ" : "ÉÉ")); + var qty = Gen.OneOf(Gen.Int[0, 9999].Select(q => q.ToString("D4", System.Globalization.CultureInfo.InvariantCulture)), Gen.Const(" ")); + var price = Gen.Int[0, 999_999].Select(p => p.ToString("D6", System.Globalization.CultureInfo.InvariantCulture)); + var tail = Gen.Char[" #"].Array[0, 3].Select(c => new string(c)); + var newline = Gen.OneOf(Gen.Const("\n"), Gen.Const("\r\n")); + var valid = Gen.Select(code, qty, price, tail, newline, (c, q, p, t, n) => c + q + p + t + n); + return Gen.Frequency( + (12, valid), + (1, newline.Select(n => "AB01XXXX123456" + n)), + (1, newline.Select(n => "AB" + n)), + (1, newline)); + } + + private static readonly Gen GenCase = + from latin in Gen.Bool + from lines in GenLine(latin).Array[0, 60] + from header in Gen.Int[0, 2] + from bom in Gen.Bool + from finalNewline in Gen.Bool + from skipInvalid in Gen.Bool + from chunk in Gen.Int[1, 64] + from degree in Gen.Int[2, 8] + select new Case(lines, header, bom, finalNewline, latin, skipInvalid, chunk, degree); + + [Fact] + public void ParallelRead_IsTheSequentialRead() + { + var layout = Layout(); + GenCase.Sample(c => + { + byte[] bytes = c.Bytes(); + int encoding = c.Latin ? Latin1 : 0; + + var sequential = FromMemory(bytes, layout, Options(c.HeaderLines, c.SkipInvalid, degreeOfParallelism: 1, encoding: encoding)); + NativeApi.LastParseError(out long sequentialLine, out int sequentialColumn); + var parallel = FromMemory(bytes, layout, Options(c.HeaderLines, c.SkipInvalid, degreeOfParallelism: c.Degree, encoding: encoding), c.ChunkSize); + NativeApi.LastParseError(out long parallelLine, out int parallelColumn); + + Assert.Equal(sequential.Status, parallel.Status); + if (sequential.Status != NativeStatus.Ok) + { + Assert.Equal((sequentialLine, sequentialColumn), (parallelLine, parallelColumn)); + return; + } + Assert.Equal(sequential.RowCount, parallel.RowCount); + Assert.Equal(sequential.LinesSkipped, parallel.LinesSkipped); + for (int column = 0; column < 3; column++) + { + Assert.Equal(sequential.Column(column), parallel.Column(column)); + } + }, iter: 2000); + } + } +} From b504234c7cb4584e851346dda0476eca7445395e Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sun, 25 Oct 2026 15:09:31 -0300 Subject: [PATCH 82/85] perf(native): lend STRING offsets to the partitions too The last copy a partitioned read made was each STRING column's offsets. The arena now lends them as well: a slice's index 0 is the previous slice's last offset, so only the first slice writes the column's leading 0, a later one claims its index 0 without writing it, and Truncate knows row 0 starts at 0 instead of reading it. Publishing rebases the offsets in place, moving down in order as values do; a slice that outgrows its room copies out and the column is copied as before. The copy was ~32 MB on the COTAHIST benchmark: fw.read 0.125 -> 0.121 s, within noise. The property test crashes when Truncate reads index 0. --- CHANGELOG.md | 2 +- docs/python-benchmarks.md | 4 +- .../Reading/ColumnBuilder.cs | 49 +++++++++++++++---- .../Reading/TableArena.cs | 33 +++++++++++-- .../Reading/TableSink.cs | 6 ++- 5 files changed, 77 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d6cabc..a2a2520 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -153,7 +153,7 @@ Windows-1252 after registering `CodePagesEncodingProvider`). COTAHIST read is about 32% faster. - Native and Python whole-file reads parse every partition straight into the final columns, at the row its byte offset gives when all lines have the first line's length, instead of copying the partitions together - afterwards. Numeric, date and time values and STRING bytes skip the copy, and a parallel COTAHIST read is + afterwards. Numeric, date and time values and STRING bytes and offsets skip the copy, and a parallel COTAHIST read is another 17% faster. A file whose lines vary in length still reads correctly; its columns are copied as before. - Writing: right-aligned `int`/`long` columns without a format put their digits straight into the line, digit-only date and time formats reuse the reader's fast path, and `fw_write_file` opens its file for diff --git a/docs/python-benchmarks.md b/docs/python-benchmarks.md index 2d59320..6e96904 100644 --- a/docs/python-benchmarks.md +++ b/docs/python-benchmarks.md @@ -130,7 +130,9 @@ each partition its slice; numeric, date and time values go straight there, and s for the column's full width on every row. Publishing moves each partition down to close the gaps that skipped lines and trimmed text leave, in memory already touched, and hands the buffers over. A partition with more rows than its slice, as when line lengths vary, copies its column out, and that column is concatenated as -before. STRING offsets are still copied: each slice's leading 0 would fall on the previous slice's last offset. +before. STRING offsets are lent too: each slice's index 0 is the previous slice's last offset, so only the first +slice writes the column's leading 0, and publishing rebases the rest in place. That last copy was ~32 MB here and +saved only about 4 ms. Same machine, same run order, best of 9: diff --git a/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs index 50800a5..c0168a8 100644 --- a/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs +++ b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs @@ -36,16 +36,27 @@ internal sealed unsafe class ColumnBuilder : IDisposable /// The maximum number of text bytes expected for this column. /// How many rows to size the buffers for up front, so a partition does not grow them row by row. /// Room for values that a owns, or null. - internal ColumnBuilder(NativeColumn column, long maxTextBytes, long expectedRows, void* lent = null) + /// For a STRING column, room for its + 1 offsets, or null. + /// Whether starts the column: only the first slice writes its leading 0. + internal ColumnBuilder(NativeColumn column, long maxTextBytes, long expectedRows, void* lent = null, int* lentOffsets = null, bool firstSlice = true) { _column = column; _maxTextBytes = maxTextBytes; switch (column.Type) { case NativeType.String: - _ints.Reserve(expectedRows + 1); + Size(_ints, lentOffsets, expectedRows + 1); Size(_bytes, lent, Math.Min(expectedRows * column.Length, maxTextBytes)); - _ints.Add(0); + // A later slice's index 0 is the previous slice's last offset, which that slice writes: this one + // claims it without writing, and reads it nowhere (Truncate knows row 0 starts at 0). + if (lentOffsets is null || firstSlice) + { + _ints.Add(0); + } + else + { + _ints.Append(1); + } break; case NativeType.Float64: Size(_doubles, lent, expectedRows); @@ -121,7 +132,8 @@ internal void Truncate(int rows) return; } _ints.Truncate(rows + 1); - _bytes.Truncate(_ints[rows]); + // Index 0 of a lent slice belongs to the slice before; row 0 always starts at 0. + _bytes.Truncate(rows == 0 ? 0 : _ints[rows]); } internal void Publish(NativeColumnRaw* target) @@ -145,15 +157,18 @@ internal void Publish(NativeColumnRaw* target) /// of are moved down in it to close the gaps between them, and it becomes the column; /// otherwise, or once a partition outgrew its slice, they are copied into fresh memory. /// - internal static void Concat(ReadOnlySpan parts, NativeColumnRaw* target, GrowableNativeBuffer? lent = null) + /// The block a STRING column's offsets were lent from, closed up and handed over the same way. + internal static void Concat(ReadOnlySpan parts, NativeColumnRaw* target, GrowableNativeBuffer? lent = null, + GrowableNativeBuffer? lentOffsets = null) { long rows = 0; foreach (ref readonly var part in parts) { rows += part.Rows; lent = part.Lent ? lent : null; + lentOffsets = part._ints.Borrowed ? lentOffsets : null; } - if (parts.Length == 1 && lent is null) + if (parts.Length == 1 && lent is null && lentOffsets is null) { parts[0].Publish(target); return; @@ -167,7 +182,7 @@ internal static void Concat(ReadOnlySpan parts, NativeColumnRaw* target->DataLen = 0; if (column.Type == NativeType.String) { - ConcatStrings(parts, rows, target, lent); + ConcatStrings(parts, rows, target, lent, lentOffsets); return; } target->Values = ConcatValues(parts, lent); @@ -207,7 +222,8 @@ internal static void Concat(ReadOnlySpan parts, NativeColumnRaw* return lent is null ? values : (void*)lent.Detach(); } - private static void ConcatStrings(ReadOnlySpan parts, long rows, NativeColumnRaw* target, GrowableNativeBuffer? lent) + private static void ConcatStrings(ReadOnlySpan parts, long rows, NativeColumnRaw* target, GrowableNativeBuffer? lent, + GrowableNativeBuffer? lentOffsets) { long total = 0; foreach (ref readonly var part in parts) @@ -220,8 +236,17 @@ private static void ConcatStrings(ReadOnlySpan parts, long rows, $"Column '{parts[0]._column.Name}' holds more than 2 GiB of text; read in batches (fw_reader_open_*)."); } byte* data = lent is null ? (byte*)Marshal.AllocHGlobal((nint)total) : lent.Pointer; - // One partition's offsets already start at 0. - int* offsets = parts.Length == 1 ? null : (int*)Marshal.AllocHGlobal((nint)((rows + 1) * sizeof(int))); + // Lent offsets are rebased in place, moving down in order as values do; one partition of its own already + // starts at 0; several of their own are copied into fresh memory. + int* offsets = null; + if (lentOffsets is not null) + { + offsets = lentOffsets.Pointer; + } + else if (parts.Length > 1) + { + offsets = (int*)Marshal.AllocHGlobal((nint)((rows + 1) * sizeof(int))); + } if (offsets is not null) { offsets[0] = 0; @@ -240,6 +265,10 @@ private static void ConcatStrings(ReadOnlySpan parts, long rows, MoveTo(part._bytes.Span, data + baseOffset); baseOffset += part._bytes.Count; } + if (lentOffsets is not null) + { + offsets = (int*)lentOffsets.Detach(); + } target->Values = offsets is null ? (void*)parts[0]._ints.Detach() : offsets; target->Data = lent is null ? data : (byte*)lent.Detach(); target->DataLen = total; diff --git a/src/FixedWidthParser.Native/Reading/TableArena.cs b/src/FixedWidthParser.Native/Reading/TableArena.cs index 64be7ea..2cea9ed 100644 --- a/src/FixedWidthParser.Native/Reading/TableArena.cs +++ b/src/FixedWidthParser.Native/Reading/TableArena.cs @@ -9,10 +9,9 @@ namespace FixedWidthParser.Native.Reading /// and publishing has nothing to copy. Every line is taken to be as long as the first, so a partition's first row /// is its offset over that length. A partition with fewer rows leaves a gap that publishing closes in place; one /// with more copies its column out of the slice, and that column is copied as before. A STRING column is lent - /// room for its whole width on every row, for its bytes. + /// room for its whole width on every row, for its bytes, and its offsets: a slice's index 0 is the previous slice's + /// last offset, so only the first slice writes the column's leading 0. /// - // ponytail: STRING offsets are still copied once, rebased; a slice's leading 0 would land on the last offset of - // the slice before. Lending one extra offset per partition would cut that copy too. internal sealed unsafe class TableArena : IDisposable { private const int ProbeBytes = 64 * 1024; @@ -22,6 +21,7 @@ internal sealed unsafe class TableArena : IDisposable private readonly int _rows; private readonly int[] _sizes; private readonly GrowableNativeBuffer?[] _blocks; + private readonly GrowableNativeBuffer?[] _offsetBlocks; private TableArena(long dataStart, long stride, int rows, NativeLayout layout, TableLimits limits) { @@ -39,6 +39,16 @@ private TableArena(long dataStart, long stride, int rows, NativeLayout layout, T _blocks[i] = block; } } + _offsetBlocks = new GrowableNativeBuffer?[_sizes.Length]; + for (int i = 0; i < _offsetBlocks.Length; i++) + { + if (layout.Columns[i].Type == NativeType.String) + { + var offsets = new GrowableNativeBuffer(); + offsets.Reserve(rows + 1L); + _offsetBlocks[i] = offsets; + } + } } /// @@ -93,12 +103,29 @@ internal int RowAt(long offset) return _blocks[column]; } + /// Where a STRING column's offset at index goes, the start of the row's text; null for other columns. + internal int* LendOffsets(int column, int row) + { + return _offsetBlocks[column] is { } block ? block.Pointer + row : null; + } + + internal GrowableNativeBuffer? OffsetsBlock(int column) + { + return _offsetBlocks[column]; + } + + /// Where a STRING column's offset at index c.Start + c.Length); _failedColumn = -1; @@ -143,7 +145,7 @@ private static unsafe void ConcatInParallel(ColumnBuilder[][] chunks, NativeColu { parts[c] = chunks[c][i]; } - ColumnBuilder.Concat(parts, (NativeColumnRaw*)target + i, arena?.Block(i)); + ColumnBuilder.Concat(parts, (NativeColumnRaw*)target + i, arena?.Block(i), arena?.OffsetsBlock(i)); }); } catch (AggregateException ex) From 20a42bf73fbde39100fb0e3a8871841fc184e826 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sun, 25 Oct 2026 15:11:02 -0300 Subject: [PATCH 83/85] feat(native): dictionary-encode STRING columns for Arrow A column of few distinct values (a record type, a market code, a currency) costs far less as a category than as strings. fw_column_spec gains dictionary, allowed on STRING columns only: the Arrow reads and streams hand such a column over as dictionary, indices in the order values first appear, so pandas reads it as category and polars as Categorical. Each column is encoded once at export with an open-addressing table, the columns in parallel; a stream batch carries a dictionary of its own. fw_table and fw.read keep plain strings, and Table.to_arrow() encodes the same columns. Categorical input writes like strings. Python takes fw.Column(dictionary=True), C++ ColumnSpec::dictionary. On the synthetic COTAHIST file with all nine string columns encoded, a read costs 23-41 ms more and the pandas frame drops from 294 to 209 MiB; its values are mostly unique, so real low-cardinality columns save more. fw_column_spec grows from 72 to 80 bytes; the C smoke test pins it. --- CHANGELOG.md | 4 + cpp/README.md | 3 + cpp/tests/arrow.cpp | 11 ++ python/README.md | 8 +- python/src/fixedwidth/_native.py | 1 + python/src/fixedwidth/layout.py | 6 + python/src/fixedwidth/reader.py | 4 +- python/src/fixedwidth/table.py | 8 +- python/src/fixedwidth/types.py | 3 + python/tests/test_arrow.py | 44 ++++++ .../Arrow/ArrowApi.Stream.cs | 2 +- src/FixedWidthParser.Native/Arrow/ArrowApi.cs | 75 ++++++++++- .../Arrow/DictionaryEncoder.cs | 125 ++++++++++++++++++ .../Layouts/NativeColumnRules.cs | 5 + .../Layouts/NativeColumnSpec.cs | 1 + .../Layouts/NativeLayout.cs | 4 + .../include/fixedwidth.h | 2 + .../include/fixedwidth.hpp | 3 + tests/FixedWidthParser.NativeSmoke/smoke.c | 3 +- .../Native/NativeArrowTests.cs | 58 ++++++++ .../Native/NativeTestLayouts.cs | 2 + 21 files changed, 360 insertions(+), 12 deletions(-) create mode 100644 src/FixedWidthParser.Native/Arrow/DictionaryEncoder.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index a2a2520..248e9f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,10 @@ Windows-1252 after registering `CodePagesEncodingProvider`). (`fixedwidth.hpp`) that also maps structs to columns through `FixedWidthMapper`. It installs as a CMake package (`find_package(fixedwidth)`) or through FetchContent, and downloads the release's native library for the platform, so it needs no .NET SDK. +- Dictionary-encoded STRING columns for Arrow (`fw_column_spec.dictionary`, Python `fw.Column(dictionary=True)`, + C++ `ColumnSpec::dictionary`): int32 indices into the column's distinct values, which pandas reads as + `category` and polars as `Categorical`. `fw_table` and `fw.read` keep strings; categorical columns write like + strings. - Batched and Arrow-stream reads from a forward-only source, as ExcelReader's `xl_stream`: an `fw_stream` of `read` and `release` callbacks (`fw_reader_open_stream`, `fw_read_arrow_stream_stream`, and `fw_set_source_error` for a callback's failure). Header lines are skipped without seeking, and the batches diff --git a/cpp/README.md b/cpp/README.md index bf20fe1..0cd6486 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -65,6 +65,9 @@ the end. `fw::layout_for()` returns the `fw::Layout` the bindings describe, f read it forward a batch at a time, in constant memory, so a decompressing or network stream works. The stream must outlive the reader; a stream that turns `bad()` fails the read. +`ColumnSpec::dictionary` hands a String column to Arrow (`read_arrow_*`) dictionary-encoded: int32 indices into +its distinct values. `fw::Table` keeps the strings. + ```cpp std::ifstream file("COTAHIST_A2023.TXT", std::ios::binary); auto reader = fw::TypedReader::open_stream(file, {.read = {.header_lines = 1, .skip_invalid = true}}); diff --git a/cpp/tests/arrow.cpp b/cpp/tests/arrow.cpp index 4c41c9d..2627256 100644 --- a/cpp/tests/arrow.cpp +++ b/cpp/tests/arrow.cpp @@ -42,6 +42,17 @@ int main() } CHECK(streamed_rows == 3); + // A dictionary column: int32 indices whose dictionary holds the distinct strings. + std::vector coded{{.name = "codneg", .start = 2, .length = 12, .type = fw::Type::String, .dictionary = true}}; + auto coded_layout = fw::Layout::create(coded); + CHECK(coded_layout.has_value()); + auto encoded = fw::read_arrow_memory(content, *coded_layout, skip_header_and_invalid()); + CHECK(encoded.has_value()); + const ArrowSchema& codes_schema = *encoded->schema().children[0]; + const ArrowArray& codes = *encoded->array().children[0]; + CHECK(std::strcmp(codes_schema.format, "i") == 0 && std::strcmp(codes_schema.dictionary->format, "u") == 0); + CHECK(codes.length == 4 && codes.dictionary->length == 4); + std::puts("arrow ok"); return 0; } diff --git a/python/README.md b/python/README.md index 5c67833..3ba9379 100644 --- a/python/README.md +++ b/python/README.md @@ -83,12 +83,18 @@ frames = await asyncio.gather(*(asyncio.to_thread(fw.read_polars, path, layout, ## Columns -`fw.Column(name, start, length, type, *, nullable, scale, format, trim_char, trim_mode, alignment, padding, overflow)`. +`fw.Column(name, start, length, type, *, nullable, scale, format, trim_char, trim_mode, alignment, padding, overflow, dictionary)`. Positions are bytes. Types: `fw.String`, `fw.Int64`, `fw.Float64`, `fw.Decimal` (int64 unscaled, Arrow `decimal128(18, scale)`), `fw.Date`, `fw.Time`, `fw.Timestamp`. `scale` is the number of implied decimals: `0000000123456` with `scale=2` reads as `1234.56`. `format` uses .NET date syntax (`yyyyMMdd`, `HHmmss`). `trim_mode`, `alignment` and `overflow` take `fw.TrimMode`, `fw.Alignment` and `fw.Overflow`. +`dictionary=True` on a `fw.String` column hands it to Arrow dictionary-encoded, so `read_pandas` gives a +`category` column and `read_polars` a `Categorical` one: for a column of few distinct values (a market code, a +currency, a record type) far less memory, and faster group-bys. The encoding costs a pass over the column, about +3 ms per million rows per column, run in parallel across columns. `fw.read`'s Table keeps plain strings, and its +`to_arrow()` encodes the same columns. Categorical columns write like strings. + `fw.Layout(columns, fw.LayoutOptions(line_length=..., filler=...))` sets how written lines are laid out: `line_length` past the last column pads each line, and `filler` (default a space) fills the gaps between columns. diff --git a/python/src/fixedwidth/_native.py b/python/src/fixedwidth/_native.py index 947865a..61f7947 100644 --- a/python/src/fixedwidth/_native.py +++ b/python/src/fixedwidth/_native.py @@ -45,6 +45,7 @@ class ColumnSpec(ctypes.Structure): ("alignment", ctypes.c_int32), ("padding", ctypes.c_int32), ("overflow", ctypes.c_int32), + ("dictionary", ctypes.c_int32), ] diff --git a/python/src/fixedwidth/layout.py b/python/src/fixedwidth/layout.py index 024ad72..535c3dd 100644 --- a/python/src/fixedwidth/layout.py +++ b/python/src/fixedwidth/layout.py @@ -71,6 +71,7 @@ def _fill(spec: _native.ColumnSpec, column: Column) -> tuple[object, ...]: spec.alignment = int(column.alignment or 0) spec.padding = ascii_option(column.padding, "padding") spec.overflow = int(column.overflow or 0) + spec.dictionary = int(column.dictionary) return name, fmt @property @@ -81,6 +82,11 @@ def columns(self) -> list[Column]: def names(self) -> list[str]: return [column.name for column in self._columns] + @property + def dictionaries(self) -> frozenset[str]: + """The columns set to reach Arrow dictionary-encoded.""" + return frozenset(column.name for column in self._columns if column.dictionary) + @property def handle(self) -> int: return self._handle or 0 diff --git a/python/src/fixedwidth/reader.py b/python/src/fixedwidth/reader.py index 71e7f9f..90a1695 100644 --- a/python/src/fixedwidth/reader.py +++ b/python/src/fixedwidth/reader.py @@ -145,7 +145,7 @@ def read(source: Source, layout: Layout, options: ReadOptions | None = None) -> else: status = lib.fw_read_memory(data.address, data.length, layout.handle, _ref(raw), ctypes.byref(table), ctypes.byref(result)) check(status) - return Table(_Owner(table), layout.names, int(result.lines_read), int(result.lines_skipped)) + return Table(_Owner(table), layout.names, int(result.lines_read), int(result.lines_skipped), layout.dictionaries) def read_arrow(source: Source, layout: Layout, options: ReadOptions | None = None) -> Any: @@ -202,7 +202,7 @@ def read_batches(source: Source, layout: Layout, options: ReaderOptions | None = if status == _native.FW_EOF: return check(status) - yield Table(_Owner(table), layout.names) + yield Table(_Owner(table), layout.names, dictionaries=layout.dictionaries) finally: lib.fw_reader_close(reader) del data diff --git a/python/src/fixedwidth/table.py b/python/src/fixedwidth/table.py index 899a2b3..fc36c65 100644 --- a/python/src/fixedwidth/table.py +++ b/python/src/fixedwidth/table.py @@ -44,9 +44,12 @@ class Table: list of `str` for strings. Null rows are marked by `validity(name)`. Views stay valid after the Table is gone; the native memory is freed when the last one is.""" - def __init__(self, owner: _Owner, names: list[str], lines_read: int = 0, lines_skipped: int = 0) -> None: + def __init__( + self, owner: _Owner, names: list[str], lines_read: int = 0, lines_skipped: int = 0, dictionaries: frozenset[str] = frozenset() + ) -> None: self._owner = owner self._names = names + self._dictionaries = dictionaries self.lines_read = lines_read self.lines_skipped = lines_skipped @@ -110,7 +113,8 @@ def _arrow_column(self, name: str, pa: Any) -> Any: if kind is ColumnType.STRING: offsets = pa.foreign_buffer(column.values, (n + 1) * 4, base=self._owner) data = pa.foreign_buffer(column.data, column.data_len, base=self._owner) if column.data else pa.py_buffer(b"") - return pa.Array.from_buffers(pa.string(), n, [validity, offsets, data]) + strings = pa.Array.from_buffers(pa.string(), n, [validity, offsets, data]) + return strings.dictionary_encode() if name in self._dictionaries else strings arrow_type = { ColumnType.INT64: pa.int64(), ColumnType.FLOAT64: pa.float64(), diff --git a/python/src/fixedwidth/types.py b/python/src/fixedwidth/types.py index 98fa0ba..bd4c42a 100644 --- a/python/src/fixedwidth/types.py +++ b/python/src/fixedwidth/types.py @@ -46,6 +46,8 @@ class Column: - `format`: the exact date/time format (.NET syntax, e.g. `yyyyMMdd`), or a number format on write. - `trim_char`, `trim_mode`: what is stripped before parsing; a null is written as `trim_char`. - `alignment`, `padding`, `overflow`: how a value is written. `padding='0'` keeps a sign leftmost. + - `dictionary`: a String column reaches Arrow dictionary-encoded, so pandas reads it as `category` and polars + as `Categorical`: much less memory for a column of few distinct values. `fw.read`'s Table keeps strings. """ name: str @@ -61,6 +63,7 @@ class Column: alignment: Alignment | None = None padding: str | None = None overflow: Overflow | None = None + dictionary: bool = False @dataclass(frozen=True) diff --git a/python/tests/test_arrow.py b/python/tests/test_arrow.py index 7f18545..9e711e4 100644 --- a/python/tests/test_arrow.py +++ b/python/tests/test_arrow.py @@ -58,3 +58,47 @@ def test_table_to_arrow(cotahist, options): table = fw.read(cotahist, layout(), options) assert table.to_arrow().column("preabe").to_pylist() == [1234.56, 678.9, 43.21] + + +def dictionary_layout() -> fw.Layout: + return fw.Layout( + [ + fw.Column(c.name, c.start, c.length, c.type, scale=c.scale, format=c.format, nullable=c.nullable, + padding=c.padding, dictionary=c.type == fw.String) + for c in layout().columns + ] + ) + + +def test_dictionary_columns_reach_pandas_and_polars_as_categories(cotahist, options): + table = fw.read_arrow(cotahist, dictionary_layout(), options) + frame = fw.read_pandas(cotahist, dictionary_layout(), options) + polars_frame = fw.read_polars(cotahist, dictionary_layout(), options) + + assert pa.types.is_dictionary(table.schema.field("codneg").type) + assert table.column("codneg").to_pylist() == ["PETR4", "VALE3", "ITUB4"] + assert str(frame["codneg"].dtype) == "category" + assert list(frame["codneg"]) == ["PETR4", "VALE3", "ITUB4"] + assert polars_frame.schema["codneg"] == pl.Categorical + assert table.column("preabe").to_pylist() == fw.read_arrow(cotahist, layout(), options).column("preabe").to_pylist() + + +def test_dictionary_columns_in_to_arrow_and_streams(cotahist, options): + converted = fw.read(cotahist, dictionary_layout(), options).to_arrow() + streamed = pa.RecordBatchReader.from_stream( + fw.read_arrow_stream(cotahist, dictionary_layout(), fw.ReaderOptions(read=options, batch_rows=2)) + ).read_all() + + assert pa.types.is_dictionary(converted.schema.field("codneg").type) + assert converted.column("codneg").to_pylist() == streamed.column("codneg").to_pylist() == ["PETR4", "VALE3", "ITUB4"] + + +def test_categorical_columns_write_like_strings(cotahist, options): + frame = fw.read_pandas(cotahist, dictionary_layout(), options) + + assert fw.write_bytes(frame, layout()) == fw.write_bytes(fw.read_pandas(cotahist, layout(), options), layout()) + + +def test_dictionary_only_on_string_columns(): + with pytest.raises(fw.FixedWidthError, match="dictionary"): + fw.Layout([fw.Column("n", 0, 4, fw.Int64, dictionary=True)]) diff --git a/src/FixedWidthParser.Native/Arrow/ArrowApi.Stream.cs b/src/FixedWidthParser.Native/Arrow/ArrowApi.Stream.cs index 63dc8c8..1d646a3 100644 --- a/src/FixedWidthParser.Native/Arrow/ArrowApi.Stream.cs +++ b/src/FixedWidthParser.Native/Arrow/ArrowApi.Stream.cs @@ -117,7 +117,7 @@ internal static int StreamGetNext(ArrowArrayStream* stream, ArrowArray* outArray } try { - *outArray = BuildArray(&table); + *outArray = BuildArray(&table, state.Reader.Layout); return 0; } catch (Exception ex) diff --git a/src/FixedWidthParser.Native/Arrow/ArrowApi.cs b/src/FixedWidthParser.Native/Arrow/ArrowApi.cs index 902123c..6ba674b 100644 --- a/src/FixedWidthParser.Native/Arrow/ArrowApi.cs +++ b/src/FixedWidthParser.Native/Arrow/ArrowApi.cs @@ -41,7 +41,7 @@ private static int Export(int status, NativeLayout layout, NativeTableRaw* table try { *outSchema = BuildSchema(layout); - *outArray = BuildArray(table); + *outArray = BuildArray(table, layout); return NativeStatus.Ok; } catch (Exception ex) @@ -67,6 +67,7 @@ internal static void ReleaseArrowSchema(IntPtr schemaPtr) return; } ReleaseChildren(schema->Children, schema->NChildren, &ReleaseArrowSchema); + ReleaseDictionary(schema->Dictionary, &ReleaseArrowSchema); FreeIfSet(schema->Format); FreeIfSet(schema->Name); schema->Release = IntPtr.Zero; @@ -80,6 +81,7 @@ internal static void ReleaseArrowArray(IntPtr arrayPtr) return; } ReleaseChildren(array->Children, array->NChildren, &ReleaseArrowArray); + ReleaseDictionary(array->Dictionary, &ReleaseArrowArray); var buffers = (IntPtr*)array->Buffers; for (long i = 0; i < array->NBuffers; i++) { @@ -100,6 +102,15 @@ private static void ReleaseChildren(IntPtr children, long count, delegate* release) + { + if (dictionary != IntPtr.Zero) + { + release(dictionary); + Marshal.FreeHGlobal(dictionary); + } + } + private static void FreeIfSet(IntPtr pointer) { if (pointer != IntPtr.Zero) @@ -129,6 +140,7 @@ internal static ArrowSchema BuildSchema(NativeLayout layout) Format = AllocUtf8Z(FormatCode(column)), Name = AllocUtf8Z(column.Name), Flags = column.Nullable ? ArrowFlags.Nullable : 0, + Dictionary = column.Dictionary ? DictionaryValuesSchema() : IntPtr.Zero, Release = SchemaReleaseCallback, }; children[schema.NChildren++] = (IntPtr)child; @@ -142,8 +154,15 @@ internal static ArrowSchema BuildSchema(NativeLayout layout) } } + private static IntPtr DictionaryValuesSchema() + { + var values = (ArrowSchema*)Marshal.AllocHGlobal(sizeof(ArrowSchema)); + *values = new ArrowSchema { Format = AllocUtf8Z("u"), Release = SchemaReleaseCallback }; + return (IntPtr)values; + } + /// Moves every column of into a struct array, leaving the table's buffers empty. - internal static ArrowArray BuildArray(NativeTableRaw* table) + internal static ArrowArray BuildArray(NativeTableRaw* table, NativeLayout layout) { var children = (IntPtr*)Marshal.AllocHGlobal(Math.Max(1, table->ColumnCount) * IntPtr.Size); var buffers = (IntPtr*)Marshal.AllocHGlobal(IntPtr.Size); @@ -163,10 +182,20 @@ internal static ArrowArray BuildArray(NativeTableRaw* table) var child = (ArrowArray*)Marshal.AllocHGlobal(sizeof(ArrowArray)); *child = default; children[array.NChildren++] = (IntPtr)child; - *child = BuildChild(table->Columns + i); } + // Encoding a dictionary column hashes every row, so the columns are built side by side. + nint columns = (nint)table->Columns; + nint built = (nint)children; + Parallel.For(0, table->ColumnCount, i => + *((ArrowArray**)built)[i] = BuildChild((NativeColumnRaw*)columns + i, layout.Columns[i].Dictionary)); return array; } + catch (AggregateException ex) + { + ReleaseArrowArray((IntPtr)(&array)); + System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw(ex.InnerExceptions[0]); + throw; + } catch { ReleaseArrowArray((IntPtr)(&array)); @@ -174,8 +203,12 @@ internal static ArrowArray BuildArray(NativeTableRaw* table) } } - private static ArrowArray BuildChild(NativeColumnRaw* column) + private static ArrowArray BuildChild(NativeColumnRaw* column, bool dictionary) { + if (dictionary) + { + return BuildDictionaryChild(column); + } bool isString = column->Type == NativeType.String; int bufferCount = isString ? 3 : 2; var buffers = (IntPtr*)Marshal.AllocHGlobal(bufferCount * IntPtr.Size); @@ -197,6 +230,37 @@ private static ArrowArray BuildChild(NativeColumnRaw* column) }; } + private static ArrowArray BuildDictionaryChild(NativeColumnRaw* column) + { + long rows = column->Length; + int* indices = DictionaryEncoder.Encode((int*)column->Values, column->Data, rows, + out int* valueOffsets, out byte* valueBytes, out long valueCount, out _); + FreeIfSet((IntPtr)column->Values); + FreeIfSet((IntPtr)column->Data); + column->Values = null; + column->Data = null; + + var valueBuffers = (IntPtr*)Marshal.AllocHGlobal(3 * IntPtr.Size); + valueBuffers[0] = IntPtr.Zero; + valueBuffers[1] = (IntPtr)valueOffsets; + valueBuffers[2] = (IntPtr)valueBytes; + var values = (ArrowArray*)Marshal.AllocHGlobal(sizeof(ArrowArray)); + *values = new ArrowArray { Length = valueCount, NBuffers = 3, Buffers = (IntPtr)valueBuffers, Release = ArrayReleaseCallback }; + + var buffers = (IntPtr*)Marshal.AllocHGlobal(2 * IntPtr.Size); + buffers[0] = Take(ref column->Validity); + buffers[1] = (IntPtr)indices; + return new ArrowArray + { + Length = rows, + NullCount = CountUnset(buffers[0], rows), + NBuffers = 2, + Buffers = (IntPtr)buffers, + Dictionary = (IntPtr)values, + Release = ArrayReleaseCallback, + }; + } + private static IntPtr Take(ref byte* pointer) { IntPtr taken = (IntPtr)pointer; @@ -253,7 +317,8 @@ private static string FormatCode(NativeColumn column) { return column.Type switch { - NativeType.String => "u", + // A dictionary column's own format is its index type; the values' utf8 is in its dictionary schema. + NativeType.String => column.Dictionary ? "i" : "u", NativeType.Int64 => "l", NativeType.Float64 => "g", NativeType.Decimal => string.Create(CultureInfo.InvariantCulture, $"d:18,{column.Scale}"), diff --git a/src/FixedWidthParser.Native/Arrow/DictionaryEncoder.cs b/src/FixedWidthParser.Native/Arrow/DictionaryEncoder.cs new file mode 100644 index 0000000..08e01f5 --- /dev/null +++ b/src/FixedWidthParser.Native/Arrow/DictionaryEncoder.cs @@ -0,0 +1,125 @@ +using System.Runtime.InteropServices; + +namespace FixedWidthParser.Native.Arrow +{ + /// + /// Dictionary-encodes one STRING column for Arrow: an int32 index per row into the column's distinct values, in + /// the order they first appear. A null row keeps its index to the empty string; its validity bit says it is null. + /// + // ponytail: encodes the finished column at export, one thread per column: ~30 ms on COTAHIST's 1M rows with + // nine dictionary columns, bound by the most varied one. Encoding each partition while it parses, then + // remapping the partitions' indices at publish, would spread that over every thread. + internal sealed unsafe class DictionaryEncoder : IDisposable + { + private const int MinSlots = 1024; + + private readonly GrowableNativeBuffer _offsets = new(); + private readonly GrowableNativeBuffer _bytes = new(); + private int[] _slots = new int[MinSlots]; // index + 1; 0 is an empty slot + private int[] _hashes = new int[MinSlots]; + private int _count; + + private DictionaryEncoder() + { + _offsets.Add(0); + } + + /// + /// The indices of a column's values, given as into + /// , and its distinct values as Arrow utf8 offsets and bytes. Every buffer is + /// allocated with ; the caller frees them. + /// + internal static int* Encode(int* offsets, byte* data, long rows, out int* valueOffsets, out byte* valueBytes, out long valueCount, out long valueBytesLength) + { + using var encoder = new DictionaryEncoder(); + int* indices = (int*)Marshal.AllocHGlobal((nint)(Math.Max(1, rows) * sizeof(int))); + try + { + for (long row = 0; row < rows; row++) + { + indices[row] = encoder.IndexOf(new ReadOnlySpan(data + offsets[row], offsets[row + 1] - offsets[row])); + } + } + catch + { + Marshal.FreeHGlobal((IntPtr)indices); + throw; + } + valueCount = encoder._count; + valueBytesLength = encoder._bytes.Count; + valueOffsets = (int*)encoder._offsets.Detach(); + valueBytes = valueBytesLength == 0 ? (byte*)Marshal.AllocHGlobal(1) : (byte*)encoder._bytes.Detach(); + return indices; + } + + private int IndexOf(ReadOnlySpan value) + { + var hasher = default(HashCode); + hasher.AddBytes(value); + int hash = hasher.ToHashCode(); + int mask = _slots.Length - 1; + int slot = hash & mask; + // The table is at most half full, so the probe always reaches an empty slot. + while (_slots[slot] != 0) + { + int entry = _slots[slot]; + if (_hashes[slot] == hash && Value(entry - 1).SequenceEqual(value)) + { + return entry - 1; + } + slot = (slot + 1) & mask; + } + return Add(value, hash, slot); + } + + private ReadOnlySpan Value(int index) + { + var offsets = _offsets.Span; + return _bytes.Span[offsets[index]..offsets[index + 1]]; + } + + private int Add(ReadOnlySpan value, int hash, int slot) + { + int index = _count++; + _bytes.AddRange(value); + _offsets.Add(_bytes.Count); + _slots[slot] = index + 1; + _hashes[slot] = hash; + // Half full at most, so a probe ends quickly at an empty slot. + if (_count * 2 > _slots.Length) + { + Rehash(); + } + return index; + } + + private void Rehash() + { + int[] slots = new int[_slots.Length * 2]; + int[] hashes = new int[slots.Length]; + int mask = slots.Length - 1; + for (int old = 0; old < _slots.Length; old++) + { + if (_slots[old] == 0) + { + continue; + } + int slot = _hashes[old] & mask; + while (slots[slot] != 0) + { + slot = (slot + 1) & mask; + } + slots[slot] = _slots[old]; + hashes[slot] = _hashes[old]; + } + _slots = slots; + _hashes = hashes; + } + + public void Dispose() + { + _offsets.Dispose(); + _bytes.Dispose(); + } + } +} diff --git a/src/FixedWidthParser.Native/Layouts/NativeColumnRules.cs b/src/FixedWidthParser.Native/Layouts/NativeColumnRules.cs index edd34af..e52ccd3 100644 --- a/src/FixedWidthParser.Native/Layouts/NativeColumnRules.cs +++ b/src/FixedWidthParser.Native/Layouts/NativeColumnRules.cs @@ -40,6 +40,10 @@ internal static class NativeColumnRules { return "truncate overflow applies only to STRING columns."; } + if (spec.Dictionary is < 0 or > 1 || (spec.Dictionary == 1 && spec.Type != NativeType.String)) + { + return "dictionary is 0 or 1, and applies only to STRING columns."; + } return null; } @@ -54,6 +58,7 @@ internal static NativeColumn Build(NativeColumnSpecRaw spec, string name, string Length = spec.Length, Type = spec.Type, Nullable = spec.Nullable != 0, + Dictionary = spec.Dictionary == 1, Scale = spec.Scale, Parse = new ColumnParseOptions(trimChar, TrimModeOf(spec.TrimMode), format), FormatOptions = new ColumnFormatOptions(AlignmentOf(spec.Alignment, spec.Type), padding, format ?? IsoFormat(spec.Type), diff --git a/src/FixedWidthParser.Native/Layouts/NativeColumnSpec.cs b/src/FixedWidthParser.Native/Layouts/NativeColumnSpec.cs index fa428c1..27e0d01 100644 --- a/src/FixedWidthParser.Native/Layouts/NativeColumnSpec.cs +++ b/src/FixedWidthParser.Native/Layouts/NativeColumnSpec.cs @@ -41,6 +41,7 @@ internal unsafe struct NativeColumnSpecRaw public int Alignment; public int Padding; public int Overflow; + public int Dictionary; } /// fw_layout_options. diff --git a/src/FixedWidthParser.Native/Layouts/NativeLayout.cs b/src/FixedWidthParser.Native/Layouts/NativeLayout.cs index b78a043..f8cba12 100644 --- a/src/FixedWidthParser.Native/Layouts/NativeLayout.cs +++ b/src/FixedWidthParser.Native/Layouts/NativeLayout.cs @@ -12,6 +12,10 @@ internal sealed class NativeColumn internal int Length { get; init; } internal int Type { get; init; } internal bool Nullable { get; init; } + + /// A STRING column handed to Arrow dictionary-encoded: int32 indices into its distinct values. + internal bool Dictionary { get; init; } + internal int Scale { get; init; } internal ColumnParseOptions Parse { get; init; } internal ColumnFormatOptions FormatOptions { get; init; } diff --git a/src/FixedWidthParser.Native/include/fixedwidth.h b/src/FixedWidthParser.Native/include/fixedwidth.h index fd7165b..b43a8d8 100644 --- a/src/FixedWidthParser.Native/include/fixedwidth.h +++ b/src/FixedWidthParser.Native/include/fixedwidth.h @@ -96,6 +96,8 @@ typedef struct fw_column_spec { int32_t alignment; /* FW_ALIGN_*, on write */ int32_t padding; /* ASCII; 0 = space, on write. '0' keeps a sign leftmost. */ int32_t overflow; /* FW_OVERFLOW_*, on write */ + int32_t dictionary; /* 1: a STRING column reaches Arrow dictionary-encoded, as int32 indices into its + * distinct values (pandas category, polars Categorical); fw_table keeps strings */ } fw_column_spec; typedef struct fw_layout_options { diff --git a/src/FixedWidthParser.Native/include/fixedwidth.hpp b/src/FixedWidthParser.Native/include/fixedwidth.hpp index b8a2c02..750b7da 100644 --- a/src/FixedWidthParser.Native/include/fixedwidth.hpp +++ b/src/FixedWidthParser.Native/include/fixedwidth.hpp @@ -136,6 +136,8 @@ namespace fw Align alignment = Align::Default; char padding = 0; Overflow overflow = Overflow::Throw; + /// A String column reaches Arrow dictionary-encoded (int32 indices into its distinct values); fw::Table keeps strings. + bool dictionary = false; }; struct LayoutOptions @@ -170,6 +172,7 @@ namespace fw .alignment = static_cast(column.alignment), .padding = static_cast(column.padding), .overflow = static_cast(column.overflow), + .dictionary = column.dictionary ? 1 : 0, }); } const fw_layout_options raw{.struct_size = sizeof(fw_layout_options), .line_length = options.line_length, diff --git a/tests/FixedWidthParser.NativeSmoke/smoke.c b/tests/FixedWidthParser.NativeSmoke/smoke.c index 94d680b..4a38799 100644 --- a/tests/FixedWidthParser.NativeSmoke/smoke.c +++ b/tests/FixedWidthParser.NativeSmoke/smoke.c @@ -37,7 +37,8 @@ FW_STATIC_ASSERT(offsetof(fw_column_spec, start) == 20, spec_start); FW_STATIC_ASSERT(offsetof(fw_column_spec, scale) == 36, spec_scale); FW_STATIC_ASSERT(offsetof(fw_column_spec, format) == 40, spec_format); FW_STATIC_ASSERT(offsetof(fw_column_spec, overflow) == 68, spec_overflow); -FW_STATIC_ASSERT(sizeof(fw_column_spec) == 72, spec_size); +FW_STATIC_ASSERT(offsetof(fw_column_spec, dictionary) == 72, spec_dictionary); +FW_STATIC_ASSERT(sizeof(fw_column_spec) == 80, spec_size); FW_STATIC_ASSERT(sizeof(fw_layout_options) == 12, layout_options_size); FW_STATIC_ASSERT(offsetof(fw_read_options, header_lines) == 8, read_header_lines); FW_STATIC_ASSERT(offsetof(fw_read_options, skip_byte_order_mark) == 32, read_bom); diff --git a/tests/FixedWidthParser.Tests/Native/NativeArrowTests.cs b/tests/FixedWidthParser.Tests/Native/NativeArrowTests.cs index 90a951e..46bc56b 100644 --- a/tests/FixedWidthParser.Tests/Native/NativeArrowTests.cs +++ b/tests/FixedWidthParser.Tests/Native/NativeArrowTests.cs @@ -63,6 +63,64 @@ private static unsafe RecordBatch ReadBatch(string content, NativeLayout layout, }; } + [Theory] + [InlineData(1)] + [InlineData(4)] + public void Arrow_DictionaryColumn_DecodesToTheSameStrings(int degree) + { + string[] codes = ["PETR4", "VALE3", "ITUB4", "", "B3SA3"]; + string content = string.Concat(Enumerable.Range(0, 5000).Select(i => $"{codes[i * 7 % codes.Length],-6}{i % 97:D5}\n")); + var options = Options(degreeOfParallelism: degree); + using var plain = ReadBatch(content, Build(new Col("code", 0, 6, TString) { Nullable = true }, new Col("n", 6, 5, TInt64)), options); + using var encoded = ReadBatch(content, Build(new Col("code", 0, 6, TString) { Nullable = true, Dictionary = 1 }, new Col("n", 6, 5, TInt64)), options); + + var dictionary = Assert.IsType(encoded.Column(0)); + var values = (StringArray)dictionary.Dictionary; + var indices = (Int32Array)dictionary.Indices; + // First-seen order: i * 7 % 5 visits 0, 2, 4, 1, 3. A null row's index points at the empty string. + Assert.Equal(["PETR4", "ITUB4", "B3SA3", "VALE3", ""], Enumerable.Range(0, values.Length).Select(i => values.GetString(i))); + var decoded = Enumerable.Range(0, indices.Length).Select(i => dictionary.IsNull(i) ? null : values.GetString(indices.GetValue(i)!.Value)); + Assert.Equal(Values(plain.Column(0)), decoded); + Assert.Equal(1000, dictionary.NullCount); + Assert.Equal(Values(plain.Column(1)), Values(encoded.Column(1))); + } + + [Fact] + public void Dictionary_OnANonStringColumn_IsRejected() + { + Assert.Equal(NativeStatus.InvalidArgument, Create(out _, new Col("n", 0, 5, TInt64) { Dictionary = 1 })); + Assert.Contains("dictionary", LastError(), StringComparison.Ordinal); + } + + [Fact] + public async Task ArrowStream_DictionaryColumn_HasADictionaryPerBatch() + { + var layout = Build(new Col("code", 0, 6, TString) { Dictionary = 1 }); + byte[] bytes = Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat("AAAAAA\nBBBBBB\nAAAAAA\n", 4))); + + var decoded = new List(); + IArrowArrayStream stream = OpenStream(bytes, layout, new NativeReaderOptionsRaw { StructSize = ReaderOptionsSize, BatchRows = 5 }); + try + { + while (await stream.ReadNextRecordBatchAsync() is { } batch) + { + using (batch) + { + var dictionary = (DictionaryArray)batch.Column(0); + var values = (StringArray)dictionary.Dictionary; + var indices = (Int32Array)dictionary.Indices; + decoded.AddRange(Enumerable.Range(0, indices.Length).Select(i => values.GetString(indices.GetValue(i)!.Value))); + } + } + } + finally + { + stream.Dispose(); + } + + Assert.Equal(Enumerable.Repeat(new[] { "AAAAAA", "BBBBBB", "AAAAAA" }, 4).SelectMany(x => x), decoded); + } + [Fact] public void Arrow_EveryType_MatchesColumnarRead() { diff --git a/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs b/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs index f0e6697..53b0761 100644 --- a/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs +++ b/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs @@ -16,6 +16,7 @@ internal sealed record Col(string Name, int Start, int Length, int Type) public int Alignment { get; init; } public int Padding { get; init; } public int Overflow { get; init; } + public int Dictionary { get; init; } public int StructSize { get; init; } = Marshal.SizeOf(); } @@ -100,6 +101,7 @@ private static NativeColumnSpecRaw ToRaw(Col column, List allocations) Alignment = column.Alignment, Padding = column.Padding, Overflow = column.Overflow, + Dictionary = column.Dictionary, }; } From 70948ae4dd542638a9581b730bdf507d8278ef3c Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Sun, 25 Oct 2026 15:11:42 -0300 Subject: [PATCH 84/85] feat(native): write to a sink callback as well as to a file or memory fw_write_stream pushes the lines to an fw_sink, a write callback that takes every byte it is given and returns 0, or -1 after fw_set_source_error. A window of one batch per thread formats in parallel, then goes to the sink in order from the calling thread, so memory stays at a window of batches whatever the table's size. Python's fw.write takes a binary stream as well as a path: an open file, gzip.open(..., "wb"), a socket's makefile. The standard streams get the native buffer itself; anything else gets a copy it may keep. Writing to an open file takes 0.150 s on the 1M-line benchmark against 0.146 s for its path. A Python exception in write() reaches the caller with its message. The C++ wrapper has write_stream(std::ostream&, ...). The C smoke test now also pins fw_stream and fw_sink and checks every stream export resolves. --- CHANGELOG.md | 3 + cpp/README.md | 3 +- cpp/tests/write.cpp | 5 + python/README.md | 8 ++ python/src/fixedwidth/_native.py | 8 ++ python/src/fixedwidth/writer.py | 55 +++++++-- python/tests/test_write.py | 47 ++++++++ src/FixedWidthParser.Native/Exports.Writer.cs | 17 +++ src/FixedWidthParser.Native/README.md | 6 +- .../Writing/CallbackWriteStream.cs | 98 ++++++++++++++++ .../Writing/WriteApi.cs | 60 ++++++++++ .../include/fixedwidth.def | 1 + .../include/fixedwidth.h | 13 +++ .../include/fixedwidth.hpp | 37 ++++++ tests/FixedWidthParser.NativeSmoke/smoke.c | 10 ++ .../Native/NativeWriteTests.cs | 107 ++++++++++++++++++ 16 files changed, 469 insertions(+), 9 deletions(-) create mode 100644 src/FixedWidthParser.Native/Writing/CallbackWriteStream.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 248e9f6..4175656 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,9 @@ Windows-1252 after registering `CodePagesEncodingProvider`). (`fixedwidth.hpp`) that also maps structs to columns through `FixedWidthMapper`. It installs as a CMake package (`find_package(fixedwidth)`) or through FetchContent, and downloads the release's native library for the platform, so it needs no .NET SDK. +- Writing to a sink: `fw_write_stream` pushes the lines, in order, to an `fw_sink`'s `write` callback while + batches format in parallel, so memory stays at a batch per thread. Python's `fw.write` takes any binary stream + (an open file, `gzip.open(..., "wb")`), as fast as writing to a path; C++ has `write_stream(std::ostream&, ...)`. - Dictionary-encoded STRING columns for Arrow (`fw_column_spec.dictionary`, Python `fw.Column(dictionary=True)`, C++ `ColumnSpec::dictionary`): int32 indices into the column's distinct values, which pandas reads as `category` and polars as `Categorical`. `fw_table` and `fw.read` keep strings; categorical columns write like diff --git a/cpp/README.md b/cpp/README.md index 0cd6486..1384ae7 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -63,7 +63,8 @@ the end. `fw::layout_for()` returns the `fw::Layout` the bindings describe, f `fw::Reader::open_stream`, `TypedReader::open_stream` and `fw::read_arrow_stream` take a `std::istream&` and read it forward a batch at a time, in constant memory, so a decompressing or network stream works. The stream -must outlive the reader; a stream that turns `bad()` fails the read. +must outlive the reader; a stream that turns `bad()` fails the read. `fw::write_stream(std::ostream&, table, +layout)` writes the same way, in order, a batch at a time. `ColumnSpec::dictionary` hands a String column to Arrow (`read_arrow_*`) dictionary-encoded: int32 indices into its distinct values. `fw::Table` keeps the strings. diff --git a/cpp/tests/write.cpp b/cpp/tests/write.cpp index 2be652f..a776564 100644 --- a/cpp/tests/write.cpp +++ b/cpp/tests/write.cpp @@ -1,6 +1,7 @@ #include "common.hpp" #include +#include #include int main() @@ -22,6 +23,10 @@ int main() auto crlf = fw::write_memory(*table, layout, {.newline = "\r\n"}); CHECK(crlf.has_value() && crlf->size() == expected.size() + 3); + std::ostringstream out; + CHECK(fw::write_stream(out, *table, layout, {.degree_of_parallelism = 4, .batch_rows = 1}).has_value()); + CHECK(out.str() == std::string(serial->begin(), serial->end())); + const std::string path = "fw_cpp_write.txt"; CHECK(fw::write_file(path, *table, layout).has_value()); std::FILE* file = std::fopen(path.c_str(), "rb"); diff --git a/python/README.md b/python/README.md index 3ba9379..9b78674 100644 --- a/python/README.md +++ b/python/README.md @@ -104,8 +104,16 @@ columns. ```python fw.write(df, "out.txt", layout, fw.WriteOptions(degree_of_parallelism=0)) data = fw.write_bytes(df, layout) + +with gzip.open("out.txt.gz", "wb") as stream: # or an open file, a socket's makefile("wb"), ... + fw.write(df, stream, layout) ``` +`fw.write` takes a path or a binary stream with `write()`. A stream gets the lines in order, a batch per call on +the calling thread, while the next batches format in parallel, so memory stays at a batch per thread; writing +to an open file is as fast as writing to its path. For the bytes themselves, `write_bytes` beats an `io.BytesIO`, +which copies itself each time it grows. + Takes pandas, polars, pyarrow or anything with `__arrow_c_stream__` (needs pyarrow), matching columns by name. `padding="0"` writes zero-padded numbers with the sign leftmost; a null writes as its column filled with `trim_char`. Parallel writes are byte-identical to serial ones. `fw.WriteOptions` also takes `encoding`, diff --git a/python/src/fixedwidth/_native.py b/python/src/fixedwidth/_native.py index 61f7947..99951cc 100644 --- a/python/src/fixedwidth/_native.py +++ b/python/src/fixedwidth/_native.py @@ -82,6 +82,13 @@ class NativeStream(ctypes.Structure): ] +SinkWrite = ctypes.CFUNCTYPE(ctypes.c_int32, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64) + + +class Sink(ctypes.Structure): + _fields_ = [("struct_size", ctypes.c_int32), ("user_data", ctypes.c_void_p), ("write", SinkWrite)] + + class Column(ctypes.Structure): _fields_ = [ ("type", ctypes.c_int32), @@ -197,6 +204,7 @@ class ArrowArrayStream(ctypes.Structure): "fw_read_arrow_stream_stream": (ctypes.c_int32, [_P(NativeStream), ctypes.c_void_p, _P(ReaderOptions), _P(ArrowArrayStream)]), "fw_write_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, _P(Table), ctypes.c_void_p, _P(WriteOptions)]), "fw_write_memory": (ctypes.c_int32, [_P(Table), ctypes.c_void_p, _P(WriteOptions), _P(Buffer)]), + "fw_write_stream": (ctypes.c_int32, [_P(Table), ctypes.c_void_p, _P(WriteOptions), _P(Sink)]), "fw_free_buffer": (None, [_P(Buffer)]), } diff --git a/python/src/fixedwidth/writer.py b/python/src/fixedwidth/writer.py index 2c28d44..31678a2 100644 --- a/python/src/fixedwidth/writer.py +++ b/python/src/fixedwidth/writer.py @@ -1,13 +1,16 @@ -"""Writing: a pandas, polars or pyarrow table through a layout, to a file or to bytes. Requires pyarrow.""" +"""Writing: a pandas, polars or pyarrow table through a layout, to a file, a stream or bytes. Requires pyarrow.""" from __future__ import annotations import ctypes +import io +import operator import os -from typing import Any +from typing import IO, Any from fixedwidth import _native from fixedwidth.layout import Layout, check +from fixedwidth.reader import _fail from fixedwidth.types import ( Column, ColumnType, @@ -17,18 +20,56 @@ ) -def write(data: Any, path: str | os.PathLike[str], layout: Layout, options: WriteOptions | None = None) -> None: - """Writes every row of `data` (pandas, polars, pyarrow, or anything with `__arrow_c_stream__`) to `path`. +def write(data: Any, destination: str | os.PathLike[str] | IO[bytes], layout: Layout, options: WriteOptions | None = None) -> None: + """Writes every row of `data` (pandas, polars, pyarrow, or anything with `__arrow_c_stream__`) to `destination`: + a path, or a binary stream with `write()` (an open file, `gzip.open(..., "wb")`, a socket's `makefile("wb")`). - Columns are matched to the layout by name. A null writes as its column filled with `trim_char`.""" + A stream gets the lines in order, a batch per `write()` call on this thread, while batches format in parallel; + memory stays at a batch per thread. Columns are matched to the layout by name. A null writes as its column + filled with `trim_char`.""" lib = _native.load_library() table, keep = _native_table(data, layout) raw, newline = _write_options(options) - name, name_len = _native.utf8(os.fspath(path)) - check(lib.fw_write_file(name, name_len, ctypes.byref(table), layout.handle, ctypes.byref(raw) if raw else None)) + raw_ref = ctypes.byref(raw) if raw else None + if isinstance(destination, (str, os.PathLike)): + name, name_len = _native.utf8(os.fspath(destination)) + check(lib.fw_write_file(name, name_len, ctypes.byref(table), layout.handle, raw_ref)) + else: + sink, callback = _sink(destination) + check(lib.fw_write_stream(ctypes.byref(table), layout.handle, raw_ref, ctypes.byref(sink))) + del callback del keep, newline +def _sink(stream: Any) -> tuple[_native.Sink, object]: + """An fw_sink over `stream`'s write(); keep the callback alive until the native call returns.""" + # The standard streams copy or write a buffer before write() returns, so they get the native memory itself; + # anything else gets a copy it may keep. + zero_copy = isinstance(stream, io.IOBase) + + def write(_user_data: int, buffer: int, length: int) -> int: + try: + view = memoryview((ctypes.c_ubyte * length).from_address(buffer)).cast("B") + data = view if zero_copy else bytes(view) + done = 0 + while done < length: + count = stream.write(data[done:]) + if count is None: + if isinstance(stream, io.RawIOBase): + raise OSError("the stream would block; writing needs a blocking stream") + break # a write() that returns nothing took everything + count = operator.index(count) + if count <= 0: + raise OSError("the stream accepted no bytes") + done += count + return 0 + except BaseException as error: # noqa: BLE001 - every failure must reach the native side + return _fail(error) + + callback = _native.SinkWrite(write) + return _native.Sink(ctypes.sizeof(_native.Sink), None, callback), callback + + def write_bytes(data: Any, layout: Layout, options: WriteOptions | None = None) -> bytes: """Same as `write()`, returning the bytes.""" lib = _native.load_library() diff --git a/python/tests/test_write.py b/python/tests/test_write.py index 62b6487..27de360 100644 --- a/python/tests/test_write.py +++ b/python/tests/test_write.py @@ -115,3 +115,50 @@ def test_write_mismatched_columns_raise(): with pytest.raises(fw.FixedWidthError, match="b"): fw.write_bytes(pa.table({"a": ["x"]}), columns) + + +def test_write_to_streams_equals_write_bytes(tmp_path, cotahist, options): + import gzip + import io + + table = fw.read_arrow(cotahist, layout(), options) + batched = fw.WriteOptions(degree_of_parallelism=4, batch_rows=1) + expected = fw.write_bytes(table, layout()) + + buffer = io.BytesIO() + fw.write(table, buffer, layout(), batched) + with gzip.open(tmp_path / "out.txt.gz", "wb") as compressed: + fw.write(table, compressed, layout(), batched) + + assert buffer.getvalue() == expected + assert gzip.decompress((tmp_path / "out.txt.gz").read_bytes()) == expected + + +class Chunks: + """A duck-typed sink that keeps every chunk it is given and returns nothing from write().""" + + def __init__(self) -> None: + self.chunks: list[object] = [] + + def write(self, data) -> None: + self.chunks.append(data) + + +def test_write_to_a_duck_typed_sink_gets_copies_it_may_keep(cotahist, options): + table = fw.read_arrow(cotahist, layout(), options) + sink = Chunks() + + fw.write(table, sink, layout(), fw.WriteOptions(degree_of_parallelism=2, batch_rows=1)) + + assert all(isinstance(chunk, bytes) for chunk in sink.chunks) + assert b"".join(sink.chunks) == fw.write_bytes(table, layout()) + + +class Full: + def write(self, data) -> int: + raise OSError("no space left on device") + + +def test_a_failing_sink_reports_its_error(cotahist, options): + with pytest.raises(fw.FixedWidthError, match="no space left on device"): + fw.write(fw.read_arrow(cotahist, layout(), options), Full(), layout()) diff --git a/src/FixedWidthParser.Native/Exports.Writer.cs b/src/FixedWidthParser.Native/Exports.Writer.cs index 0073bdf..aa03a60 100644 --- a/src/FixedWidthParser.Native/Exports.Writer.cs +++ b/src/FixedWidthParser.Native/Exports.Writer.cs @@ -27,6 +27,23 @@ public static int WriteMemory(NativeTableRaw* table, nint layout, NativeWriteOpt return WithLayout(layout, l => WriteApi.WriteMemory(table, l, options, outBuffer)); } + [UnmanagedCallersOnly(EntryPoint = "fw_write_stream")] + public static int WriteStream(NativeTableRaw* table, nint layout, NativeWriteOptionsRaw* options, NativeSinkRaw* sink) + { + if (table is null) + { + return InvalidArgument("fw_write_stream needs a table."); + } +#pragma warning disable CA2000 // Disposed by the using below; the sink has nothing to release. + if (!CallbackWriteStream.TryCreate(sink, out var created)) + { + return NativeStatus.InvalidArgument; + } +#pragma warning restore CA2000 + using var stream = created; + return WithLayout(layout, l => WriteApi.WriteStream(table, l, options, stream)); + } + [UnmanagedCallersOnly(EntryPoint = "fw_free_buffer")] public static void FreeBuffer(NativeBufferRaw* buffer) { diff --git a/src/FixedWidthParser.Native/README.md b/src/FixedWidthParser.Native/README.md index 994ef94..d53b0e5 100644 --- a/src/FixedWidthParser.Native/README.md +++ b/src/FixedWidthParser.Native/README.md @@ -37,7 +37,7 @@ struct starts with `struct_size`; `NULL` means all defaults. | Batches | `fw_reader_open_file`, `fw_reader_open_memory`, `fw_reader_open_stream`, `fw_reader_next`, `fw_reader_close` | | Arrow | `fw_read_arrow_file`, `fw_read_arrow_memory`, `fw_read_arrow_stream_file`, `fw_read_arrow_stream_memory`, `fw_read_arrow_stream_stream` | | Streams | `fw_set_source_error` | -| Writes | `fw_write_file`, `fw_write_memory`, `fw_free_buffer` | +| Writes | `fw_write_file`, `fw_write_memory`, `fw_write_stream`, `fw_free_buffer` | A batched read or an Arrow stream can pull from an `fw_stream` instead of a file or memory: a `read` callback that fills a buffer and returns the count, 0 at the end or -1 after `fw_set_source_error`, and an optional @@ -45,6 +45,10 @@ that fills a buffer and returns the count, 0 at the end or -1 after `fw_set_sour pipe, a socket or a decompressor works; the batches stay in constant memory. Whole-table reads take a file or memory, since they split the source into byte ranges. +`fw_write_stream` pushes the lines to an `fw_sink` instead: a `write` callback that takes every byte it is +given and returns 0, or -1 after `fw_set_source_error`. It is called in order from the calling thread, a batch at +a time, while batches format in parallel, so memory stays at a batch per thread. + `include/fixedwidth.def` lists them for import-library generation on Windows; a test fails when it drifts from the `[UnmanagedCallersOnly]` entry points. diff --git a/src/FixedWidthParser.Native/Writing/CallbackWriteStream.cs b/src/FixedWidthParser.Native/Writing/CallbackWriteStream.cs new file mode 100644 index 0000000..49ce0ae --- /dev/null +++ b/src/FixedWidthParser.Native/Writing/CallbackWriteStream.cs @@ -0,0 +1,98 @@ +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native.Writing +{ + /// fw_sink. + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct NativeSinkRaw + { + public int StructSize; + public void* UserData; + public delegate* unmanaged Write; + } + + /// The caller's fw_sink as a write-only : every byte handed to it, in order. + internal sealed unsafe class CallbackWriteStream : Stream + { + private readonly delegate* unmanaged _write; + private readonly void* _userData; + + private CallbackWriteStream(NativeSinkRaw raw) + { + _write = raw.Write; + _userData = raw.UserData; + } + + /// Wraps , or explains why it cannot. + internal static bool TryCreate(NativeSinkRaw* raw, [NotNullWhen(true)] out CallbackWriteStream? stream) + { + stream = null; + if (raw is null || raw->StructSize != sizeof(NativeSinkRaw) || raw->Write is null) + { + NativeApi.SetLastError("fw_write_stream needs a non-NULL sink whose struct_size is sizeof(fw_sink) and whose write is set."); + return false; + } + stream = new CallbackWriteStream(*raw); + return true; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Write(ReadOnlySpan buffer) + { + if (buffer.IsEmpty) + { + return; + } + SourceErrors.Clear(); + int status; + fixed (byte* source = buffer) + { + status = _write(_userData, source, buffer.Length); + } + if (status != 0) + { + throw SourceErrors.Failed("fw_sink.write", -1, buffer.Length); + } + } + + public override void Write(byte[] buffer, int offset, int count) + { + Write(buffer.AsSpan(offset, count)); + } + + public override void Flush() + { + // Every write has reached the sink when its callback returns. + } + + public override int Read(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + } +} diff --git a/src/FixedWidthParser.Native/Writing/WriteApi.cs b/src/FixedWidthParser.Native/Writing/WriteApi.cs index bc28f8c..351b13c 100644 --- a/src/FixedWidthParser.Native/Writing/WriteApi.cs +++ b/src/FixedWidthParser.Native/Writing/WriteApi.cs @@ -60,6 +60,66 @@ internal static int WriteFile(ReadOnlySpan path, NativeTableRaw* table, Na } } + /// + /// Writes to in order, from the calling thread. A window of one batch per thread formats + /// in parallel, then goes to the sink, so memory stays at a window of batches whatever the table's size. + /// + // ponytail: the sink waits while a window formats, and the window while the sink writes; overlapping them + // with two windows would hide the slower of the two. + internal static int WriteStream(NativeTableRaw* table, NativeLayout layout, NativeWriteOptionsRaw* options, Stream sink) + { + int status = Prepare(table, layout, options, out var formatter, out var settings, out long lineBytes); + if (status != NativeStatus.Ok) + { + return status; + } + long rows = formatter!.Rows; + long batchRows = Math.Max(1, Math.Min(settings!.BatchRows, int.MaxValue / lineBytes)); + long batches = (rows + batchRows - 1) / batchRows; + int window = (int)Math.Min(settings.DegreeOfParallelism, Math.Max(1, batches)); + var buffers = new byte[window][]; + try + { + foreach (ref byte[] buffer in buffers.AsSpan()) + { + buffer = ArrayPool.Shared.Rent((int)(Math.Min(batchRows, rows) * lineBytes)); + } + for (long first = 0; first < batches; first += window) + { + int count = (int)Math.Min(window, batches - first); + Parallel.For(0, count, new ParallelOptions { MaxDegreeOfParallelism = window }, i => + { + long row = (first + i) * batchRows; + FormatBatch(formatter, settings, layout, row, buffers[i].AsSpan(0, (int)(Math.Min(batchRows, rows - row) * lineBytes))); + }); + for (int i = 0; i < count; i++) + { + long row = (first + i) * batchRows; + sink.Write(buffers[i], 0, (int)(Math.Min(batchRows, rows - row) * lineBytes)); + } + } + sink.Flush(); + return NativeStatus.Ok; + } + catch (AggregateException ex) + { + NativeApi.SetLastError(ex.InnerExceptions[0].Message); + return NativeStatus.Error; + } + catch (Exception ex) + { + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + finally + { + foreach (byte[]? buffer in buffers.Where(b => b is not null)) + { + ArrayPool.Shared.Return(buffer); + } + } + } + internal static void FreeBuffer(NativeBufferRaw* buffer) { if (buffer is null) diff --git a/src/FixedWidthParser.Native/include/fixedwidth.def b/src/FixedWidthParser.Native/include/fixedwidth.def index 2f09346..6ca0717 100644 --- a/src/FixedWidthParser.Native/include/fixedwidth.def +++ b/src/FixedWidthParser.Native/include/fixedwidth.def @@ -21,3 +21,4 @@ EXPORTS fw_set_source_error fw_write_file fw_write_memory + fw_write_stream diff --git a/src/FixedWidthParser.Native/include/fixedwidth.h b/src/FixedWidthParser.Native/include/fixedwidth.h index b43a8d8..6f3627f 100644 --- a/src/FixedWidthParser.Native/include/fixedwidth.h +++ b/src/FixedWidthParser.Native/include/fixedwidth.h @@ -225,6 +225,19 @@ int32_t fw_write_memory(const fw_table* table, const fw_layout* layout, const fw_write_options* options, fw_buffer* out_buffer); void fw_free_buffer(fw_buffer* buffer); +/* A destination the library pushes the lines to, in order: a pipe, a socket, a compressor, a language's own + * stream objects. write consumes all len bytes and returns 0, or -1 after fw_set_source_error to say why. It is + * called from the thread that called fw_write_stream, a batch per call, while batches format in parallel. */ +typedef struct fw_sink { + int32_t struct_size; + void* user_data; + int32_t (*write)(void* user_data, const uint8_t* buf, int64_t len); +} fw_sink; + +/* The same lines as fw_write_file, to a sink; memory stays at one batch per thread whatever the table's size. */ +int32_t fw_write_stream(const fw_table* table, const fw_layout* layout, const fw_write_options* options, + const fw_sink* sink); + #ifdef __cplusplus } #endif diff --git a/src/FixedWidthParser.Native/include/fixedwidth.hpp b/src/FixedWidthParser.Native/include/fixedwidth.hpp index 750b7da..79d2cf6 100644 --- a/src/FixedWidthParser.Native/include/fixedwidth.hpp +++ b/src/FixedWidthParser.Native/include/fixedwidth.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -445,6 +446,42 @@ namespace fw return write_memory(table.get(), layout, options); } + namespace detail + { + // fw_sink.write over a std::ostream: -1 once the stream is bad, or if it throws. + inline int32_t write_ostream(void* user_data, const uint8_t* buffer, int64_t length) noexcept + { + constexpr std::string_view failed = "the std::ostream failed"; + try + { + auto& out = *static_cast(user_data); + out.write(reinterpret_cast(buffer), static_cast(length)); + if (!out.bad()) + { + return 0; + } + } + catch (...) + { + } + fw_set_source_error(bytes(failed), static_cast(failed.size())); + return -1; + } + } + + /// Writes to `out` in order, a batch at a time, while batches format in parallel; memory stays at a batch per thread. + inline Result write_stream(std::ostream& out, const fw_table& table, const Layout& layout, const WriteOptions& options = {}) + { + const fw_write_options raw = detail::write_options(options); + const fw_sink sink{.struct_size = sizeof(fw_sink), .user_data = &out, .write = &detail::write_ostream}; + return detail::check(fw_write_stream(&table, layout.get(), &raw, &sink)); + } + + inline Result write_stream(std::ostream& out, const Table& table, const Layout& layout, const WriteOptions& options = {}) + { + return write_stream(out, table.get(), layout, options); + } + // ---- Struct mapping ---------------------------------------------------------------------------- // // Specialize FixedWidthMapper with a `static constexpr auto get_bindings()` returning a std::tuple of diff --git a/tests/FixedWidthParser.NativeSmoke/smoke.c b/tests/FixedWidthParser.NativeSmoke/smoke.c index 4a38799..aab6319 100644 --- a/tests/FixedWidthParser.NativeSmoke/smoke.c +++ b/tests/FixedWidthParser.NativeSmoke/smoke.c @@ -39,6 +39,11 @@ FW_STATIC_ASSERT(offsetof(fw_column_spec, format) == 40, spec_format); FW_STATIC_ASSERT(offsetof(fw_column_spec, overflow) == 68, spec_overflow); FW_STATIC_ASSERT(offsetof(fw_column_spec, dictionary) == 72, spec_dictionary); FW_STATIC_ASSERT(sizeof(fw_column_spec) == 80, spec_size); +FW_STATIC_ASSERT(offsetof(fw_stream, user_data) == 8, stream_user_data); +FW_STATIC_ASSERT(offsetof(fw_stream, read) == 16, stream_read); +FW_STATIC_ASSERT(sizeof(fw_stream) == 32, stream_size); +FW_STATIC_ASSERT(offsetof(fw_sink, write) == 16, sink_write); +FW_STATIC_ASSERT(sizeof(fw_sink) == 24, sink_size); FW_STATIC_ASSERT(sizeof(fw_layout_options) == 12, layout_options_size); FW_STATIC_ASSERT(offsetof(fw_read_options, header_lines) == 8, read_header_lines); FW_STATIC_ASSERT(offsetof(fw_read_options, skip_byte_order_mark) == 32, read_bom); @@ -137,6 +142,11 @@ static void load(const char* path) symbol(lib, "fw_reader_open_file"); symbol(lib, "fw_read_arrow_file"); symbol(lib, "fw_read_arrow_stream_file"); + /* Exercised by the C++ tests through std::istream and std::ostream. */ + symbol(lib, "fw_reader_open_stream"); + symbol(lib, "fw_read_arrow_stream_stream"); + symbol(lib, "fw_set_source_error"); + symbol(lib, "fw_write_stream"); } /* A COTAHIST-like file: a header, quotes, an invalid line and a trailer. */ diff --git a/tests/FixedWidthParser.Tests/Native/NativeWriteTests.cs b/tests/FixedWidthParser.Tests/Native/NativeWriteTests.cs index 43a5685..75abb9e 100644 --- a/tests/FixedWidthParser.Tests/Native/NativeWriteTests.cs +++ b/tests/FixedWidthParser.Tests/Native/NativeWriteTests.cs @@ -208,6 +208,113 @@ public void Write_File_EqualsMemory() } } + [Theory] + [InlineData(1, 0)] + [InlineData(4, 1)] + [InlineData(4, 7)] + [InlineData(16, 3)] + public void WriteStream_EqualsMemory(int degree, long batchRows) + { + var layout = EveryTypeLayout(); + var table = ReadTable(Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat(EveryType, 25))), layout); + try + { + var options = WriteOptions(degree, batchRows); + using var sink = new MemoryStream(); + Assert.Equal(NativeStatus.Ok, WriteApi.WriteStream(&table, layout, &options, sink)); + + Assert.Equal(WriteMemory(&table, layout).Bytes, sink.ToArray()); + } + finally + { + NativeTable.Free(&table); + } + } + + private sealed class Collected + { + public readonly List Bytes = []; + public int Calls; + public int FailAfter = int.MaxValue; + } + + [UnmanagedCallersOnly] + private static int SinkWrite(void* userData, byte* buffer, long length) + { + var collected = (Collected)GCHandle.FromIntPtr((IntPtr)userData).Target!; + if (++collected.Calls > collected.FailAfter) + { + SourceErrors.Set("the disk is full"); + return -1; + } + collected.Bytes.AddRange(new ReadOnlySpan(buffer, (int)length).ToArray()); + return 0; + } + + private static int WriteThroughSink(NativeTableRaw* table, NativeLayout layout, NativeWriteOptionsRaw options, Collected collected) + { + var handle = GCHandle.Alloc(collected); + try + { + var raw = new NativeSinkRaw { StructSize = sizeof(NativeSinkRaw), UserData = (void*)GCHandle.ToIntPtr(handle), Write = &SinkWrite }; + Assert.True(CallbackWriteStream.TryCreate(&raw, out var sink)); + using (sink) + { + return WriteApi.WriteStream(table, layout, &options, sink); + } + } + finally + { + handle.Free(); + } + } + + [Fact] + public void WriteStream_CallbackSink_GetsEveryBatchInOrder() + { + var layout = EveryTypeLayout(); + var table = ReadTable(Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat(EveryType, 10))), layout); + try + { + var collected = new Collected(); + Assert.Equal(NativeStatus.Ok, WriteThroughSink(&table, layout, WriteOptions(degree: 4, batchRows: 3), collected)); + + Assert.Equal(WriteMemory(&table, layout).Bytes, collected.Bytes); + Assert.Equal(7, collected.Calls); + } + finally + { + NativeTable.Free(&table); + } + } + + [Fact] + public void WriteStream_FailingSink_ReportsItsMessage() + { + var layout = EveryTypeLayout(); + var table = ReadTable(Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat(EveryType, 10))), layout); + try + { + var collected = new Collected { FailAfter = 2 }; + Assert.Equal(NativeStatus.Error, WriteThroughSink(&table, layout, WriteOptions(degree: 4, batchRows: 3), collected)); + + Assert.Equal("the disk is full", LastError()); + Assert.Equal(3, collected.Calls); + } + finally + { + NativeTable.Free(&table); + } + } + + [Fact] + public void WriteStream_SinkWithoutWrite_IsRejected() + { + var raw = new NativeSinkRaw { StructSize = sizeof(NativeSinkRaw) }; + Assert.False(CallbackWriteStream.TryCreate(&raw, out _)); + Assert.Contains("sizeof(fw_sink)", LastError(), StringComparison.Ordinal); + } + [Fact] public void Write_Latin1_EncodesText() { From 85fd766d0fc80965c3fcac43362ede6755daada3 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Mon, 26 Oct 2026 10:39:15 -0300 Subject: [PATCH 85/85] chore: update package references and suppress additional warnings --- Directory.Build.props | 22 +++++++++---------- .../FixedWidthAsyncRecordEnumerable.cs | 4 ++-- tests/Benchmarks/Benchmarks.csproj | 1 + 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 92a410e..677892e 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -2,8 +2,8 @@ latest All - - + + 9999 $(NoWarn);CA1062;CA1008;1591;RS0016;RS0037;RS0026;S1244;SS002;SS008;SS019;MA0016;CA1819;S6964;NU1902;NU1901;IDE0008;IDE0046;IDE0058;CA1515;CA1002 @@ -19,19 +19,19 @@ $(DefineConstants.Replace("TRACE;", "")) - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all @@ -39,15 +39,15 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all @@ -63,11 +63,11 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs b/src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs index e311615..0de00f7 100644 --- a/src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs +++ b/src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs @@ -34,9 +34,9 @@ internal FixedWidthAsyncRecordEnumerable(TParser parser, string path, FixedWidth _options = options; } -#pragma warning disable HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type +#pragma warning disable HLQ006, VSTHRD200 // GetEnumerator() or GetAsyncEnumerator() should return a value type public AsyncRecordEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) -#pragma warning restore HLQ006 +#pragma warning restore HLQ006, VSTHRD200 { #pragma warning disable CA2000, IDISP001 // The enumerator owns and disposes the file stream. var stream = _stream ?? new FileStream(_path!, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1, FileOptions.Asynchronous | FileOptions.SequentialScan); diff --git a/tests/Benchmarks/Benchmarks.csproj b/tests/Benchmarks/Benchmarks.csproj index 4f246c0..3b3140c 100644 --- a/tests/Benchmarks/Benchmarks.csproj +++ b/tests/Benchmarks/Benchmarks.csproj @@ -6,6 +6,7 @@ enable enable true + $(NoWarn);CA1707